Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 141 additions & 0 deletions internal/schedule/execution_checkpoint_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package schedule

import (
"testing"
"time"

"agentflow/internal/domain"
"agentflow/internal/repo"
"agentflow/internal/tracker"
)

func TestExecutionCheckpointsDispatchedRunsBeforeNextDispatch(t *testing.T) {
st, err := repo.OpenTest()
if err != nil {
t.Fatal(err)
}
t.Cleanup(st.Close)
for _, id := range []string{"issue-a", "issue-b"} {
if err := st.PutIssue(domain.Issue{ID: id, Title: id, ProjectID: "platform", Status: "todo"}); err != nil {
t.Fatal(err)
}
}
job := domain.ScheduleJob{ID: "job", ProjectID: "platform", WorkflowID: "wf", Enabled: true}
if err := st.PutScheduleJob(job); err != nil {
t.Fatal(err)
}
svc := New(st, tracker.NewService(), nil)
blocked, release := make(chan struct{}), make(chan struct{})
done := make(chan error, 1)
calls := 0
var firstIssue string
svc.SetRunDispatcher(func(_ domain.ScheduleJob, issue domain.Issue, _ string) (string, error) {
calls++
if calls == 1 {
firstIssue = issue.ID
} else {
close(blocked)
<-release
}
return "run-" + issue.ID, nil
})
go func() {
_, err := svc.RunNow(job.ID)
done <- err
}()
t.Cleanup(func() {
close(release)
select {
case err := <-done:
if err != nil {
t.Errorf("dispatch completion: %v", err)
}
case <-time.After(5 * time.Second):
t.Error("dispatch did not stop")
}
})
select {
case <-blocked:
case <-time.After(5 * time.Second):
t.Fatal("second dispatch did not start")
}
stored, _ := st.ScheduleJob(job.ID)
id := domain.ScheduleJobExecutionID(stored.LastTraceID)
execution, ok := st.ScheduleJobExecution(id)
if !ok || execution.Status != "running" || len(execution.Runs) != 1 || execution.Runs[0].RunID != "run-"+firstIssue || execution.Pulled != 2 || execution.Dispatched != 1 {
t.Fatalf("running execution lost a successful dispatch: %+v", execution)
}
// Recovery reads only durable execution data, as a successor process would.
if err := st.FailRunningScheduleJobExecutions(time.Now().UTC().Add(time.Second), interruptedReason); err != nil {
t.Fatal(err)
}
recovered, _ := st.ScheduleJobExecution(id)
if recovered.Status != "failed" || len(recovered.Runs) != 1 || recovered.Runs[0].RunID != execution.Runs[0].RunID || recovered.Dispatched != 1 {
t.Fatalf("recovery lost dispatched run attribution: %+v", recovered)
}
}

func TestExecutionStopsDispatchingWhenRunCheckpointFails(t *testing.T) {
st, err := repo.OpenTest()
if err != nil {
t.Fatal(err)
}
t.Cleanup(st.Close)
for _, id := range []string{"issue-a", "issue-b"} {
if err := st.PutIssue(domain.Issue{ID: id, ProjectID: "platform", Status: "todo"}); err != nil {
t.Fatal(err)
}
}
job := domain.ScheduleJob{ID: "job", ProjectID: "platform", WorkflowID: "wf", Enabled: true}
if err := st.PutScheduleJob(job); err != nil {
t.Fatal(err)
}
// The initial execution insert and terminal failure update still work.
if err := st.DB.Exec(`CREATE TRIGGER reject_run_checkpoint BEFORE UPDATE ON schedule_job_executions
WHEN NEW.status = 'running' BEGIN SELECT RAISE(FAIL, 'checkpoint unavailable'); END`).Error; err != nil {
t.Fatal(err)
}
svc := New(st, tracker.NewService(), nil)
calls := 0
svc.SetRunDispatcher(func(_ domain.ScheduleJob, issue domain.Issue, _ string) (string, error) {
calls++
return "run-" + issue.ID, nil
})
result, err := svc.RunNow(job.ID)
if err == nil || calls != 1 || result.LastStatus != "failed" || result.LastDispatched != 1 {
t.Fatalf("checkpoint failure allowed further dispatch: calls=%d result=%+v err=%v", calls, result, err)
}
execution, _ := st.ScheduleJobExecution(domain.ScheduleJobExecutionID(result.LastTraceID))
if execution.Status != "failed" || execution.Dispatched != 1 || len(execution.Runs) != 1 || execution.Runs[0].RunID == "" {
t.Fatalf("failed checkpoint lost the completed dispatch: %+v", execution)
}
}

func TestExecutionCheckpointsCountOnlyTheFinishedExecution(t *testing.T) {
st, err := repo.OpenTest()
if err != nil {
t.Fatal(err)
}
t.Cleanup(st.Close)
for _, id := range []string{"issue-a", "issue-b"} {
if err := st.PutIssue(domain.Issue{ID: id, ProjectID: "platform", Status: "todo"}); err != nil {
t.Fatal(err)
}
}
job := domain.ScheduleJob{ID: "job", ProjectID: "platform", WorkflowID: "wf", Enabled: true}
if err := st.PutScheduleJob(job); err != nil {
t.Fatal(err)
}
svc := New(st, tracker.NewService(), nil)
svc.SetRunDispatcher(func(_ domain.ScheduleJob, issue domain.Issue, _ string) (string, error) {
return "run-" + issue.ID, nil
})
result, err := svc.RunNow(job.ID)
if err != nil || result.LastDispatched != 2 {
t.Fatalf("dispatch result = %+v, %v", result, err)
}
current, _ := st.ScheduleJobExecutionOverview([]string{job.ID}, time.Now().UTC())
if current.Total != 1 || current.Finished != 1 || current.Success != 1 || current.Running != 0 {
t.Fatalf("intermediate checkpoints affected execution counts: %+v", current)
}
}
13 changes: 12 additions & 1 deletion internal/schedule/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,10 +361,21 @@ func (s *Service) executeJob(job domain.ScheduleJob, triggerOrigin string, exec
job.Blockers = nil
return finish(job, err)
}
dispatched++
if exec != nil {
exec.Runs = append(exec.Runs, domain.ScheduleJobExecutionRun{IssueID: issue.ID, IssueTitle: issue.Title, RunID: runID})
exec.Pulled, exec.Dispatched = job.LastPulled, dispatched
// The downstream run already exists. Save its attribution before the
// next dispatch can block or the process can be interrupted.
if err := s.repo.PutScheduleJobExecution(*exec); err != nil {
log.Printf("schedule job %s trace=%s: record dispatched run: %v", job.ID, exec.TraceID, err)
job.LastStatus = "failed"
job.LastError = "保存已派发运行记录失败"
job.LastDispatched = dispatched
job.Blockers = nil
return finish(job, errors.New(job.LastError))
}
}
dispatched++
}
job.LastDispatched = dispatched
if len(blocked) > 0 {
Expand Down