Skip to content

Commit a7575c7

Browse files
d-csTrigger.dev RepoOps
authored andcommitted
fix(run-engine, run-store): treat finished runs with expired snapshot state as already finished
Once a finished run's execution snapshot state has aged out of the Redis snapshot store, operations that only needed to know the run had finished were failing instead of completing: cancelling the run, an idempotent `triggerAndWait` that hit the cached run, a late waitpoint completion, the queue repair tool, the dev TTL job, and a stale queue message. The store now reports expiry with a distinct, non-retriable error, and each of those paths treats a finished run whose snapshot state has expired as already finished, using the run record. Expired state on a run that is not finished still fails closed. Mono-RevId: ee089985af8a7c4e6bc1ecbac38554dcd7ed53b2
1 parent b50d739 commit a7575c7

15 files changed

Lines changed: 2242 additions & 55 deletions

‎internal-packages/run-engine/src/engine/index.ts‎

Lines changed: 82 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -78,13 +78,14 @@ import {
7878
ExecutionSnapshotSystem,
7979
getExecutionSnapshotsSince,
8080
getLatestExecutionSnapshot,
81+
getLatestExecutionSnapshotOrFinishedRun,
8182
} from "./systems/executionSnapshotSystem.js";
8283
import {
8384
PARKED_ON_EXTERNAL_DEPLOYMENT_STATUS_REASON,
8485
PendingVersionSystem,
8586
} from "./systems/pendingVersionSystem.js";
8687
import { RaceSimulationSystem } from "./systems/raceSimulationSystem.js";
87-
import { RunAttemptSystem } from "./systems/runAttemptSystem.js";
88+
import { type CancelRunResult, RunAttemptSystem } from "./systems/runAttemptSystem.js";
8889
import { NoopPendingVersionRunIdLookup } from "./services/pendingVersionLookup.js";
8990
import type { SystemResources } from "./systems/systems.js";
9091
import {
@@ -1753,7 +1754,7 @@ export class RunEngine {
17531754
finalizeRun?: boolean;
17541755
bulkActionId?: string;
17551756
tx?: PrismaClientOrTransaction;
1756-
}): Promise<ExecutionResult & { alreadyFinished: boolean }> {
1757+
}): Promise<CancelRunResult> {
17571758
return this.runAttemptSystem.cancelRun({
17581759
runId,
17591760
organizationId,
@@ -2718,13 +2719,35 @@ export class RunEngine {
27182719
}
27192720

27202721
async #repairRun(runId: string, dryRun: boolean, organizationId: string) {
2721-
const snapshot = await getLatestExecutionSnapshot(
2722+
const { snapshot, finishedRun } = await getLatestExecutionSnapshotOrFinishedRun(
27222723
this.prisma,
27232724
runId,
27242725
this.runStore,
2725-
undefined,
2726-
organizationId
2726+
organizationId,
2727+
{ id: true }
27272728
);
2729+
if (finishedRun) {
2730+
// The run finished and its snapshot state expired, so any concurrency slot it holds has leaked.
2731+
this.logger.warn("RunEngine.repairRun: snapshot state expired, run already finished", {
2732+
runId,
2733+
});
2734+
2735+
if (!dryRun) {
2736+
await this.worker.enqueueOnce({
2737+
id: `repair-in-progress-run:${runId}`,
2738+
job: "repairSnapshot",
2739+
payload: { runId, snapshotId: null, executionStatus: "FINISHED" },
2740+
availableAt: new Date(Date.now() + this.repairSnapshotTimeoutMs),
2741+
});
2742+
}
2743+
2744+
return {
2745+
action: "repairSnapshot",
2746+
runId,
2747+
snapshotStatus: "FINISHED",
2748+
snapshotId: null,
2749+
};
2750+
}
27282751

27292752
if (
27302753
snapshot.executionStatus === "QUEUED" ||
@@ -3173,13 +3196,57 @@ export class RunEngine {
31733196
executionStatus,
31743197
}: {
31753198
runId: string;
3176-
snapshotId: string;
3199+
/** null when scheduled for a finished run whose snapshot state had already expired. */
3200+
snapshotId: string | null;
31773201
executionStatus: string;
31783202
}) {
31793203
return await this.runLock.lock("handleRepairSnapshot", [runId], async () => {
31803204
// No snapshot mirror in off-only mode, so the repair heals only the queue below. The M1+
31813205
// rebuild reintroduces a mirror-repair step here.
3182-
const latestSnapshot = await getLatestExecutionSnapshot(this.prisma, runId, this.runStore);
3206+
const { snapshot: latestSnapshot, finishedRun } =
3207+
await getLatestExecutionSnapshotOrFinishedRun(
3208+
this.prisma,
3209+
runId,
3210+
this.runStore,
3211+
undefined,
3212+
{
3213+
id: true,
3214+
queue: true,
3215+
concurrencyKey: true,
3216+
runtimeEnvironmentId: true,
3217+
}
3218+
);
3219+
if (finishedRun) {
3220+
// The run is finished whichever snapshot the job was queued from (a live one that has since
3221+
// finished and expired, or none when queued after expiry), so its slot has leaked either way.
3222+
this.logger.warn("RunEngine.handleRepairSnapshot: snapshot state expired, run finished", {
3223+
runId,
3224+
snapshotId,
3225+
repairExecutionStatus: executionStatus,
3226+
});
3227+
3228+
const env = await this.controlPlaneResolver.resolveEnv(finishedRun.runtimeEnvironmentId);
3229+
if (!env) {
3230+
this.logger.error("RunEngine.handleRepairSnapshot FINISHED environment not found", {
3231+
runId,
3232+
});
3233+
return;
3234+
}
3235+
3236+
await this.#clearRunFromConcurrencySets({
3237+
runId,
3238+
orgId: env.organizationId,
3239+
queue: finishedRun.queue,
3240+
concurrencyKey: finishedRun.concurrencyKey ?? undefined,
3241+
env: {
3242+
id: env.id,
3243+
type: env.type,
3244+
project: { id: env.projectId },
3245+
organization: { id: env.organizationId },
3246+
},
3247+
});
3248+
return;
3249+
}
31833250

31843251
if (latestSnapshot.id !== snapshotId) {
31853252
this.logger.log(
@@ -3264,8 +3331,7 @@ export class RunEngine {
32643331
return;
32653332
}
32663333

3267-
// We need to clear this run from the current concurrency sets
3268-
await this.runQueue.clearMessageFromConcurrencySets({
3334+
await this.#clearRunFromConcurrencySets({
32693335
runId,
32703336
orgId: latestSnapshot.organizationId,
32713337
queue: taskRun.queue,
@@ -3291,6 +3357,13 @@ export class RunEngine {
32913357
});
32923358
}
32933359

3360+
async #clearRunFromConcurrencySets(
3361+
params: Parameters<RunQueue["clearMessageFromConcurrencySets"]>[0]
3362+
) {
3363+
// We need to clear this run from the current concurrency sets
3364+
await this.runQueue.clearMessageFromConcurrencySets(params);
3365+
}
3366+
32943367
/**
32953368
* Applies `defaultMaxTtl` to a run's TTL:
32963369
* - No max configured → pass through as-is.

‎internal-packages/run-engine/src/engine/systems/dequeueSystem.ts‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@ import { getMachinePreset } from "../machinePresets.js";
2424
import { isDequeueableExecutionStatus, isExecuting } from "../statuses.js";
2525
import type { WorkerQueueSelection } from "../../run-queue/types.js";
2626
import type { RunEngineOptions, WorkerQueueDequeueOptions } from "../types.js";
27-
import type { ExecutionSnapshotSystem } from "./executionSnapshotSystem.js";
28-
import { getLatestExecutionSnapshot } from "./executionSnapshotSystem.js";
27+
import {
28+
getLatestExecutionSnapshotOrFinishedRun,
29+
type ExecutionSnapshotSystem,
30+
} from "./executionSnapshotSystem.js";
2931
import type { RunAttemptSystem } from "./runAttemptSystem.js";
3032
import type { SystemResources } from "./systems.js";
3133
import { snapshotRouteFromSnapshot, type SnapshotRouteWire } from "@internal/run-store";
@@ -197,13 +199,18 @@ export class DequeueSystem {
197199
"dequeueFromWorkerQueue",
198200
[runId],
199201
async () => {
200-
const snapshot = await getLatestExecutionSnapshot(
202+
const { snapshot, finishedRun } = await getLatestExecutionSnapshotOrFinishedRun(
201203
prisma,
202204
runId,
203205
this.$.runStore,
204-
undefined,
205-
orgId
206+
orgId,
207+
{ id: true }
206208
);
209+
if (finishedRun) {
210+
// A stale message for a finished run: remove it instead of starting the run.
211+
await this.#acknowledgeExpiredFinishedRun(orgId, runId);
212+
return;
213+
}
207214
preparedFromSnapshotId = snapshot.id;
208215

209216
if (!isDequeueableExecutionStatus(snapshot.executionStatus)) {
@@ -758,11 +765,14 @@ export class DequeueSystem {
758765
// Reentrant: tryNackAndRequeue's own lock on this run nests inside this one.
759766
await this.$.runLock.lock("dequeueFromWorkerQueueRecovery", [runId], async () => {
760767
// Preparation may have committed a transition before throwing, so read current state.
761-
const [snapshotError, snapshot] = await tryCatch(
762-
getLatestExecutionSnapshot(prisma, runId, this.$.runStore)
768+
// A finished run's stale message is removed; any read error (including a failed row
769+
// read after expiry) falls through to the nack below.
770+
const [snapshotError, read] = await tryCatch(
771+
getLatestExecutionSnapshotOrFinishedRun(prisma, runId, this.$.runStore, undefined, {
772+
id: true,
773+
})
763774
);
764-
765-
if (snapshotError) {
775+
if (snapshotError || !read) {
766776
// Do not replace execution state we could not read with a checkpoint-less snapshot.
767777
const requeued = await this.$.runQueue.nackMessage({ orgId, messageId: runId });
768778
if (requeued === false) {
@@ -775,6 +785,11 @@ export class DequeueSystem {
775785
}
776786
return;
777787
}
788+
if (read.finishedRun) {
789+
await this.#acknowledgeExpiredFinishedRun(orgId, runId);
790+
return;
791+
}
792+
const snapshot = read.snapshot;
778793

779794
// A PENDING_EXECUTING snapshot written by this preparation was never handed to a worker,
780795
// so it is still ours to requeue. Any other non-dequeueable state is owned elsewhere.
@@ -847,6 +862,14 @@ export class DequeueSystem {
847862
);
848863
}
849864

865+
async #acknowledgeExpiredFinishedRun(orgId: string, runId: string): Promise<void> {
866+
this.$.logger.warn(
867+
"dequeueFromWorkerQueue: snapshot state expired for a finished run, removing from queue",
868+
{ runId, orgId }
869+
);
870+
await this.$.runQueue.acknowledgeMessage(orgId, runId);
871+
}
872+
850873
async #pendingVersion({
851874
orgId,
852875
runId,

‎internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@ import type {
1818
SnapshotReadContext,
1919
SnapshotRouteWire,
2020
} from "@internal/run-store";
21-
import { snapshotRouteFromSnapshot } from "@internal/run-store";
21+
import { SnapshotStateExpiredError, snapshotRouteFromSnapshot } from "@internal/run-store";
2222
import { ExecutionSnapshotNotFoundError, ServiceValidationError } from "../errors.js";
23+
import { isFinalRunStatus } from "../statuses.js";
2324
import type { HeartbeatTimeouts } from "../types.js";
2425
import type { SystemResources } from "./systems.js";
2526

@@ -213,6 +214,62 @@ async function fetchWaitpointsInChunks(
213214
return allWaitpoints;
214215
}
215216

217+
/**
218+
* True when a snapshot read failed because a redis-primary run's state aged out and the run row is
219+
* final. Snapshot state only gets a TTL from a FINISHED entry, so a final row means the run finished.
220+
* Expired state on a non-final row is lost state and must keep failing closed.
221+
*/
222+
export function isExpiredSnapshotOfFinishedRun(error: unknown, runStatus: TaskRunStatus): boolean {
223+
return error instanceof SnapshotStateExpiredError && isFinalRunStatus(runStatus);
224+
}
225+
226+
type FinishedRunRow<S extends Prisma.TaskRunSelect> = Prisma.TaskRunGetPayload<{
227+
select: S;
228+
}> & { status: TaskRunStatus };
229+
230+
export type LatestExecutionSnapshotOrFinishedRun<S extends Prisma.TaskRunSelect> =
231+
| { snapshot: EnhancedExecutionSnapshot; finishedRun: null }
232+
| { snapshot: null; finishedRun: FinishedRunRow<S> };
233+
234+
/**
235+
* Like `getLatestExecutionSnapshot`, but when the run's snapshot state has expired and its run row
236+
* is final, returns that row (selected with `runSelect` plus `status`) instead of throwing. Every
237+
* other error, and expired state on a non-final row, is rethrown unchanged.
238+
*/
239+
export async function getLatestExecutionSnapshotOrFinishedRun<S extends Prisma.TaskRunSelect>(
240+
prisma: PrismaClientOrTransaction,
241+
runId: string,
242+
runStore: RunStore,
243+
organizationId: SnapshotReadContext | undefined,
244+
runSelect: S
245+
): Promise<LatestExecutionSnapshotOrFinishedRun<S>> {
246+
try {
247+
const snapshot = await getLatestExecutionSnapshot(
248+
prisma,
249+
runId,
250+
runStore,
251+
undefined,
252+
organizationId
253+
);
254+
return { snapshot, finishedRun: null };
255+
} catch (error) {
256+
if (!(error instanceof SnapshotStateExpiredError)) {
257+
throw error;
258+
}
259+
// Prisma's select-payload mapping does not compose over an intersection, so the row is read with
260+
// `status` added and typed as the caller's selection plus `status`.
261+
const run = (await runStore.findRun(
262+
{ id: runId },
263+
{ select: { ...runSelect, status: true } },
264+
prisma
265+
)) as FinishedRunRow<S> | null;
266+
if (!run || !isExpiredSnapshotOfFinishedRun(error, run.status)) {
267+
throw error;
268+
}
269+
return { snapshot: null, finishedRun: run };
270+
}
271+
}
272+
216273
/**
217274
* Gets the most recent valid snapshot for a run. When `environmentId` is provided the read is scoped
218275
* to that environment (tenant boundary): a run in another environment reads as not-found and rejects

‎internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,18 @@ import type {
6666
import {
6767
executionResultFromSnapshot,
6868
getLatestExecutionSnapshot,
69+
getLatestExecutionSnapshotOrFinishedRun,
6970
} from "./executionSnapshotSystem.js";
7071
import type { SystemResources } from "./systems.js";
7172
import { snapshotRouteFromSnapshot, type SnapshotRouteWire } from "@internal/run-store";
7273
import type { WaitpointSystem } from "./waitpointSystem.js";
7374
import { BatchId, RunId } from "@trigger.dev/core/v3/isomorphic";
7475

76+
/** A run whose snapshot state has expired has no snapshot to report, only its final run row. */
77+
export type CancelRunResult =
78+
| (ExecutionResult & { alreadyFinished: boolean })
79+
| { alreadyFinished: true; run: ExecutionResult["run"]; snapshot: null };
80+
7581
export type RunAttemptSystemOptions = {
7682
resources: SystemResources;
7783
executionSnapshotSystem: ExecutionSnapshotSystem;
@@ -1060,6 +1066,12 @@ export class RunAttemptSystem {
10601066
attemptDurationMs: completion.usage?.durationMs,
10611067
tx: prisma,
10621068
});
1069+
if (result.snapshot === null) {
1070+
// Not reachable for a run with a live attempt.
1071+
throw new Error(
1072+
`cancelRun returned no snapshot for run ${runId} with a live attempt`
1073+
);
1074+
}
10631075
return {
10641076
attemptStatus:
10651077
result.snapshot.executionStatus === "PENDING_CANCEL"
@@ -1604,18 +1616,26 @@ export class RunAttemptSystem {
16041616
bulkActionId?: string;
16051617
attemptDurationMs?: number;
16061618
tx?: PrismaClientOrTransaction;
1607-
}): Promise<ExecutionResult & { alreadyFinished: boolean }> {
1619+
}): Promise<CancelRunResult> {
16081620
const prisma = tx ?? this.$.prisma;
16091621

16101622
return startSpan(this.$.tracer, "cancelRun", async (span) => {
1611-
return this.$.runLock.lock("cancelRun", [runId], async () => {
1612-
const latestSnapshot = await getLatestExecutionSnapshot(
1613-
prisma,
1614-
runId,
1615-
this.$.runStore,
1616-
undefined,
1617-
organizationId
1618-
);
1623+
return this.$.runLock.lock("cancelRun", [runId], async (): Promise<CancelRunResult> => {
1624+
const { snapshot: latestSnapshot, finishedRun } =
1625+
await getLatestExecutionSnapshotOrFinishedRun(
1626+
prisma,
1627+
runId,
1628+
this.$.runStore,
1629+
organizationId,
1630+
{ id: true, friendlyId: true, attemptNumber: true, taskEventStore: true }
1631+
);
1632+
if (finishedRun) {
1633+
if (bulkActionId) {
1634+
await this.$.runStore.recordBulkActionMembership(runId, bulkActionId, prisma);
1635+
}
1636+
this.$.logger.warn("cancelRun: snapshot state expired, run already finished", { runId });
1637+
return { alreadyFinished: true, run: finishedRun, snapshot: null };
1638+
}
16191639

16201640
//already finished, do nothing
16211641
if (latestSnapshot.executionStatus === "FINISHED") {

0 commit comments

Comments
 (0)