@@ -78,13 +78,14 @@ import {
7878 ExecutionSnapshotSystem ,
7979 getExecutionSnapshotsSince ,
8080 getLatestExecutionSnapshot ,
81+ getLatestExecutionSnapshotOrFinishedRun ,
8182} from "./systems/executionSnapshotSystem.js" ;
8283import {
8384 PARKED_ON_EXTERNAL_DEPLOYMENT_STATUS_REASON ,
8485 PendingVersionSystem ,
8586} from "./systems/pendingVersionSystem.js" ;
8687import { RaceSimulationSystem } from "./systems/raceSimulationSystem.js" ;
87- import { RunAttemptSystem } from "./systems/runAttemptSystem.js" ;
88+ import { type CancelRunResult , RunAttemptSystem } from "./systems/runAttemptSystem.js" ;
8889import { NoopPendingVersionRunIdLookup } from "./services/pendingVersionLookup.js" ;
8990import type { SystemResources } from "./systems/systems.js" ;
9091import {
@@ -1753,7 +1754,7 @@ export class RunEngine {
17531754 finalizeRun ?: boolean ;
17541755 bulkActionId ?: string ;
17551756 tx ?: PrismaClientOrTransaction ;
1756- } ) : Promise < ExecutionResult & { alreadyFinished : boolean } > {
1757+ } ) : Promise < CancelRunResult > {
17571758 return this . runAttemptSystem . cancelRun ( {
17581759 runId,
17591760 organizationId,
@@ -2718,13 +2719,35 @@ export class RunEngine {
27182719 }
27192720
27202721 async #repairRun( runId : string , dryRun : boolean , organizationId : string ) {
2721- const snapshot = await getLatestExecutionSnapshot (
2722+ const { snapshot, finishedRun } = await getLatestExecutionSnapshotOrFinishedRun (
27222723 this . prisma ,
27232724 runId ,
27242725 this . runStore ,
2725- undefined ,
2726- organizationId
2726+ organizationId ,
2727+ { id : true }
27272728 ) ;
2729+ if ( finishedRun ) {
2730+ // The run finished and its snapshot state expired, so any concurrency slot it holds has leaked.
2731+ this . logger . warn ( "RunEngine.repairRun: snapshot state expired, run already finished" , {
2732+ runId,
2733+ } ) ;
2734+
2735+ if ( ! dryRun ) {
2736+ await this . worker . enqueueOnce ( {
2737+ id : `repair-in-progress-run:${ runId } ` ,
2738+ job : "repairSnapshot" ,
2739+ payload : { runId, snapshotId : null , executionStatus : "FINISHED" } ,
2740+ availableAt : new Date ( Date . now ( ) + this . repairSnapshotTimeoutMs ) ,
2741+ } ) ;
2742+ }
2743+
2744+ return {
2745+ action : "repairSnapshot" ,
2746+ runId,
2747+ snapshotStatus : "FINISHED" ,
2748+ snapshotId : null ,
2749+ } ;
2750+ }
27282751
27292752 if (
27302753 snapshot . executionStatus === "QUEUED" ||
@@ -3173,13 +3196,57 @@ export class RunEngine {
31733196 executionStatus,
31743197 } : {
31753198 runId : string ;
3176- snapshotId : string ;
3199+ /** null when scheduled for a finished run whose snapshot state had already expired. */
3200+ snapshotId : string | null ;
31773201 executionStatus : string ;
31783202 } ) {
31793203 return await this . runLock . lock ( "handleRepairSnapshot" , [ runId ] , async ( ) => {
31803204 // No snapshot mirror in off-only mode, so the repair heals only the queue below. The M1+
31813205 // rebuild reintroduces a mirror-repair step here.
3182- const latestSnapshot = await getLatestExecutionSnapshot ( this . prisma , runId , this . runStore ) ;
3206+ const { snapshot : latestSnapshot , finishedRun } =
3207+ await getLatestExecutionSnapshotOrFinishedRun (
3208+ this . prisma ,
3209+ runId ,
3210+ this . runStore ,
3211+ undefined ,
3212+ {
3213+ id : true ,
3214+ queue : true ,
3215+ concurrencyKey : true ,
3216+ runtimeEnvironmentId : true ,
3217+ }
3218+ ) ;
3219+ if ( finishedRun ) {
3220+ // The run is finished whichever snapshot the job was queued from (a live one that has since
3221+ // finished and expired, or none when queued after expiry), so its slot has leaked either way.
3222+ this . logger . warn ( "RunEngine.handleRepairSnapshot: snapshot state expired, run finished" , {
3223+ runId,
3224+ snapshotId,
3225+ repairExecutionStatus : executionStatus ,
3226+ } ) ;
3227+
3228+ const env = await this . controlPlaneResolver . resolveEnv ( finishedRun . runtimeEnvironmentId ) ;
3229+ if ( ! env ) {
3230+ this . logger . error ( "RunEngine.handleRepairSnapshot FINISHED environment not found" , {
3231+ runId,
3232+ } ) ;
3233+ return ;
3234+ }
3235+
3236+ await this . #clearRunFromConcurrencySets( {
3237+ runId,
3238+ orgId : env . organizationId ,
3239+ queue : finishedRun . queue ,
3240+ concurrencyKey : finishedRun . concurrencyKey ?? undefined ,
3241+ env : {
3242+ id : env . id ,
3243+ type : env . type ,
3244+ project : { id : env . projectId } ,
3245+ organization : { id : env . organizationId } ,
3246+ } ,
3247+ } ) ;
3248+ return ;
3249+ }
31833250
31843251 if ( latestSnapshot . id !== snapshotId ) {
31853252 this . logger . log (
@@ -3264,8 +3331,7 @@ export class RunEngine {
32643331 return ;
32653332 }
32663333
3267- // We need to clear this run from the current concurrency sets
3268- await this . runQueue . clearMessageFromConcurrencySets ( {
3334+ await this . #clearRunFromConcurrencySets( {
32693335 runId,
32703336 orgId : latestSnapshot . organizationId ,
32713337 queue : taskRun . queue ,
@@ -3291,6 +3357,13 @@ export class RunEngine {
32913357 } ) ;
32923358 }
32933359
3360+ async #clearRunFromConcurrencySets(
3361+ params : Parameters < RunQueue [ "clearMessageFromConcurrencySets" ] > [ 0 ]
3362+ ) {
3363+ // We need to clear this run from the current concurrency sets
3364+ await this . runQueue . clearMessageFromConcurrencySets ( params ) ;
3365+ }
3366+
32943367 /**
32953368 * Applies `defaultMaxTtl` to a run's TTL:
32963369 * - No max configured → pass through as-is.
0 commit comments