fix(webapp): converge the post-deploy engine re-sync when markers keep moving

The single re-read left a window where a second override during the
same deploy could be overwritten by a stale re-sync. The deploy now
re-reads and re-syncs until the markers stop moving (bounded), and any
marker moving after the final read is healed by that actor's own
engine sync.

Author: matt-aitken

apps/webapp/app/v3/services/createBackgroundWorker.server.ts

@@ -818,30 +818,43 @@ async function createWorkerQueue(
   /**
    * The optimistic markers only guard the Postgres write; an override or reset can
    * still land between that write and the engine sync above, which would leave the
-   * engine holding this deploy's stale values. Re-read the markers and re-sync once
-   * from the fresh row when they moved: every actor writes Postgres before its own
-   * engine sync, so whoever syncs last is syncing the freshest row.
+   * engine holding this deploy's stale values. Re-read the markers and re-sync from
+   * the fresh row until they stop moving (bounded): every actor writes Postgres
+   * before its own engine sync, so re-syncing whatever is freshest converges. A
+   * marker moving after the final read is healed by that actor's own engine sync
+   * or the next deploy.
    */
-  const freshQueue = await prisma.taskQueue.findFirst({
-    where: { id: taskQueue.id },
-    select: {
-      name: true,
-      paused: true,
-      concurrencyLimit: true,
-      totalConcurrencyLimit: true,
-      concurrencyLimitOverriddenAt: true,
-      totalConcurrencyLimitOverriddenAt: true,
-    },
-  });
+  let syncedMarkers = {
+    concurrency: taskQueue.concurrencyLimitOverriddenAt?.getTime(),
+    total: taskQueue.totalConcurrencyLimitOverriddenAt?.getTime(),
+  };
+
+  for (let i = 0; i < 3; i++) {
+    const freshQueue = await prisma.taskQueue.findFirst({
+      where: { id: taskQueue.id },
+      select: {
+        name: true,
+        paused: true,
+        concurrencyLimit: true,
+        totalConcurrencyLimit: true,
+        concurrencyLimitOverriddenAt: true,
+        totalConcurrencyLimitOverriddenAt: true,
+      },
+    });
+
+    if (
+      !freshQueue ||
+      (freshQueue.concurrencyLimitOverriddenAt?.getTime() === syncedMarkers.concurrency &&
+        freshQueue.totalConcurrencyLimitOverriddenAt?.getTime() === syncedMarkers.total)
+    ) {
+      break;
+    }
 
-  if (
-    freshQueue &&
-    (freshQueue.concurrencyLimitOverriddenAt?.getTime() !==
-      taskQueue.concurrencyLimitOverriddenAt?.getTime() ||
-      freshQueue.totalConcurrencyLimitOverriddenAt?.getTime() !==
-        taskQueue.totalConcurrencyLimitOverriddenAt?.getTime())
-  ) {
     await syncQueueLimitsToEngine(freshQueue);
+    syncedMarkers = {
+      concurrency: freshQueue.concurrencyLimitOverriddenAt?.getTime(),
+      total: freshQueue.totalConcurrencyLimitOverriddenAt?.getTime(),
+    };
   }
 
   return taskQueue;