From d6ec115348d0581fc2e6729298db7f31c776d1d6 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Tue, 7 Apr 2026 16:11:31 -0700 Subject: [PATCH 01/15] v0.6.29: login improvements, posthog telemetry (#4026) * feat(posthog): Add tracking on mothership abort (#4023) Co-authored-by: Theodore Li * fix(login): fix captcha headers for manual login (#4025) * fix(signup): fix turnstile key loading * fix(login): fix captcha header passing * Catch user already exists, remove login form captcha --- apps/sim/app/(auth)/signup/signup-form.tsx | 11 +++-------- .../app/workspace/[workspaceId]/home/home.tsx | 12 ++++++++++-- .../w/[workflowId]/components/panel/panel.tsx | 19 ++++++++++++++++++- apps/sim/lib/posthog/events.ts | 5 +++++ 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/apps/sim/app/(auth)/signup/signup-form.tsx b/apps/sim/app/(auth)/signup/signup-form.tsx index 55a0508ec1b..afb27cd729a 100644 --- a/apps/sim/app/(auth)/signup/signup-form.tsx +++ b/apps/sim/app/(auth)/signup/signup-form.tsx @@ -270,10 +270,8 @@ function SignupFormContent({ name: sanitizedName, }, { - fetchOptions: { - headers: { - ...(token ? { 'x-captcha-response': token } : {}), - }, + headers: { + ...(token ? { 'x-captcha-response': token } : {}), }, onError: (ctx) => { logger.error('Signup error:', ctx.error) @@ -282,10 +280,7 @@ function SignupFormContent({ let errorCode = 'unknown' if (ctx.error.code?.includes('USER_ALREADY_EXISTS')) { errorCode = 'user_already_exists' - errorMessage.push( - 'An account with this email already exists. Please sign in instead.' - ) - setEmailError(errorMessage[0]) + setEmailError('An account with this email already exists. Please sign in instead.') } else if ( ctx.error.code?.includes('BAD_REQUEST') || ctx.error.message?.includes('Email and password sign up is not enabled') diff --git a/apps/sim/app/workspace/[workspaceId]/home/home.tsx b/apps/sim/app/workspace/[workspaceId]/home/home.tsx index d76f17ff454..38367339197 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/home.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/home.tsx @@ -223,6 +223,14 @@ export function Home({ chatId }: HomeProps = {}) { posthogRef.current = posthog }, [posthog]) + const handleStopGeneration = useCallback(() => { + captureEvent(posthogRef.current, 'task_generation_aborted', { + workspace_id: workspaceId, + view: 'mothership', + }) + stopGeneration() + }, [stopGeneration, workspaceId]) + const handleSubmit = useCallback( (text: string, fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[]) => { const trimmed = text.trim() @@ -334,7 +342,7 @@ export function Home({ chatId }: HomeProps = {}) { defaultValue={initialPrompt} onSubmit={handleSubmit} isSending={isSending} - onStopGeneration={stopGeneration} + onStopGeneration={handleStopGeneration} userId={session?.user?.id} onContextAdd={handleContextAdd} /> @@ -359,7 +367,7 @@ export function Home({ chatId }: HomeProps = {}) { isSending={isSending} isReconnecting={isReconnecting} onSubmit={handleSubmit} - onStopGeneration={stopGeneration} + onStopGeneration={handleStopGeneration} messageQueue={messageQueue} onRemoveQueuedMessage={removeFromQueue} onSendQueuedMessage={sendNow} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx index 4d485c763ce..da51910789b 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx @@ -4,6 +4,7 @@ import { memo, useCallback, useEffect, useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { History, Plus, Square } from 'lucide-react' import { useParams, useRouter } from 'next/navigation' +import { usePostHog } from 'posthog-js/react' import { useShallow } from 'zustand/react/shallow' import { BubbleChatClose, @@ -33,6 +34,7 @@ import { import { Lock, Unlock, Upload } from '@/components/emcn/icons' import { VariableIcon } from '@/components/icons' import { useSession } from '@/lib/auth/auth-client' +import { captureEvent } from '@/lib/posthog/client' import { generateWorkflowJson } from '@/lib/workflows/operations/import-export' import { ConversationListItem } from '@/app/workspace/[workspaceId]/components' import { MothershipChat } from '@/app/workspace/[workspaceId]/home/components' @@ -101,6 +103,9 @@ export const Panel = memo(function Panel({ workspaceId: propWorkspaceId }: Panel const params = useParams() const workspaceId = propWorkspaceId ?? (params.workspaceId as string) + const posthog = usePostHog() + const posthogRef = useRef(posthog) + const panelRef = useRef(null) const fileInputRef = useRef(null) const { activeTab, setActiveTab, panelWidth, _hasHydrated, setHasHydrated } = usePanelStore( @@ -264,6 +269,10 @@ export const Panel = memo(function Panel({ workspaceId: propWorkspaceId }: Panel loadCopilotChats() }, [loadCopilotChats]) + useEffect(() => { + posthogRef.current = posthog + }, [posthog]) + const handleCopilotSelectChat = useCallback((chat: { id: string; title: string | null }) => { setCopilotChatId(chat.id) setCopilotChatTitle(chat.title) @@ -394,6 +403,14 @@ export const Panel = memo(function Panel({ workspaceId: propWorkspaceId }: Panel [copilotEditQueuedMessage] ) + const handleCopilotStopGeneration = useCallback(() => { + captureEvent(posthogRef.current, 'task_generation_aborted', { + workspace_id: workspaceId, + view: 'copilot', + }) + copilotStopGeneration() + }, [copilotStopGeneration, workspaceId]) + const handleCopilotSubmit = useCallback( (text: string, fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[]) => { const trimmed = text.trim() @@ -833,7 +850,7 @@ export const Panel = memo(function Panel({ workspaceId: propWorkspaceId }: Panel isSending={copilotIsSending} isReconnecting={copilotIsReconnecting} onSubmit={handleCopilotSubmit} - onStopGeneration={copilotStopGeneration} + onStopGeneration={handleCopilotStopGeneration} messageQueue={copilotMessageQueue} onRemoveQueuedMessage={copilotRemoveFromQueue} onSendQueuedMessage={copilotSendNow} diff --git a/apps/sim/lib/posthog/events.ts b/apps/sim/lib/posthog/events.ts index 537a9864282..faf9895bf62 100644 --- a/apps/sim/lib/posthog/events.ts +++ b/apps/sim/lib/posthog/events.ts @@ -378,6 +378,11 @@ export interface PostHogEventMap { workspace_id: string } + task_generation_aborted: { + workspace_id: string + view: 'mothership' | 'copilot' + } + task_message_sent: { workspace_id: string has_attachments: boolean From bd68eeb90ad44181db598b6704c83d8694a7290b Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Thu, 16 Jul 2026 18:33:16 -0700 Subject: [PATCH 02/15] feat(ci): promote Trigger.dev tasks in lockstep with the ECS traffic cutover --- .github/scripts/wait-for-ecs-cutover.sh | 105 +++++++++ .github/workflows/ci.yml | 287 ++++++++++++++++++++++++ 2 files changed, 392 insertions(+) create mode 100755 .github/scripts/wait-for-ecs-cutover.sh diff --git a/.github/scripts/wait-for-ecs-cutover.sh b/.github/scripts/wait-for-ecs-cutover.sh new file mode 100755 index 00000000000..4ec90125804 --- /dev/null +++ b/.github/scripts/wait-for-ecs-cutover.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Waits for the ECS blue/green deploy triggered by a specific app image push to +# reach its traffic cutover (CodeDeploy AllowTraffic == Succeeded), then exits 0. +# +# ECR app images use a floating tag (latest/staging) with no git SHA, so the +# only durable key linking this CI push to its ECS deploy is the image DIGEST. +# Correlation: image digest -> CodePipeline execution (AppEcrImage revision) -> +# Deploy action externalExecutionId (== CodeDeploy deployment id) -> AllowTraffic. +# +# Usage: wait-for-ecs-cutover.sh +# Requires: awscli v2, configured credentials with codedeploy + codepipeline read. +set -euo pipefail + +PIPELINE="${1:?pipeline name required}" +DIGEST="${2:?image digest required}" + +POLL_INTERVAL="${POLL_INTERVAL:-15}" +# 70 min covers a prod deploy whose Deploy stage is queued behind a prior +# deploy's ~50-min termination bake before its own traffic shift begins. +OVERALL_TIMEOUT="${OVERALL_TIMEOUT:-4200}" + +deadline=$(( $(date +%s) + OVERALL_TIMEOUT )) +remaining() { echo $(( deadline - $(date +%s) )); } +log() { echo "[wait-for-ecs-cutover] $*"; } +fail_if_expired() { + if [ "$(remaining)" -le 0 ]; then + log "ERROR: timed out after ${OVERALL_TIMEOUT}s waiting for: $1" + exit 1 + fi +} + +log "Pipeline: $PIPELINE" +log "Target app image digest: $DIGEST" + +# Phase A: find the pipeline execution whose ECR source revision matches our +# digest. --max-items bounds the fetch (the CLI otherwise auto-paginates the whole +# execution history); our push is the newest execution, so it's on the first page. +# The revisionId match is done server-side via JMESPath; grep isolates the UUID +# from any trailing pagination-token line in text output. +EXECUTION_ID="" +while [ -z "$EXECUTION_ID" ]; do + fail_if_expired "pipeline execution matching digest" + EXECUTION_ID=$(aws codepipeline list-pipeline-executions \ + --pipeline-name "$PIPELINE" --max-items 30 \ + --query "pipelineExecutionSummaries[?sourceRevisions[?actionName=='ECR_Source' && revisionId=='$DIGEST']].pipelineExecutionId" \ + --output text 2>/dev/null | tr '\t ' '\n\n' | grep -Em1 '^[0-9a-f-]{36}$' || true) + if [ -z "$EXECUTION_ID" ]; then + log "No matching pipeline execution yet; retry in ${POLL_INTERVAL}s (remaining $(remaining)s)" + sleep "$POLL_INTERVAL" + fi +done +log "Matched pipeline execution: $EXECUTION_ID" + +# Phase B: resolve the CodeDeploy deployment id from the Deploy action. This may +# stay empty for a while if the Deploy stage is queued behind a prior deploy. +DEPLOYMENT_ID="" +while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; do + fail_if_expired "CodeDeploy deployment id (Deploy stage may be queued behind a prior deploy's bake)" + status=$(aws codepipeline get-pipeline-execution \ + --pipeline-name "$PIPELINE" --pipeline-execution-id "$EXECUTION_ID" \ + --query 'pipelineExecution.status' --output text 2>/dev/null || true) + case "$status" in + Failed|Stopped|Superseded) + log "ERROR: pipeline execution $EXECUTION_ID ended in status $status before deploy" + exit 1 + ;; + esac + DEPLOYMENT_ID=$(aws codepipeline list-action-executions \ + --pipeline-name "$PIPELINE" \ + --filter pipelineExecutionId="$EXECUTION_ID" \ + --query "actionExecutionDetails[?stageName=='Deploy'].output.executionResult.externalExecutionId | [0]" \ + --output text 2>/dev/null || true) + if [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; then + log "Deploy stage not started yet (pipeline status: $status); retry in ${POLL_INTERVAL}s (remaining $(remaining)s)" + sleep "$POLL_INTERVAL" + fi +done +log "CodeDeploy deployment: $DEPLOYMENT_ID" + +# Phase C: wait for the traffic cutover (AllowTraffic lifecycle event Succeeded). +while true; do + fail_if_expired "AllowTraffic (traffic cutover)" + dstatus=$(aws deploy get-deployment --deployment-id "$DEPLOYMENT_ID" \ + --query 'deploymentInfo.status' --output text 2>/dev/null || true) + case "$dstatus" in + Failed|Stopped) + log "ERROR: CodeDeploy deployment $DEPLOYMENT_ID ended in status $dstatus; not promoting" + exit 1 + ;; + esac + target_id=$(aws deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \ + --query 'targetIds[0]' --output text 2>/dev/null || true) + at_status="" + if [ -n "$target_id" ] && [ "$target_id" != "None" ]; then + at_status=$(aws deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$target_id" \ + --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \ + --output text 2>/dev/null || true) + if [ "$at_status" = "Succeeded" ]; then + log "Traffic cutover complete (AllowTraffic Succeeded) for $DEPLOYMENT_ID" + exit 0 + fi + fi + log "Deployment $DEPLOYMENT_ID status=$dstatus AllowTraffic=${at_status:-pending}; wait ${POLL_INTERVAL}s (remaining $(remaining)s)" + sleep "$POLL_INTERVAL" +done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75d45ac5e46..3d0db39e88c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -268,6 +268,118 @@ jobs: fi bunx trigger.dev@4.5.12 deploy --env preview --branch dev-sim + # Staging: build & upload the Trigger.dev task version WITHOUT promoting it + # (--skip-promotion). New runs keep executing the old version until the promote + # job flips it at the ECS traffic cutover, so this deploy carries zero + # app<->task skew and runs in parallel with the image build. The captured + # version output is consumed by promote-trigger-staging. + deploy-trigger-staging: + name: Deploy Trigger.dev (Staging) + needs: [migrate] + if: github.event_name == 'push' && github.ref == 'refs/heads/staging' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 15 + outputs: + version: ${{ steps.deploy.outputs.version }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Deploy to Trigger.dev (skip promotion) + id: deploy + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + bunx trigger.dev@4.4.3 deploy --env staging --skip-promotion 2>&1 | tee deploy.log + # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation. + VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) + if [ -z "$VERSION" ]; then + echo "ERROR: could not parse deployed version from deploy output" >&2 + exit 1 + fi + echo "Captured deployed version: $VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + # Production: same skip-promotion deploy as staging, for the prod environment. + deploy-trigger-production: + name: Deploy Trigger.dev (Production) + needs: [migrate] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 15 + outputs: + version: ${{ steps.deploy.outputs.version }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Deploy to Trigger.dev (skip promotion) + id: deploy + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + bunx trigger.dev@4.4.3 deploy --env prod --skip-promotion 2>&1 | tee deploy.log + # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation. + VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) + if [ -z "$VERSION" ]; then + echo "ERROR: could not parse deployed version from deploy output" >&2 + exit 1 + fi + echo "Captured deployed version: $VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR. # Runs in parallel with tests — only immutable sha tags are pushed here, and # the CodePipeline EventBridge triggers filter on exactly the @@ -386,6 +498,7 @@ jobs: - name: Build and push images if: steps.meta.outputs.skip != 'true' + id: build uses: ./.github/actions/docker-build with: provider: ${{ vars.CI_PROVIDER }} @@ -394,6 +507,25 @@ jobs: tags: ${{ steps.meta.outputs.tags }} max-cache-size-mb: ${{ matrix.cache_mb }} + # Publish the app image digest so promote-trigger-* can correlate this push + # to its ECS CodePipeline execution. promote-images retags this same sha + # image to latest/staging (preserving the digest), so the pipeline's ECR + # source revision equals this digest — the only durable key (the deploy tag + # is floating). App leg only. + - name: Publish app image digest + if: matrix.ecr_repo_secret == 'ECR_APP' + run: | + mkdir -p digest + echo "${{ steps.build.outputs.digest }}" > digest/app-image-digest.txt + + - name: Upload app image digest + if: matrix.ecr_repo_secret == 'ECR_APP' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: app-image-digest + path: digest/app-image-digest.txt + retention-days: 1 + # Promote the sha-tagged ECR images to the deploy tags once tests and # migrations pass. Pushing the ECR latest/staging tag is what triggers # CodePipeline, so this seconds-long manifest retag is the deploy gate — @@ -415,6 +547,11 @@ jobs: permissions: contents: read id-token: write + outputs: + # Whether the deploy tag was actually moved (false on a stale-run guard + # skip). promote-trigger-* keys off this so tasks are never promoted when + # the app itself wasn't. + promoted: ${{ steps.guard.outputs.fresh }} steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 @@ -474,6 +611,156 @@ jobs: "${REGISTRY}/${repo}:${{ github.sha }}" done + # Staging: promote the skip-promoted Trigger.dev version at the exact moment the + # ECS app deploy shifts traffic (CodeDeploy AllowTraffic), so tasks and app cut + # over in lockstep. The promote-images retag above is what triggers the ECS + # pipeline; this job polls it (via the app image digest) and promotes at cutover. + # Skipped when promote-images skipped the tag move (stale run) — tasks then + # correctly stay on the old version. If the app deploy fails or never cuts over, + # promote never fires and this job fails visibly. + promote-trigger-staging: + name: Promote Trigger.dev (Staging) + needs: [promote-images, deploy-trigger-staging] + if: >- + github.event_name == 'push' && github.ref == 'refs/heads/staging' && + needs.promote-images.outputs.promoted == 'true' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 75 + permissions: + contents: read + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Download app image digest + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: app-image-digest + path: digest + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ secrets.STAGING_AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.STAGING_AWS_REGION }} + + - name: Wait for ECS traffic cutover + run: | + set -eo pipefail + DIGEST=$(cat digest/app-image-digest.txt) + bash .github/scripts/wait-for-ecs-cutover.sh sim-staging-us-east-1-app-deployment "$DIGEST" + + - name: Promote Trigger.dev version + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + VERSION: ${{ needs.deploy-trigger-staging.outputs.version }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + if [ -z "$VERSION" ]; then + echo "ERROR: no deployed version passed from deploy-trigger-staging" >&2 + exit 1 + fi + echo "Promoting Trigger.dev version $VERSION (staging) at ECS cutover" + bunx trigger.dev@4.4.3 promote "$VERSION" --env staging + + # Production: same lockstep promotion as staging, for the prod environment. + promote-trigger-production: + name: Promote Trigger.dev (Production) + needs: [promote-images, deploy-trigger-production] + if: >- + github.event_name == 'push' && github.ref == 'refs/heads/main' && + needs.promote-images.outputs.promoted == 'true' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 75 + permissions: + contents: read + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Download app image digest + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: app-image-digest + path: digest + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Wait for ECS traffic cutover + run: | + set -eo pipefail + DIGEST=$(cat digest/app-image-digest.txt) + bash .github/scripts/wait-for-ecs-cutover.sh sim-production-us-east-1-app-deployment "$DIGEST" + + - name: Promote Trigger.dev version + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + VERSION: ${{ needs.deploy-trigger-production.outputs.version }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + if [ -z "$VERSION" ]; then + echo "ERROR: no deployed version passed from deploy-trigger-production" >&2 + exit 1 + fi + echo "Promoting Trigger.dev version $VERSION (production) at ECS cutover" + bunx trigger.dev@4.4.3 promote "$VERSION" --env prod + # Build ARM64 images for GHCR (main branch only, runs in parallel with # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64 # are applied by create-ghcr-manifests after the gate, so a failing run From e1e136dc6a508383e1c4df175b54e7a53bfdf712 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Sun, 19 Jul 2026 12:17:49 -0700 Subject: [PATCH 03/15] =?UTF-8?q?fix(ci):=20harden=20Trigger.dev=20cutover?= =?UTF-8?q?=20gate=20=E2=80=94=20reject=20stale=20executions,=20verify=20a?= =?UTF-8?q?ll=20ECS=20targets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/scripts/wait-for-ecs-cutover.sh | 93 +++++++---- .github/workflows/ci.yml | 195 ++++++------------------ 2 files changed, 112 insertions(+), 176 deletions(-) diff --git a/.github/scripts/wait-for-ecs-cutover.sh b/.github/scripts/wait-for-ecs-cutover.sh index 4ec90125804..f7ea7970567 100755 --- a/.github/scripts/wait-for-ecs-cutover.sh +++ b/.github/scripts/wait-for-ecs-cutover.sh @@ -1,23 +1,32 @@ #!/usr/bin/env bash # Waits for the ECS blue/green deploy triggered by a specific app image push to -# reach its traffic cutover (CodeDeploy AllowTraffic == Succeeded), then exits 0. +# reach its traffic cutover (CodeDeploy AllowTraffic == Succeeded on every ECS +# target), then exits 0. # # ECR app images use a floating tag (latest/staging) with no git SHA, so the # only durable key linking this CI push to its ECS deploy is the image DIGEST. -# Correlation: image digest -> CodePipeline execution (AppEcrImage revision) -> +# Correlation: image digest -> CodePipeline execution (ECR_Source revision) -> # Deploy action externalExecutionId (== CodeDeploy deployment id) -> AllowTraffic. # -# Usage: wait-for-ecs-cutover.sh -# Requires: awscli v2, configured credentials with codedeploy + codepipeline read. +# The digest alone is ambiguous: a prior run with the same image could match an +# older, already-cutover execution and promote too early. SINCE_EPOCH (the time +# the deploy tag was retagged, i.e. when THIS push's pipeline was triggered) +# disambiguates — only an execution that started at/after the retag is ours. +# +# Usage: wait-for-ecs-cutover.sh +# Requires: awscli v2, python3, credentials with codedeploy + codepipeline read. set -euo pipefail PIPELINE="${1:?pipeline name required}" DIGEST="${2:?image digest required}" +SINCE_EPOCH="${3:?since-epoch (retag time) required}" POLL_INTERVAL="${POLL_INTERVAL:-15}" # 70 min covers a prod deploy whose Deploy stage is queued behind a prior # deploy's ~50-min termination bake before its own traffic shift begins. OVERALL_TIMEOUT="${OVERALL_TIMEOUT:-4200}" +# Tolerate minor clock skew between the runner (retag time) and CodePipeline. +SINCE_SKEW="${SINCE_SKEW:-120}" deadline=$(( $(date +%s) + OVERALL_TIMEOUT )) remaining() { echo $(( deadline - $(date +%s) )); } @@ -31,21 +40,44 @@ fail_if_expired() { log "Pipeline: $PIPELINE" log "Target app image digest: $DIGEST" +log "Requiring execution started at/after epoch $SINCE_EPOCH (minus ${SINCE_SKEW}s skew)" -# Phase A: find the pipeline execution whose ECR source revision matches our -# digest. --max-items bounds the fetch (the CLI otherwise auto-paginates the whole -# execution history); our push is the newest execution, so it's on the first page. -# The revisionId match is done server-side via JMESPath; grep isolates the UUID -# from any trailing pagination-token line in text output. +# Phase A: find the newest pipeline execution whose ECR source revision matches +# our digest AND that started at/after the retag. The since filter rejects a +# stale historical execution reusing the same digest. --max-items bounds the +# fetch (the CLI otherwise auto-paginates the whole history). EXECUTION_ID="" while [ -z "$EXECUTION_ID" ]; do - fail_if_expired "pipeline execution matching digest" - EXECUTION_ID=$(aws codepipeline list-pipeline-executions \ + fail_if_expired "pipeline execution matching digest since retag" + matches=$(aws codepipeline list-pipeline-executions \ --pipeline-name "$PIPELINE" --max-items 30 \ - --query "pipelineExecutionSummaries[?sourceRevisions[?actionName=='ECR_Source' && revisionId=='$DIGEST']].pipelineExecutionId" \ - --output text 2>/dev/null | tr '\t ' '\n\n' | grep -Em1 '^[0-9a-f-]{36}$' || true) + --query "pipelineExecutionSummaries[?sourceRevisions[?actionName=='ECR_Source' && revisionId=='$DIGEST']].[startTime, pipelineExecutionId]" \ + --output text 2>/dev/null || true) + EXECUTION_ID=$(printf '%s\n' "$matches" | SINCE="$SINCE_EPOCH" SKEW="$SINCE_SKEW" python3 -c ' +import sys, os, datetime +since = float(os.environ["SINCE"]) - float(os.environ["SKEW"]) +best_epoch = None +best_id = None +for line in sys.stdin: + parts = line.rstrip("\n").split("\t") + if len(parts) < 2: + continue + ts, eid = parts[0].strip(), parts[1].strip() + if not ts or not eid or "-" not in eid: + continue + try: + epoch = float(ts) + except ValueError: + try: + epoch = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp() + except ValueError: + continue + if epoch >= since and (best_epoch is None or epoch > best_epoch): + best_epoch, best_id = epoch, eid +print(best_id or "") +' 2>/dev/null || true) if [ -z "$EXECUTION_ID" ]; then - log "No matching pipeline execution yet; retry in ${POLL_INTERVAL}s (remaining $(remaining)s)" + log "No matching post-retag pipeline execution yet; retry in ${POLL_INTERVAL}s (remaining $(remaining)s)" sleep "$POLL_INTERVAL" fi done @@ -77,9 +109,11 @@ while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; do done log "CodeDeploy deployment: $DEPLOYMENT_ID" -# Phase C: wait for the traffic cutover (AllowTraffic lifecycle event Succeeded). +# Phase C: wait for the traffic cutover. Require AllowTraffic == Succeeded on +# EVERY ECS target, so a multi-target deploy can't promote while one target is +# still mid-cutover or failed. while true; do - fail_if_expired "AllowTraffic (traffic cutover)" + fail_if_expired "AllowTraffic (traffic cutover) on all targets" dstatus=$(aws deploy get-deployment --deployment-id "$DEPLOYMENT_ID" \ --query 'deploymentInfo.status' --output text 2>/dev/null || true) case "$dstatus" in @@ -88,18 +122,25 @@ while true; do exit 1 ;; esac - target_id=$(aws deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \ - --query 'targetIds[0]' --output text 2>/dev/null || true) - at_status="" - if [ -n "$target_id" ] && [ "$target_id" != "None" ]; then - at_status=$(aws deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$target_id" \ - --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \ - --output text 2>/dev/null || true) - if [ "$at_status" = "Succeeded" ]; then - log "Traffic cutover complete (AllowTraffic Succeeded) for $DEPLOYMENT_ID" + target_ids=$(aws deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \ + --query 'targetIds' --output text 2>/dev/null || true) + if [ -n "$target_ids" ] && [ "$target_ids" != "None" ]; then + all_ok=1 + ntargets=0 + for tid in $target_ids; do + ntargets=$((ntargets + 1)) + at_status=$(aws deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$tid" \ + --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \ + --output text 2>/dev/null || true) + if [ "$at_status" != "Succeeded" ]; then + all_ok=0 + fi + done + if [ "$ntargets" -gt 0 ] && [ "$all_ok" = "1" ]; then + log "Traffic cutover complete (AllowTraffic Succeeded on all $ntargets target(s)) for $DEPLOYMENT_ID" exit 0 fi fi - log "Deployment $DEPLOYMENT_ID status=$dstatus AllowTraffic=${at_status:-pending}; wait ${POLL_INTERVAL}s (remaining $(remaining)s)" + log "Deployment $DEPLOYMENT_ID status=$dstatus; not all targets past AllowTraffic; wait ${POLL_INTERVAL}s (remaining $(remaining)s)" sleep "$POLL_INTERVAL" done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d0db39e88c..f931ca1ecf8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -268,69 +268,19 @@ jobs: fi bunx trigger.dev@4.5.12 deploy --env preview --branch dev-sim - # Staging: build & upload the Trigger.dev task version WITHOUT promoting it - # (--skip-promotion). New runs keep executing the old version until the promote - # job flips it at the ECS traffic cutover, so this deploy carries zero - # app<->task skew and runs in parallel with the image build. The captured - # version output is consumed by promote-trigger-staging. - deploy-trigger-staging: - name: Deploy Trigger.dev (Staging) + # Main/staging: build & upload the Trigger.dev task version WITHOUT promoting it + # (--skip-promotion). New runs keep executing the OLD promoted version until + # promote-trigger flips it at the ECS traffic cutover — so the app cutting over + # never changes which task version runs until promote-trigger (which depends on + # this job) promotes the version uploaded here. Runs in parallel with the build; + # intentionally NOT gating the app deploy on it, to avoid coupling every app / + # realtime / pii / migration deploy to trigger.dev availability. + deploy-trigger: + name: Deploy Trigger.dev needs: [migrate] - if: github.event_name == 'push' && github.ref == 'refs/heads/staging' - runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 15 - outputs: - version: ${{ steps.deploy.outputs.version }} - steps: - - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - - name: Setup Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - with: - bun-version: 1.3.13 - - - name: Cache Bun dependencies - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: | - ~/.bun/install/cache - node_modules - **/node_modules - key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} - restore-keys: | - ${{ runner.os }}-bun- - - - name: Install dependencies - run: bun install --frozen-lockfile - - - name: Deploy to Trigger.dev (skip promotion) - id: deploy - working-directory: ./apps/sim - env: - TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} - TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} - run: | - set -eo pipefail - if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then - echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 - exit 1 - fi - bunx trigger.dev@4.4.3 deploy --env staging --skip-promotion 2>&1 | tee deploy.log - # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation. - VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) - if [ -z "$VERSION" ]; then - echo "ERROR: could not parse deployed version from deploy output" >&2 - exit 1 - fi - echo "Captured deployed version: $VERSION" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - # Production: same skip-promotion deploy as staging, for the prod environment. - deploy-trigger-production: - name: Deploy Trigger.dev (Production) - needs: [migrate] - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: >- + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 outputs: @@ -364,13 +314,14 @@ jobs: env: TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + TRIGGER_ENV: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }} run: | set -eo pipefail if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 exit 1 fi - bunx trigger.dev@4.4.3 deploy --env prod --skip-promotion 2>&1 | tee deploy.log + bunx trigger.dev@4.4.3 deploy --env "$TRIGGER_ENV" --skip-promotion 2>&1 | tee deploy.log # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation. VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) if [ -z "$VERSION" ]; then @@ -549,9 +500,13 @@ jobs: id-token: write outputs: # Whether the deploy tag was actually moved (false on a stale-run guard - # skip). promote-trigger-* keys off this so tasks are never promoted when + # skip). promote-trigger keys off this so tasks are never promoted when # the app itself wasn't. promoted: ${{ steps.guard.outputs.fresh }} + # Epoch when the deploy tag was retagged (this push's ECS pipeline trigger). + # promote-trigger passes it to the poll script so a stale pipeline execution + # reusing the same image digest can't satisfy the cutover gate. + retag_epoch: ${{ steps.promote.outputs.retag_epoch }} steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 @@ -581,6 +536,7 @@ jobs: fi - name: Promote images to deploy tags + id: promote if: steps.guard.outputs.fresh == 'true' env: ECR_REPOS: >- @@ -589,6 +545,11 @@ jobs: ${{ secrets.ECR_REALTIME }} ${{ secrets.ECR_PII }} run: | + # Record the retag time BEFORE moving any tag — this is when the ECS + # pipeline for this push is triggered. promote-trigger uses it to + # reject an older pipeline execution reusing the same image digest. + echo "retag_epoch=$(date +%s)" >> "$GITHUB_OUTPUT" + REGISTRY="${{ steps.login-ecr.outputs.registry }}" if [ "${{ github.ref }}" = "refs/heads/main" ]; then @@ -611,18 +572,20 @@ jobs: "${REGISTRY}/${repo}:${{ github.sha }}" done - # Staging: promote the skip-promoted Trigger.dev version at the exact moment the - # ECS app deploy shifts traffic (CodeDeploy AllowTraffic), so tasks and app cut - # over in lockstep. The promote-images retag above is what triggers the ECS - # pipeline; this job polls it (via the app image digest) and promotes at cutover. + # Main/staging: promote the skip-promoted Trigger.dev version at the exact moment + # the ECS app deploy shifts traffic (CodeDeploy AllowTraffic on every target), so + # tasks and app cut over in lockstep. The promote-images retag is what triggers + # the ECS pipeline; this job correlates it via the app image digest + retag epoch + # (rejecting a stale execution reusing the digest) and promotes at cutover. # Skipped when promote-images skipped the tag move (stale run) — tasks then # correctly stay on the old version. If the app deploy fails or never cuts over, # promote never fires and this job fails visibly. - promote-trigger-staging: - name: Promote Trigger.dev (Staging) - needs: [promote-images, deploy-trigger-staging] + promote-trigger: + name: Promote Trigger.dev + needs: [promote-images, deploy-trigger] if: >- - github.event_name == 'push' && github.ref == 'refs/heads/staging' && + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') && needs.promote-images.outputs.promoted == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 75 @@ -661,93 +624,25 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: - role-to-assume: ${{ secrets.STAGING_AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.STAGING_AWS_REGION }} + role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }} + aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }} - name: Wait for ECS traffic cutover - run: | - set -eo pipefail - DIGEST=$(cat digest/app-image-digest.txt) - bash .github/scripts/wait-for-ecs-cutover.sh sim-staging-us-east-1-app-deployment "$DIGEST" - - - name: Promote Trigger.dev version - working-directory: ./apps/sim env: - TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} - TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} - VERSION: ${{ needs.deploy-trigger-staging.outputs.version }} - run: | - set -eo pipefail - if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then - echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 - exit 1 - fi - if [ -z "$VERSION" ]; then - echo "ERROR: no deployed version passed from deploy-trigger-staging" >&2 - exit 1 - fi - echo "Promoting Trigger.dev version $VERSION (staging) at ECS cutover" - bunx trigger.dev@4.4.3 promote "$VERSION" --env staging - - # Production: same lockstep promotion as staging, for the prod environment. - promote-trigger-production: - name: Promote Trigger.dev (Production) - needs: [promote-images, deploy-trigger-production] - if: >- - github.event_name == 'push' && github.ref == 'refs/heads/main' && - needs.promote-images.outputs.promoted == 'true' - runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 75 - permissions: - contents: read - id-token: write - steps: - - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - - name: Setup Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - with: - bun-version: 1.3.13 - - - name: Cache Bun dependencies - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: | - ~/.bun/install/cache - node_modules - **/node_modules - key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} - restore-keys: | - ${{ runner.os }}-bun- - - - name: Install dependencies - run: bun install --frozen-lockfile - - - name: Download app image digest - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - name: app-image-digest - path: digest - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Wait for ECS traffic cutover + PIPELINE: sim-${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}-us-east-1-app-deployment + RETAG_EPOCH: ${{ needs.promote-images.outputs.retag_epoch }} run: | set -eo pipefail DIGEST=$(cat digest/app-image-digest.txt) - bash .github/scripts/wait-for-ecs-cutover.sh sim-production-us-east-1-app-deployment "$DIGEST" + bash .github/scripts/wait-for-ecs-cutover.sh "$PIPELINE" "$DIGEST" "$RETAG_EPOCH" - name: Promote Trigger.dev version working-directory: ./apps/sim env: TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} - VERSION: ${{ needs.deploy-trigger-production.outputs.version }} + TRIGGER_ENV: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }} + VERSION: ${{ needs.deploy-trigger.outputs.version }} run: | set -eo pipefail if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then @@ -755,11 +650,11 @@ jobs: exit 1 fi if [ -z "$VERSION" ]; then - echo "ERROR: no deployed version passed from deploy-trigger-production" >&2 + echo "ERROR: no deployed version passed from deploy-trigger" >&2 exit 1 fi - echo "Promoting Trigger.dev version $VERSION (production) at ECS cutover" - bunx trigger.dev@4.4.3 promote "$VERSION" --env prod + echo "Promoting Trigger.dev version $VERSION ($TRIGGER_ENV) at ECS cutover" + bunx trigger.dev@4.4.3 promote "$VERSION" --env "$TRIGGER_ENV" # Build ARM64 images for GHCR (main branch only, runs in parallel with # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64 From baf3d0b52c7d9aee9a6f29b11e7dd769ec1b20a7 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Sun, 19 Jul 2026 13:55:34 -0700 Subject: [PATCH 04/15] fix(ci): widen promote-trigger job timeout above the poll budget --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f931ca1ecf8..ff69183dd2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -588,7 +588,10 @@ jobs: (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') && needs.promote-images.outputs.promoted == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 75 + # Must exceed the poll script's OVERALL_TIMEOUT (70 min, covering a prod deploy + # queued behind a ~50-min bake) PLUS runner setup + the final promote step, so + # the Actions timeout never kills the job before the script's own deadline. + timeout-minutes: 90 permissions: contents: read id-token: write From d0e117a570b46e1b2183cd71e44eed9a5e7a52c2 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Mon, 20 Jul 2026 10:40:51 -0700 Subject: [PATCH 05/15] fix(ci): hold AWS session for the full poll and skip wait when the app image is unchanged --- .github/workflows/ci.yml | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff69183dd2e..e5213228ebe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -507,6 +507,9 @@ jobs: # promote-trigger passes it to the poll script so a stale pipeline execution # reusing the same image digest can't satisfy the cutover gate. retag_epoch: ${{ steps.promote.outputs.retag_epoch }} + # 'false' when the app deploy tag didn't move to a new digest (no ECS deploy). + # promote-trigger promotes immediately in that case instead of waiting. + app_image_changed: ${{ steps.promote.outputs.app_image_changed }} steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 @@ -558,6 +561,23 @@ jobs: ECR_TAG="staging" fi + # Detect whether the APP deploy tag actually moves to a new digest. If + # this commit's app image is byte-identical to the currently-deployed one + # (e.g. a commit that doesn't touch the app image — docs/CI-only), the + # retag is a no-op, ECR fires no push event, and no ECS app deploy runs. + # promote-trigger reads this to promote immediately instead of waiting for + # a cutover that will never happen. + get_digest() { docker buildx imagetools inspect "$1" 2>/dev/null | awk '/^Digest:/{print $2; exit}'; } + APP_REF="${REGISTRY}/${{ secrets.ECR_APP }}" + NEW_APP_DIGEST="$(get_digest "${APP_REF}:${{ github.sha }}")" + PREV_APP_DIGEST="$(get_digest "${APP_REF}:${ECR_TAG}" || true)" + if [ -n "$NEW_APP_DIGEST" ] && [ "$NEW_APP_DIGEST" = "$PREV_APP_DIGEST" ]; then + echo "app_image_changed=false" >> "$GITHUB_OUTPUT" + echo "ℹ️ App deploy tag ${ECR_TAG} already points at ${NEW_APP_DIGEST}; no ECS app deploy will be triggered." + else + echo "app_image_changed=true" >> "$GITHUB_OUTPUT" + fi + # Verify every sha image exists before moving any deploy tag, so a # missing/expired image aborts the whole promotion up front. for repo in $ECR_REPOS; do @@ -629,8 +649,17 @@ jobs: with: role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }} aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }} - + # The poll can run up to ~70 min (prod deploy queued behind a bake), which + # outlasts the default 1h session. Hold the session for the full job so AWS + # calls don't start failing mid-poll. Requires the deploy role's + # MaxSessionDuration to be >= this value (roles are managed outside the repo). + role-duration-seconds: 5400 + + # Skip the cutover wait when the app image didn't change (no ECS deploy was + # triggered) — otherwise the poll would hang until timeout. Promotion still + # runs below, immediately, since there is no app cutover to align with. - name: Wait for ECS traffic cutover + if: needs.promote-images.outputs.app_image_changed == 'true' env: PIPELINE: sim-${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}-us-east-1-app-deployment RETAG_EPOCH: ${{ needs.promote-images.outputs.retag_epoch }} @@ -656,7 +685,7 @@ jobs: echo "ERROR: no deployed version passed from deploy-trigger" >&2 exit 1 fi - echo "Promoting Trigger.dev version $VERSION ($TRIGGER_ENV) at ECS cutover" + echo "Promoting Trigger.dev version $VERSION ($TRIGGER_ENV)" bunx trigger.dev@4.4.3 promote "$VERSION" --env "$TRIGGER_ENV" # Build ARM64 images for GHCR (main branch only, runs in parallel with From 88527719c913993a4b8fc0df68673e1afa7695e4 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Mon, 20 Jul 2026 10:44:25 -0700 Subject: [PATCH 06/15] feat(ci): extend lockstep Trigger.dev promotion to dev (preview branch) --- .github/workflows/ci.yml | 146 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 141 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5213228ebe..3a1fa8bd777 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,7 +215,19 @@ jobs: env: ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }} + # App leg only: capture the digest the :dev tag currently points at, BEFORE + # this build overwrites it, so promote-trigger-dev can tell whether the app + # image actually changed (a no-op :dev push triggers no ECS deploy). + - name: Capture previous :dev app digest + id: prevdigest + if: matrix.ecr_repo_secret == 'ECR_APP' + run: | + REF="${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev" + PREV=$(docker buildx imagetools inspect "$REF" 2>/dev/null | awk '/^Digest:/{print $2; exit}' || true) + echo "digest=${PREV}" >> "$GITHUB_OUTPUT" + - name: Build and push + id: build uses: ./.github/actions/docker-build with: provider: ${{ vars.CI_PROVIDER }} @@ -224,15 +236,45 @@ jobs: tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev max-cache-size-mb: ${{ matrix.cache_mb }} - # Dev: deploy Trigger.dev background tasks to the preview "dev-sim" branch. - # Gated after migrate-dev for the same reason as build-dev — the new task - # code runs against the dev DB, so the schema must be pushed first. + # App leg only: publish the metadata promote-trigger-dev needs to correlate + # this push to its dev ECS deploy and decide whether to wait. Dev has no + # promote-images job, so this stands in for its retag_epoch/app_image_changed + # outputs. The epoch is recorded just after the :dev push (the pipeline trigger). + - name: Publish dev cutover metadata + if: matrix.ecr_repo_secret == 'ECR_APP' + run: | + mkdir -p dev-meta + NEW="${{ steps.build.outputs.digest }}" + PREV="${{ steps.prevdigest.outputs.digest }}" + echo "$NEW" > dev-meta/digest.txt + date +%s > dev-meta/retag_epoch.txt + if [ -n "$NEW" ] && [ "$NEW" = "$PREV" ]; then + echo "false" > dev-meta/app_image_changed.txt + echo "ℹ️ :dev already points at ${NEW}; no ECS dev deploy will be triggered." + else + echo "true" > dev-meta/app_image_changed.txt + fi + + - name: Upload dev cutover metadata + if: matrix.ecr_repo_secret == 'ECR_APP' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: dev-cutover-meta + path: dev-meta/ + retention-days: 1 + + # Dev: build & upload the Trigger.dev task version WITHOUT promoting it + # (--skip-promotion) to the preview "dev-sim" branch. promote-trigger-dev flips + # it at the dev ECS traffic cutover. Gated after migrate-dev so the schema is + # pushed before the new task version can run against the dev DB. deploy-trigger-dev: name: Deploy Trigger.dev (Dev) needs: [migrate-dev] if: github.event_name == 'push' && github.ref == 'refs/heads/dev' runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 15 + outputs: + version: ${{ steps.deploy.outputs.version }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -256,17 +298,111 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile --ignore-scripts - - name: Deploy to Trigger.dev + - name: Deploy to Trigger.dev (skip promotion) + id: deploy working-directory: ./apps/sim env: TRIGGER_ACCESS_TOKEN: ${{ secrets.DEV_TRIGGER_ACCESS_TOKEN }} TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} run: | + set -eo pipefail if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then echo "ERROR: DEV_TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 exit 1 fi - bunx trigger.dev@4.5.12 deploy --env preview --branch dev-sim + bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim --skip-promotion 2>&1 | tee deploy.log + VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) + if [ -z "$VERSION" ]; then + echo "ERROR: could not parse deployed version from deploy output" >&2 + exit 1 + fi + echo "Captured deployed version: $VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + # Dev: promote the skip-promoted preview version at the dev ECS traffic cutover. + # Dev has no promote-images gate (build-dev pushes :dev directly), so the digest, + # trigger epoch, and app-image-changed signal come from build-dev's artifact. + # trigger.dev supports promoting a specific preview branch: promote --env preview + # --branch dev-sim. + promote-trigger-dev: + name: Promote Trigger.dev (Dev) + needs: [build-dev, deploy-trigger-dev] + if: github.event_name == 'push' && github.ref == 'refs/heads/dev' + runs-on: blacksmith-4vcpu-ubuntu-2404 + # Dev bake is 5 min; the poll budget (30 min) and session (40 min) are sized for + # that with margin, well short of the prod path's 70/90. + timeout-minutes: 40 + permissions: + contents: read + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Download dev cutover metadata + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: dev-cutover-meta + path: dev-meta + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ secrets.DEV_AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.DEV_AWS_REGION }} + role-duration-seconds: 2400 + + - name: Wait for ECS traffic cutover + env: + OVERALL_TIMEOUT: "1800" + run: | + set -eo pipefail + CHANGED=$(cat dev-meta/app_image_changed.txt) + if [ "$CHANGED" != "true" ]; then + echo "App image unchanged — no dev ECS deploy triggered; promoting immediately." + exit 0 + fi + DIGEST=$(cat dev-meta/digest.txt) + EPOCH=$(cat dev-meta/retag_epoch.txt) + bash .github/scripts/wait-for-ecs-cutover.sh sim-dev-us-east-1-app-deployment "$DIGEST" "$EPOCH" + + - name: Promote Trigger.dev version + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.DEV_TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + VERSION: ${{ needs.deploy-trigger-dev.outputs.version }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: DEV_TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + if [ -z "$VERSION" ]; then + echo "ERROR: no deployed version passed from deploy-trigger-dev" >&2 + exit 1 + fi + echo "Promoting Trigger.dev version $VERSION (preview / dev-sim)" + bunx trigger.dev@4.4.3 promote "$VERSION" --env preview --branch dev-sim # Main/staging: build & upload the Trigger.dev task version WITHOUT promoting it # (--skip-promotion). New runs keep executing the OLD promoted version until From a431d1914eb72a8e56431ce137fcf1b8295b0719 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Mon, 20 Jul 2026 10:54:04 -0700 Subject: [PATCH 07/15] fix(ci): don't block dev task promotion on a non-app build-dev leg failure --- .github/workflows/ci.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a1fa8bd777..51befb92528 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -327,7 +327,15 @@ jobs: promote-trigger-dev: name: Promote Trigger.dev (Dev) needs: [build-dev, deploy-trigger-dev] - if: github.event_name == 'push' && github.ref == 'refs/heads/dev' + # Run as long as the task upload succeeded, even if a NON-app build-dev leg + # (realtime/pii/migrations) failed: the app leg pushes :dev independently and + # may have already triggered the ECS deploy, so an unrelated image failure must + # not strand the app on the old task version. The app-metadata artifact (only + # the app leg uploads it) is the real signal that an app deploy happened. + if: >- + !cancelled() && + github.event_name == 'push' && github.ref == 'refs/heads/dev' && + needs.deploy-trigger-dev.result == 'success' runs-on: blacksmith-4vcpu-ubuntu-2404 # Dev bake is 5 min; the poll budget (30 min) and session (40 min) are sized for # that with margin, well short of the prod path's 70/90. @@ -358,13 +366,28 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile + # Tolerate a missing artifact: it's only uploaded by the app leg, so its + # absence means the app image didn't build → no ECS deploy happened. - name: Download dev cutover metadata + id: meta + continue-on-error: true uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: dev-cutover-meta path: dev-meta + - name: Determine whether an app deploy happened + id: appdeploy + run: | + if [ -f dev-meta/app_image_changed.txt ]; then + echo "deployed=true" >> "$GITHUB_OUTPUT" + else + echo "deployed=false" >> "$GITHUB_OUTPUT" + echo "::warning::No app-image metadata (app leg did not build); skipping dev task promotion." + fi + - name: Configure AWS credentials + if: steps.appdeploy.outputs.deployed == 'true' uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE_TO_ASSUME }} @@ -372,6 +395,7 @@ jobs: role-duration-seconds: 2400 - name: Wait for ECS traffic cutover + if: steps.appdeploy.outputs.deployed == 'true' env: OVERALL_TIMEOUT: "1800" run: | @@ -386,6 +410,7 @@ jobs: bash .github/scripts/wait-for-ecs-cutover.sh sim-dev-us-east-1-app-deployment "$DIGEST" "$EPOCH" - name: Promote Trigger.dev version + if: steps.appdeploy.outputs.deployed == 'true' working-directory: ./apps/sim env: TRIGGER_ACCESS_TOKEN: ${{ secrets.DEV_TRIGGER_ACCESS_TOKEN }} From 99bb55af5eb805b531a7aec6df688cf21e7df66b Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Mon, 20 Jul 2026 11:21:57 -0700 Subject: [PATCH 08/15] chore(ci): use one Trigger.dev PAT for all envs (drop DEV_TRIGGER_ACCESS_TOKEN) --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51befb92528..726baa7e370 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -302,12 +302,12 @@ jobs: id: deploy working-directory: ./apps/sim env: - TRIGGER_ACCESS_TOKEN: ${{ secrets.DEV_TRIGGER_ACCESS_TOKEN }} + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} run: | set -eo pipefail if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then - echo "ERROR: DEV_TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 exit 1 fi bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim --skip-promotion 2>&1 | tee deploy.log @@ -413,13 +413,13 @@ jobs: if: steps.appdeploy.outputs.deployed == 'true' working-directory: ./apps/sim env: - TRIGGER_ACCESS_TOKEN: ${{ secrets.DEV_TRIGGER_ACCESS_TOKEN }} + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} VERSION: ${{ needs.deploy-trigger-dev.outputs.version }} run: | set -eo pipefail if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then - echo "ERROR: DEV_TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 exit 1 fi if [ -z "$VERSION" ]; then From e3ac689ca455017da87a8065073945249319c735 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Mon, 20 Jul 2026 11:33:17 -0700 Subject: [PATCH 09/15] fix(ci): reliable digest reads for no-op detection, robust version parse, pre-push dev epoch --- .github/workflows/ci.yml | 68 ++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 726baa7e370..7f92ea9f546 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,15 +215,18 @@ jobs: env: ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }} - # App leg only: capture the digest the :dev tag currently points at, BEFORE - # this build overwrites it, so promote-trigger-dev can tell whether the app - # image actually changed (a no-op :dev push triggers no ECS deploy). - - name: Capture previous :dev app digest + # App leg only: stamp the trigger epoch and capture the :dev digest BEFORE the + # build/push overwrites it. Stamping the epoch pre-build (not after the push, + # like it was) guarantees it precedes the :dev push that triggers the pipeline, + # so the dev ECS execution's startTime can't land before the epoch and get + # rejected by the cutover poll. The digest read uses the ECR API so an absent + # tag ("None", first deploy → changed) is distinct from a read error. + - name: Capture pre-build :dev state id: prevdigest if: matrix.ecr_repo_secret == 'ECR_APP' run: | - REF="${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev" - PREV=$(docker buildx imagetools inspect "$REF" 2>/dev/null | awk '/^Digest:/{print $2; exit}' || true) + echo "epoch=$(date +%s)" >> "$GITHUB_OUTPUT" + PREV="$(aws ecr batch-get-image --repository-name "${{ steps.ecr-repo.outputs.name }}" --image-ids imageTag=dev --query 'images[0].imageId.imageDigest' --output text 2>/dev/null)" || PREV="__ERR__" echo "digest=${PREV}" >> "$GITHUB_OUTPUT" - name: Build and push @@ -239,16 +242,22 @@ jobs: # App leg only: publish the metadata promote-trigger-dev needs to correlate # this push to its dev ECS deploy and decide whether to wait. Dev has no # promote-images job, so this stands in for its retag_epoch/app_image_changed - # outputs. The epoch is recorded just after the :dev push (the pipeline trigger). + # outputs. The epoch and prev digest come from the pre-build step above. - name: Publish dev cutover metadata if: matrix.ecr_repo_secret == 'ECR_APP' run: | mkdir -p dev-meta NEW="${{ steps.build.outputs.digest }}" PREV="${{ steps.prevdigest.outputs.digest }}" + if [ -z "$NEW" ]; then + echo "ERROR: build did not report an image digest" >&2 + exit 1 + fi echo "$NEW" > dev-meta/digest.txt - date +%s > dev-meta/retag_epoch.txt - if [ -n "$NEW" ] && [ "$NEW" = "$PREV" ]; then + echo "${{ steps.prevdigest.outputs.epoch }}" > dev-meta/retag_epoch.txt + # PREV=__ERR__ (read failed) falls through to changed=true (wait) — the safe + # direction. A real no-op is detected when the read succeeds and matches. + if [ "$PREV" != "__ERR__" ] && [ "$NEW" = "$PREV" ]; then echo "false" > dev-meta/app_image_changed.txt echo "ℹ️ :dev already points at ${NEW}; no ECS dev deploy will be triggered." else @@ -311,7 +320,16 @@ jobs: exit 1 fi bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim --skip-promotion 2>&1 | tee deploy.log - VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) + # Anchor on the "version" keyword and take the FIRST match: with + # --skip-promotion the CLI can print the unchanged current version AFTER + # the one it just deployed, and dashboard URLs carry other IDs — so a bare + # last-match could promote the wrong version. Fall back to a bare + # first-match only if no version-labelled line is present. + CLEAN=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log) + VERSION=$(printf '%s\n' "$CLEAN" | grep -oiE 'version[[:space:]]+v?20[0-9]{6}\.[0-9]+' | grep -oE '20[0-9]{6}\.[0-9]+' | head -n1 || true) + if [ -z "$VERSION" ]; then + VERSION=$(printf '%s\n' "$CLEAN" | grep -oE '20[0-9]{6}\.[0-9]+' | head -n1 || true) + fi if [ -z "$VERSION" ]; then echo "ERROR: could not parse deployed version from deploy output" >&2 exit 1 @@ -484,7 +502,16 @@ jobs: fi bunx trigger.dev@4.4.3 deploy --env "$TRIGGER_ENV" --skip-promotion 2>&1 | tee deploy.log # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation. - VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) + # Anchor on the "version" keyword and take the FIRST match: with + # --skip-promotion the CLI can print the unchanged current version AFTER + # the one it just deployed, and dashboard URLs carry other IDs — so a bare + # last-match could promote the wrong version. Fall back to a bare + # first-match only if no version-labelled line is present. + CLEAN=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log) + VERSION=$(printf '%s\n' "$CLEAN" | grep -oiE 'version[[:space:]]+v?20[0-9]{6}\.[0-9]+' | grep -oE '20[0-9]{6}\.[0-9]+' | head -n1 || true) + if [ -z "$VERSION" ]; then + VERSION=$(printf '%s\n' "$CLEAN" | grep -oE '20[0-9]{6}\.[0-9]+' | head -n1 || true) + fi if [ -z "$VERSION" ]; then echo "ERROR: could not parse deployed version from deploy output" >&2 exit 1 @@ -728,11 +755,20 @@ jobs: # retag is a no-op, ECR fires no push event, and no ECS app deploy runs. # promote-trigger reads this to promote immediately instead of waiting for # a cutover that will never happen. - get_digest() { docker buildx imagetools inspect "$1" 2>/dev/null | awk '/^Digest:/{print $2; exit}'; } - APP_REF="${REGISTRY}/${{ secrets.ECR_APP }}" - NEW_APP_DIGEST="$(get_digest "${APP_REF}:${{ github.sha }}")" - PREV_APP_DIGEST="$(get_digest "${APP_REF}:${ECR_TAG}" || true)" - if [ -n "$NEW_APP_DIGEST" ] && [ "$NEW_APP_DIGEST" = "$PREV_APP_DIGEST" ]; then + # + # Read digests via the ECR API, which cleanly returns "None" for an absent + # tag (first deploy → changed) vs a non-zero exit on a real read error. On + # a read error we fall through to changed=true (wait) — the safe direction + # (old tasks stay current, job fails visibly) rather than promoting early. + APP_REPO="${{ secrets.ECR_APP }}" + ecr_digest() { aws ecr batch-get-image --repository-name "$1" --image-ids imageTag="$2" --query 'images[0].imageId.imageDigest' --output text 2>/dev/null; } + NEW_APP_DIGEST="$(ecr_digest "$APP_REPO" "${{ github.sha }}")" || NEW_APP_DIGEST="__ERR__" + PREV_APP_DIGEST="$(ecr_digest "$APP_REPO" "${ECR_TAG}")" || PREV_APP_DIGEST="__ERR__" + if [ "$NEW_APP_DIGEST" = "__ERR__" ] || [ "$NEW_APP_DIGEST" = "None" ] || [ -z "$NEW_APP_DIGEST" ]; then + echo "ERROR: could not resolve the new app image digest for ${{ github.sha }}" >&2 + exit 1 + fi + if [ "$PREV_APP_DIGEST" != "__ERR__" ] && [ "$NEW_APP_DIGEST" = "$PREV_APP_DIGEST" ]; then echo "app_image_changed=false" >> "$GITHUB_OUTPUT" echo "ℹ️ App deploy tag ${ECR_TAG} already points at ${NEW_APP_DIGEST}; no ECS app deploy will be triggered." else From 67d8224d4c576cf6ff294f19bfcc4418258a74f6 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Mon, 20 Jul 2026 11:41:41 -0700 Subject: [PATCH 10/15] fix(ci): give dev promote-trigger a 20-min margin over its poll budget --- .github/workflows/ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f92ea9f546..1f353acb448 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -355,8 +355,10 @@ jobs: github.event_name == 'push' && github.ref == 'refs/heads/dev' && needs.deploy-trigger-dev.result == 'success' runs-on: blacksmith-4vcpu-ubuntu-2404 - # Dev bake is 5 min; the poll budget (30 min) and session (40 min) are sized for - # that with margin, well short of the prod path's 70/90. + # Dev bake is 5 min and dev deploys don't queue behind a bake (serialized by the + # ci- group), so a 20-min poll is ample; the 40-min job leaves ~20 min for + # setup + promote above it (mirrors the prod 90-vs-70 margin), and the 40-min + # session outlasts the poll. timeout-minutes: 40 permissions: contents: read @@ -415,7 +417,7 @@ jobs: - name: Wait for ECS traffic cutover if: steps.appdeploy.outputs.deployed == 'true' env: - OVERALL_TIMEOUT: "1800" + OVERALL_TIMEOUT: "1200" run: | set -eo pipefail CHANGED=$(cat dev-meta/app_image_changed.txt) From 81d73daa33437874100d8c51ff6fdfb03a6188b5 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Mon, 20 Jul 2026 11:47:42 -0700 Subject: [PATCH 11/15] fix(ci): require promote-images + deploy-trigger success explicitly for promote-trigger --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f353acb448..da8366a015e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -802,9 +802,15 @@ jobs: promote-trigger: name: Promote Trigger.dev needs: [promote-images, deploy-trigger] + # Require both upstreams to have SUCCEEDED explicitly (not just promoted==true): + # a job if without a status-check function keeps the implicit success() gate, but + # spelling it out removes any doubt that a failed promote-images/deploy-trigger + # can't reach this job and promote tasks with no app retag/cutover. if: >- github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') && + needs.promote-images.result == 'success' && + needs.deploy-trigger.result == 'success' && needs.promote-images.outputs.promoted == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 # Must exceed the poll script's OVERALL_TIMEOUT (70 min, covering a prod deploy From da569702fdba9abb8a07d329a1cb662ae82e2c81 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 16 Sep 2026 12:59:53 -0700 Subject: [PATCH 12/15] fix(ci): verify Trigger promotion against the deployed image and cutover --- .github/actions/docker-build/action.yml | 7 + .github/scripts/get-ecr-image-digest.sh | 28 ++++ .github/scripts/test-trigger-deploy.py | 187 +++++++++++++++++++++ .github/scripts/wait-for-ecs-cutover.sh | 191 ++++++++++------------ .github/workflows/ci.yml | 208 ++++++++---------------- .github/workflows/test-build.yml | 3 + 6 files changed, 376 insertions(+), 248 deletions(-) create mode 100644 .github/scripts/get-ecr-image-digest.sh create mode 100644 .github/scripts/test-trigger-deploy.py diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 72d90e8e8bb..2be32c5591c 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -30,6 +30,11 @@ inputs: bypass an input `default:` entirely. required: false +outputs: + digest: + description: The image digest returned by the selected build provider. + value: ${{ steps.build-blacksmith.outputs.digest || steps.build-github.outputs.digest }} + # Registry logins must precede this action. provenance/sbom stay off: attestation # manifests break `imagetools create` retagging in promote-images. runs: @@ -64,6 +69,7 @@ runs: cache-key: ${{ steps.cache-key.outputs.value }} - name: Build and push (Blacksmith) + id: build-blacksmith if: inputs.provider == '' || inputs.provider == 'blacksmith' uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 with: @@ -169,6 +175,7 @@ runs: # No cache-to: type=gha — it shares the 10 GB repo quota with the cache mounts. - name: Build and push (GitHub) + id: build-github if: inputs.provider != '' && inputs.provider != 'blacksmith' uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: diff --git a/.github/scripts/get-ecr-image-digest.sh b/.github/scripts/get-ecr-image-digest.sh new file mode 100644 index 00000000000..f8de53f925e --- /dev/null +++ b/.github/scripts/get-ecr-image-digest.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Read one ECR tag. Only ImageNotFound is optional; AWS and response errors fail. +set -euo pipefail +REPOSITORY="${1:?repository required}" +TAG="${2:?tag required}" +ALLOW_MISSING="${3:-}" +if [ -n "$ALLOW_MISSING" ] && [ "$ALLOW_MISSING" != '--allow-missing' ]; then + echo 'ERROR: expected --allow-missing or no third argument' >&2 + exit 1 +fi +export AWS_PAGER='' +aws ecr batch-get-image --repository-name "$REPOSITORY" --image-ids imageTag="$TAG" --output json | + ALLOW_MISSING="$ALLOW_MISSING" python3 -c ' +import json, os, re, sys +response = json.load(sys.stdin) +images, failures = response["images"], response["failures"] +if failures: + if not images and len(failures) == 1 and failures[0]["failureCode"] == "ImageNotFound" and os.environ["ALLOW_MISSING"]: + print("") + sys.exit(0) + raise SystemExit("ERROR: ECR image lookup failed: " + ", ".join(f["failureCode"] for f in failures)) +if len(images) != 1: + raise SystemExit("ERROR: expected exactly one ECR image") +digest = images[0]["imageId"]["imageDigest"] +if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): + raise SystemExit("ERROR: invalid ECR image digest") +print(digest) +' diff --git a/.github/scripts/test-trigger-deploy.py b/.github/scripts/test-trigger-deploy.py new file mode 100644 index 00000000000..6041631acd6 --- /dev/null +++ b/.github/scripts/test-trigger-deploy.py @@ -0,0 +1,187 @@ +"""Exercise the deployment gates with scripted AWS responses; no live mutations.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +SCRIPTS = Path(__file__).resolve().parent +DIGEST = 'sha256:' + 'a' * 64 +OTHER_DIGEST = 'sha256:' + 'b' * 64 + + +def execution(start=1000, digest=DIGEST, identifier='execution-current'): + return { + 'startTime': start, + 'pipelineExecutionId': identifier, + 'sourceRevisions': [{'actionName': 'ECR_Source', 'revisionId': digest}], + } + + +class DeploymentGateTests(unittest.TestCase): + def run_script(self, script, args, responses): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture = root / 'responses.json' + fixture.write_text(json.dumps(responses)) + (root / 'aws').write_text('''#!/usr/bin/env python3 +import json, os, pathlib, sys +root = pathlib.Path(os.environ['FIXTURE_DIR']) +args = sys.argv[1:] +if args[0] == '--cli-connect-timeout': + args = args[4:] +service, operation = args[:2] +key = operation +if operation == 'get-deployment-target': + key += ':' + args[args.index('--target-id') + 1] +with (root / 'calls').open('a') as stream: + stream.write(' '.join(args) + '\\n') +responses = json.loads((root / 'responses.json').read_text()) +if key not in responses: + raise SystemExit('Unexpected AWS call: ' + key) +response = responses[key] +if isinstance(response, list): + response = response.pop(0) + responses[key] = response if not responses[key] else responses[key] + (root / 'responses.json').write_text(json.dumps(responses)) +if response.get('error'): + sys.stderr.write(response['error']) + sys.exit(254) +print(response.get('text', json.dumps(response.get('json')))) +''') + (root / 'date').write_text('''#!/usr/bin/env python3 +import os, pathlib +path = pathlib.Path(os.environ['FIXTURE_DIR']) / 'clock' +value = int(path.read_text()) if path.exists() else 1000 +path.write_text(str(value + 1)) +print(value) +''') + (root / 'sleep').write_text('#!/bin/sh\nexit 0\n') + for name in ('aws', 'date', 'sleep'): + (root / name).chmod(0o755) + result = subprocess.run( + ['bash', str(SCRIPTS / script), *args], + env={**os.environ, 'PATH': f'{root}:{os.environ["PATH"]}', + 'FIXTURE_DIR': str(root), 'POLL_INTERVAL': '1', 'OVERALL_TIMEOUT': '12'}, + capture_output=True, text=True, timeout=10, + ) + calls = (root / 'calls').read_text() if (root / 'calls').exists() else '' + return result, calls + + def poll(self, updates=None, since='1000'): + responses = { + 'list-pipeline-executions': {'json': [execution()]}, + 'get-pipeline-execution': {'text': 'InProgress'}, + 'list-action-executions': {'text': 'd-current'}, + 'get-deployment': {'text': 'InProgress'}, + 'list-deployment-targets': {'text': 'target-one\ttarget-two'}, + 'get-deployment-target:target-one': {'text': 'Succeeded'}, + 'get-deployment-target:target-two': {'text': 'Succeeded'}, + } + responses.update(updates or {}) + return self.run_script('wait-for-ecs-cutover.sh', ['app-pipeline', DIGEST, since], responses) + + def test_waits_for_every_target(self): + result, calls = self.poll({'get-deployment-target:target-two': [ + {'text': 'InProgress'}, {'text': 'Succeeded'}]}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(calls.count('--target-id target-two'), 2) + + def test_rejects_stale_execution_inside_former_clock_skew_window(self): + result, calls = self.poll({'list-pipeline-executions': {'json': [execution(start=999)]}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('timed out', result.stdout) + self.assertNotIn('get-pipeline-execution ', calls) + + def test_chooses_newest_matching_execution(self): + result, calls = self.poll({'list-pipeline-executions': {'json': [ + execution(1000, identifier='execution-old'), execution(1001)]}}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('--pipeline-execution-id execution-current', calls) + + def test_iso_timestamps(self): + result, _ = self.poll({'list-pipeline-executions': {'json': [ + execution('1970-01-01T00:16:40+00:00')]}}) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_access_denial_fails_immediately(self): + result, calls = self.poll({'list-pipeline-executions': {'error': 'AccessDeniedException'}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('AccessDeniedException', result.stderr) + self.assertEqual(len(calls.splitlines()), 1) + + def test_credentials_expiring_during_target_poll_fail(self): + result, _ = self.poll({'get-deployment-target:target-two': {'error': 'ExpiredToken'}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('ExpiredToken', result.stderr) + + def test_failed_and_superseded_pipeline_never_reach_deployment(self): + for status in ('Failed', 'Stopped', 'Superseded'): + with self.subTest(status=status): + result, calls = self.poll({'get-pipeline-execution': {'text': status}}) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('get-deployment ', calls) + + def test_waits_for_queued_deploy_action(self): + result, calls = self.poll({'list-action-executions': [{'text': 'None'}, {'text': 'd-current'}]}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(calls.count('list-action-executions '), 2) + + def test_failed_deployment_never_accepts_old_cutover(self): + result, calls = self.poll({'get-deployment': {'text': 'Failed'}}) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('get-deployment-target ', calls) + + def test_empty_targets_cannot_satisfy_gate(self): + result, _ = self.poll({'list-deployment-targets': {'text': ''}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('timed out', result.stdout) + + def test_failed_target_fails_immediately(self): + result, _ = self.poll({'get-deployment-target:target-two': {'text': 'Failed'}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('cutover status Failed', result.stdout) + + def test_unchanged_image_verifies_existing_cutover(self): + result, calls = self.poll({'list-pipeline-executions': {'json': [execution(start=900)]}}, since='0') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('get-deployment-target ', calls) + + def test_unchanged_image_rejects_latest_different_deploy(self): + result, calls = self.poll({'list-pipeline-executions': {'json': [ + execution(start=900), execution(start=999, digest=OTHER_DIGEST)]}}, since='0') + self.assertNotEqual(result.returncode, 0) + self.assertIn('cutover is unverified', result.stderr) + self.assertNotIn('get-deployment ', calls) + + def test_unchanged_image_rejects_failed_previous_deploy(self): + result, _ = self.poll({'get-deployment': {'text': 'Failed'}}, since='0') + self.assertNotEqual(result.returncode, 0) + + def test_invalid_metadata_fails_before_aws(self): + result, calls = self.poll(since='corrupted') + self.assertNotEqual(result.returncode, 0) + self.assertEqual(calls, '') + + def test_ecr_digest_and_missing_tag(self): + result, _ = self.run_script('get-ecr-image-digest.sh', ['app', 'deploy'], { + 'batch-get-image': {'json': {'images': [{'imageId': {'imageDigest': DIGEST}}], 'failures': []}}}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), DIGEST) + missing = {'batch-get-image': {'json': {'images': [], 'failures': [{'failureCode': 'ImageNotFound'}]}}} + result, _ = self.run_script('get-ecr-image-digest.sh', ['app', 'deploy', '--allow-missing'], missing) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), '') + result, _ = self.run_script('get-ecr-image-digest.sh', ['app', 'deploy'], missing) + self.assertNotEqual(result.returncode, 0) + + def test_ecr_response_failures_are_not_missing_images(self): + for response in ({'error': 'AccessDeniedException'}, {'json': {'images': [], 'failures': [{'failureCode': 'KmsError'}]}}, {'json': {'images': [], 'failures': []}}): + with self.subTest(response=response): + result, _ = self.run_script('get-ecr-image-digest.sh', ['app', 'deploy', '--allow-missing'], {'batch-get-image': response}) + self.assertNotEqual(result.returncode, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/wait-for-ecs-cutover.sh b/.github/scripts/wait-for-ecs-cutover.sh index f7ea7970567..a925c061ad9 100755 --- a/.github/scripts/wait-for-ecs-cutover.sh +++ b/.github/scripts/wait-for-ecs-cutover.sh @@ -1,146 +1,121 @@ #!/usr/bin/env bash -# Waits for the ECS blue/green deploy triggered by a specific app image push to -# reach its traffic cutover (CodeDeploy AllowTraffic == Succeeded on every ECS -# target), then exits 0. -# -# ECR app images use a floating tag (latest/staging) with no git SHA, so the -# only durable key linking this CI push to its ECS deploy is the image DIGEST. -# Correlation: image digest -> CodePipeline execution (ECR_Source revision) -> -# Deploy action externalExecutionId (== CodeDeploy deployment id) -> AllowTraffic. -# -# The digest alone is ambiguous: a prior run with the same image could match an -# older, already-cutover execution and promote too early. SINCE_EPOCH (the time -# the deploy tag was retagged, i.e. when THIS push's pipeline was triggered) -# disambiguates — only an execution that started at/after the retag is ours. -# +# Resolve a pushed app digest to CodePipeline -> CodeDeploy -> every ECS target's +# AllowTraffic event. An unchanged tag uses since-epoch=0 to verify the latest +# pipeline execution instead of assuming the tagged image is already serving. # Usage: wait-for-ecs-cutover.sh -# Requires: awscli v2, python3, credentials with codedeploy + codepipeline read. set -euo pipefail PIPELINE="${1:?pipeline name required}" DIGEST="${2:?image digest required}" -SINCE_EPOCH="${3:?since-epoch (retag time) required}" - +SINCE_EPOCH="${3:?since-epoch required}" POLL_INTERVAL="${POLL_INTERVAL:-15}" -# 70 min covers a prod deploy whose Deploy stage is queued behind a prior -# deploy's ~50-min termination bake before its own traffic shift begins. OVERALL_TIMEOUT="${OVERALL_TIMEOUT:-4200}" -# Tolerate minor clock skew between the runner (retag time) and CodePipeline. -SINCE_SKEW="${SINCE_SKEW:-120}" +if ! [[ "$PIPELINE" =~ ^[A-Za-z0-9.@_-]+$ && "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ && "$SINCE_EPOCH" =~ ^[0-9]+$ && "$POLL_INTERVAL" =~ ^[1-9][0-9]*$ && "$OVERALL_TIMEOUT" =~ ^[1-9][0-9]*$ ]]; then + echo 'ERROR: invalid pipeline, digest, epoch, or polling budget' >&2 + exit 1 +fi +export AWS_PAGER='' +export AWS_RETRY_MODE=standard +export AWS_MAX_ATTEMPTS=3 deadline=$(( $(date +%s) + OVERALL_TIMEOUT )) -remaining() { echo $(( deadline - $(date +%s) )); } log() { echo "[wait-for-ecs-cutover] $*"; } -fail_if_expired() { - if [ "$(remaining)" -le 0 ]; then - log "ERROR: timed out after ${OVERALL_TIMEOUT}s waiting for: $1" +check_deadline() { + if [ "$(date +%s)" -ge "$deadline" ]; then + log "ERROR: timed out after ${OVERALL_TIMEOUT}s waiting for $1" exit 1 fi } +aws_read() { + aws --cli-connect-timeout 10 --cli-read-timeout 30 "$@" +} -log "Pipeline: $PIPELINE" -log "Target app image digest: $DIGEST" -log "Requiring execution started at/after epoch $SINCE_EPOCH (minus ${SINCE_SKEW}s skew)" - -# Phase A: find the newest pipeline execution whose ECR source revision matches -# our digest AND that started at/after the retag. The since filter rejects a -# stale historical execution reusing the same digest. --max-items bounds the -# fetch (the CLI otherwise auto-paginates the whole history). -EXECUTION_ID="" +EXECUTION_ID='' while [ -z "$EXECUTION_ID" ]; do - fail_if_expired "pipeline execution matching digest since retag" - matches=$(aws codepipeline list-pipeline-executions \ - --pipeline-name "$PIPELINE" --max-items 30 \ - --query "pipelineExecutionSummaries[?sourceRevisions[?actionName=='ECR_Source' && revisionId=='$DIGEST']].[startTime, pipelineExecutionId]" \ - --output text 2>/dev/null || true) - EXECUTION_ID=$(printf '%s\n' "$matches" | SINCE="$SINCE_EPOCH" SKEW="$SINCE_SKEW" python3 -c ' -import sys, os, datetime -since = float(os.environ["SINCE"]) - float(os.environ["SKEW"]) -best_epoch = None -best_id = None -for line in sys.stdin: - parts = line.rstrip("\n").split("\t") - if len(parts) < 2: - continue - ts, eid = parts[0].strip(), parts[1].strip() - if not ts or not eid or "-" not in eid: - continue - try: - epoch = float(ts) - except ValueError: - try: - epoch = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp() - except ValueError: - continue - if epoch >= since and (best_epoch is None or epoch > best_epoch): - best_epoch, best_id = epoch, eid -print(best_id or "") -' 2>/dev/null || true) + check_deadline 'the matching pipeline execution' + executions=$(aws_read codepipeline list-pipeline-executions \ + --pipeline-name "$PIPELINE" --max-items 30 \ + --query 'pipelineExecutionSummaries' --output json) + EXECUTION_ID=$(printf '%s\n' "$executions" | SINCE="$SINCE_EPOCH" DIGEST="$DIGEST" python3 -c ' +import datetime, json, os, sys +since = int(os.environ["SINCE"]) +def epoch(execution): + value = execution["startTime"] + if isinstance(value, (int, float)): + return value + return datetime.datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() +def matches(execution): + return any(r["actionName"] == "ECR_Source" and r.get("revisionId") == os.environ["DIGEST"] for r in execution.get("sourceRevisions", [])) +executions = sorted(json.load(sys.stdin), key=epoch, reverse=True) +if since == 0: + if not executions or not matches(executions[0]): + raise SystemExit("ERROR: unchanged app tag does not match the latest pipeline execution; cutover is unverified") + selected = executions[0] +else: + selected = next((e for e in executions if epoch(e) >= since and matches(e)), None) +print(selected["pipelineExecutionId"] if selected else "") +') if [ -z "$EXECUTION_ID" ]; then - log "No matching post-retag pipeline execution yet; retry in ${POLL_INTERVAL}s (remaining $(remaining)s)" + log 'No matching execution since this push; waiting' sleep "$POLL_INTERVAL" fi done log "Matched pipeline execution: $EXECUTION_ID" -# Phase B: resolve the CodeDeploy deployment id from the Deploy action. This may -# stay empty for a while if the Deploy stage is queued behind a prior deploy. -DEPLOYMENT_ID="" -while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; do - fail_if_expired "CodeDeploy deployment id (Deploy stage may be queued behind a prior deploy's bake)" - status=$(aws codepipeline get-pipeline-execution \ - --pipeline-name "$PIPELINE" --pipeline-execution-id "$EXECUTION_ID" \ - --query 'pipelineExecution.status' --output text 2>/dev/null || true) +DEPLOYMENT_ID='' +while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = 'None' ]; do + check_deadline 'the CodeDeploy deployment (the Deploy stage may be queued)' + status=$(aws_read codepipeline get-pipeline-execution \ + --pipeline-name "$PIPELINE" --pipeline-execution-id "$EXECUTION_ID" \ + --query 'pipelineExecution.status' --output text) case "$status" in - Failed|Stopped|Superseded) - log "ERROR: pipeline execution $EXECUTION_ID ended in status $status before deploy" - exit 1 - ;; + Failed|Stopped|Stopping|Superseded|Cancelled) + log "ERROR: pipeline execution ended in $status; not promoting"; exit 1 ;; + InProgress|Succeeded) ;; + *) log "ERROR: unexpected pipeline status: $status"; exit 1 ;; esac - DEPLOYMENT_ID=$(aws codepipeline list-action-executions \ - --pipeline-name "$PIPELINE" \ - --filter pipelineExecutionId="$EXECUTION_ID" \ - --query "actionExecutionDetails[?stageName=='Deploy'].output.executionResult.externalExecutionId | [0]" \ - --output text 2>/dev/null || true) - if [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; then - log "Deploy stage not started yet (pipeline status: $status); retry in ${POLL_INTERVAL}s (remaining $(remaining)s)" + DEPLOYMENT_ID=$(aws_read codepipeline list-action-executions \ + --pipeline-name "$PIPELINE" --filter pipelineExecutionId="$EXECUTION_ID" \ + --query "actionExecutionDetails[?stageName=='Deploy'].output.executionResult.externalExecutionId | [0]" \ + --output text) + if [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = 'None' ]; then + if [ "$status" = 'Succeeded' ]; then + log 'ERROR: successful pipeline has no CodeDeploy deployment'; exit 1 + fi sleep "$POLL_INTERVAL" fi done log "CodeDeploy deployment: $DEPLOYMENT_ID" -# Phase C: wait for the traffic cutover. Require AllowTraffic == Succeeded on -# EVERY ECS target, so a multi-target deploy can't promote while one target is -# still mid-cutover or failed. while true; do - fail_if_expired "AllowTraffic (traffic cutover) on all targets" - dstatus=$(aws deploy get-deployment --deployment-id "$DEPLOYMENT_ID" \ - --query 'deploymentInfo.status' --output text 2>/dev/null || true) - case "$dstatus" in - Failed|Stopped) - log "ERROR: CodeDeploy deployment $DEPLOYMENT_ID ended in status $dstatus; not promoting" - exit 1 - ;; + check_deadline 'AllowTraffic on every ECS target' + status=$(aws_read deploy get-deployment --deployment-id "$DEPLOYMENT_ID" \ + --query 'deploymentInfo.status' --output text) + case "$status" in + Failed|Stopped) log "ERROR: deployment ended in $status; not promoting"; exit 1 ;; + Created|Queued|InProgress|Baking|Ready|Succeeded) ;; + *) log "ERROR: unexpected deployment status: $status"; exit 1 ;; esac - target_ids=$(aws deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \ - --query 'targetIds' --output text 2>/dev/null || true) - if [ -n "$target_ids" ] && [ "$target_ids" != "None" ]; then + target_ids=$(aws_read deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \ + --query 'targetIds' --output text) + if [ -n "$target_ids" ] && [ "$target_ids" != 'None' ]; then all_ok=1 - ntargets=0 - for tid in $target_ids; do - ntargets=$((ntargets + 1)) - at_status=$(aws deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$tid" \ - --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \ - --output text 2>/dev/null || true) - if [ "$at_status" != "Succeeded" ]; then - all_ok=0 - fi + for target in $target_ids; do + cutover=$(aws_read deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$target" \ + --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \ + --output text) + case "$cutover" in + Succeeded) ;; + Failed|Skipped|Unknown) log "ERROR: target $target cutover status $cutover"; exit 1 ;; + Pending|InProgress|None|'') all_ok=0 ;; + *) log "ERROR: unexpected cutover status: $cutover"; exit 1 ;; + esac done - if [ "$ntargets" -gt 0 ] && [ "$all_ok" = "1" ]; then - log "Traffic cutover complete (AllowTraffic Succeeded on all $ntargets target(s)) for $DEPLOYMENT_ID" + if [ "$all_ok" = 1 ]; then + log 'Traffic cutover complete on every ECS target' exit 0 fi fi - log "Deployment $DEPLOYMENT_ID status=$dstatus; not all targets past AllowTraffic; wait ${POLL_INTERVAL}s (remaining $(remaining)s)" + log 'Traffic cutover is not complete; waiting' sleep "$POLL_INTERVAL" done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da8366a015e..25efcb0bd8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,18 +215,14 @@ jobs: env: ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }} - # App leg only: stamp the trigger epoch and capture the :dev digest BEFORE the - # build/push overwrites it. Stamping the epoch pre-build (not after the push, - # like it was) guarantees it precedes the :dev push that triggers the pipeline, - # so the dev ECS execution's startTime can't land before the epoch and get - # rejected by the cutover poll. The digest read uses the ECR API so an absent - # tag ("None", first deploy → changed) is distinct from a read error. + # Capture the previous tag and timestamp before the build pushes :dev. + # Only a missing tag is allowed; failed reads abort before deployment. - name: Capture pre-build :dev state id: prevdigest if: matrix.ecr_repo_secret == 'ECR_APP' run: | echo "epoch=$(date +%s)" >> "$GITHUB_OUTPUT" - PREV="$(aws ecr batch-get-image --repository-name "${{ steps.ecr-repo.outputs.name }}" --image-ids imageTag=dev --query 'images[0].imageId.imageDigest' --output text 2>/dev/null)" || PREV="__ERR__" + PREV=$(bash .github/scripts/get-ecr-image-digest.sh "${{ steps.ecr-repo.outputs.name }}" dev --allow-missing) echo "digest=${PREV}" >> "$GITHUB_OUTPUT" - name: Build and push @@ -255,9 +251,7 @@ jobs: fi echo "$NEW" > dev-meta/digest.txt echo "${{ steps.prevdigest.outputs.epoch }}" > dev-meta/retag_epoch.txt - # PREV=__ERR__ (read failed) falls through to changed=true (wait) — the safe - # direction. A real no-op is detected when the read succeeds and matches. - if [ "$PREV" != "__ERR__" ] && [ "$NEW" = "$PREV" ]; then + if [ "$NEW" = "$PREV" ]; then echo "false" > dev-meta/app_image_changed.txt echo "ℹ️ :dev already points at ${NEW}; no ECS dev deploy will be triggered." else @@ -283,7 +277,7 @@ jobs: runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 15 outputs: - version: ${{ steps.deploy.outputs.version }} + version: ${{ steps.deploy.outputs.deploymentVersion }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -319,23 +313,16 @@ jobs: echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 exit 1 fi - bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim --skip-promotion 2>&1 | tee deploy.log - # Anchor on the "version" keyword and take the FIRST match: with - # --skip-promotion the CLI can print the unchanged current version AFTER - # the one it just deployed, and dashboard URLs carry other IDs — so a bare - # last-match could promote the wrong version. Fall back to a bare - # first-match only if no version-labelled line is present. - CLEAN=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log) - VERSION=$(printf '%s\n' "$CLEAN" | grep -oiE 'version[[:space:]]+v?20[0-9]{6}\.[0-9]+' | grep -oE '20[0-9]{6}\.[0-9]+' | head -n1 || true) - if [ -z "$VERSION" ]; then - VERSION=$(printf '%s\n' "$CLEAN" | grep -oE '20[0-9]{6}\.[0-9]+' | head -n1 || true) - fi - if [ -z "$VERSION" ]; then - echo "ERROR: could not parse deployed version from deploy output" >&2 + bunx trigger.dev@4.5.12 deploy --env preview --branch dev-sim --skip-promotion + + - name: Validate deployment version output + env: + VERSION: ${{ steps.deploy.outputs.deploymentVersion }} + run: | + if ! [[ "$VERSION" =~ ^[0-9]{8}\.[0-9]+$ ]]; then + echo "ERROR: Trigger.dev did not report a valid deploymentVersion output" >&2 exit 1 fi - echo "Captured deployed version: $VERSION" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" # Dev: promote the skip-promoted preview version at the dev ECS traffic cutover. # Dev has no promote-images gate (build-dev pushes :dev directly), so the digest, @@ -354,7 +341,7 @@ jobs: !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/dev' && needs.deploy-trigger-dev.result == 'success' - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Dev bake is 5 min and dev deploys don't queue behind a bake (serialized by the # ci- group), so a 20-min poll is ample; the 40-min job leaves ~20 min for # setup + promote above it (mirrors the prod 90-vs-70 margin), and the 40-min @@ -370,7 +357,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.13 + bun-version: 1.4.1 - name: Cache Bun dependencies uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 @@ -384,30 +371,15 @@ jobs: ${{ runner.os }}-bun- - name: Install dependencies - run: bun install --frozen-lockfile + run: bun install --frozen-lockfile --ignore-scripts - # Tolerate a missing artifact: it's only uploaded by the app leg, so its - # absence means the app image didn't build → no ECS deploy happened. - name: Download dev cutover metadata - id: meta - continue-on-error: true uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: dev-cutover-meta path: dev-meta - - name: Determine whether an app deploy happened - id: appdeploy - run: | - if [ -f dev-meta/app_image_changed.txt ]; then - echo "deployed=true" >> "$GITHUB_OUTPUT" - else - echo "deployed=false" >> "$GITHUB_OUTPUT" - echo "::warning::No app-image metadata (app leg did not build); skipping dev task promotion." - fi - - name: Configure AWS credentials - if: steps.appdeploy.outputs.deployed == 'true' uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE_TO_ASSUME }} @@ -415,22 +387,21 @@ jobs: role-duration-seconds: 2400 - name: Wait for ECS traffic cutover - if: steps.appdeploy.outputs.deployed == 'true' env: OVERALL_TIMEOUT: "1200" run: | set -eo pipefail CHANGED=$(cat dev-meta/app_image_changed.txt) - if [ "$CHANGED" != "true" ]; then - echo "App image unchanged — no dev ECS deploy triggered; promoting immediately." - exit 0 - fi DIGEST=$(cat dev-meta/digest.txt) EPOCH=$(cat dev-meta/retag_epoch.txt) + case "$CHANGED" in + true) ;; + false) EPOCH=0 ;; + *) echo "ERROR: invalid app image change metadata" >&2; exit 1 ;; + esac bash .github/scripts/wait-for-ecs-cutover.sh sim-dev-us-east-1-app-deployment "$DIGEST" "$EPOCH" - name: Promote Trigger.dev version - if: steps.appdeploy.outputs.deployed == 'true' working-directory: ./apps/sim env: TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} @@ -447,7 +418,7 @@ jobs: exit 1 fi echo "Promoting Trigger.dev version $VERSION (preview / dev-sim)" - bunx trigger.dev@4.4.3 promote "$VERSION" --env preview --branch dev-sim + bunx trigger.dev@4.5.12 promote "$VERSION" --env preview --branch dev-sim # Main/staging: build & upload the Trigger.dev task version WITHOUT promoting it # (--skip-promotion). New runs keep executing the OLD promoted version until @@ -460,12 +431,14 @@ jobs: name: Deploy Trigger.dev needs: [migrate] if: >- + !cancelled() && + needs.migrate.result == 'success' && github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 15 outputs: - version: ${{ steps.deploy.outputs.version }} + version: ${{ steps.deploy.outputs.deploymentVersion }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -473,7 +446,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.13 + bun-version: 1.4.1 - name: Cache Bun dependencies uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 @@ -487,7 +460,7 @@ jobs: ${{ runner.os }}-bun- - name: Install dependencies - run: bun install --frozen-lockfile + run: bun install --frozen-lockfile --ignore-scripts - name: Deploy to Trigger.dev (skip promotion) id: deploy @@ -502,24 +475,16 @@ jobs: echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 exit 1 fi - bunx trigger.dev@4.4.3 deploy --env "$TRIGGER_ENV" --skip-promotion 2>&1 | tee deploy.log - # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation. - # Anchor on the "version" keyword and take the FIRST match: with - # --skip-promotion the CLI can print the unchanged current version AFTER - # the one it just deployed, and dashboard URLs carry other IDs — so a bare - # last-match could promote the wrong version. Fall back to a bare - # first-match only if no version-labelled line is present. - CLEAN=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log) - VERSION=$(printf '%s\n' "$CLEAN" | grep -oiE 'version[[:space:]]+v?20[0-9]{6}\.[0-9]+' | grep -oE '20[0-9]{6}\.[0-9]+' | head -n1 || true) - if [ -z "$VERSION" ]; then - VERSION=$(printf '%s\n' "$CLEAN" | grep -oE '20[0-9]{6}\.[0-9]+' | head -n1 || true) - fi - if [ -z "$VERSION" ]; then - echo "ERROR: could not parse deployed version from deploy output" >&2 + bunx trigger.dev@4.5.12 deploy --env "$TRIGGER_ENV" --skip-promotion + + - name: Validate deployment version output + env: + VERSION: ${{ steps.deploy.outputs.deploymentVersion }} + run: | + if ! [[ "$VERSION" =~ ^[0-9]{8}\.[0-9]+$ ]]; then + echo "ERROR: Trigger.dev did not report a valid deploymentVersion output" >&2 exit 1 fi - echo "Captured deployed version: $VERSION" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR. # Runs in parallel with tests — only immutable sha tags are pushed here, and @@ -639,7 +604,6 @@ jobs: - name: Build and push images if: steps.meta.outputs.skip != 'true' - id: build uses: ./.github/actions/docker-build with: provider: ${{ vars.CI_PROVIDER }} @@ -648,25 +612,6 @@ jobs: tags: ${{ steps.meta.outputs.tags }} max-cache-size-mb: ${{ matrix.cache_mb }} - # Publish the app image digest so promote-trigger-* can correlate this push - # to its ECS CodePipeline execution. promote-images retags this same sha - # image to latest/staging (preserving the digest), so the pipeline's ECR - # source revision equals this digest — the only durable key (the deploy tag - # is floating). App leg only. - - name: Publish app image digest - if: matrix.ecr_repo_secret == 'ECR_APP' - run: | - mkdir -p digest - echo "${{ steps.build.outputs.digest }}" > digest/app-image-digest.txt - - - name: Upload app image digest - if: matrix.ecr_repo_secret == 'ECR_APP' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: app-image-digest - path: digest/app-image-digest.txt - retention-days: 1 - # Promote the sha-tagged ECR images to the deploy tags once tests and # migrations pass. Pushing the ECR latest/staging tag is what triggers # CodePipeline, so this seconds-long manifest retag is the deploy gate — @@ -697,10 +642,13 @@ jobs: # promote-trigger passes it to the poll script so a stale pipeline execution # reusing the same image digest can't satisfy the cutover gate. retag_epoch: ${{ steps.promote.outputs.retag_epoch }} - # 'false' when the app deploy tag didn't move to a new digest (no ECS deploy). - # promote-trigger promotes immediately in that case instead of waiting. + # Unchanged tags verify the latest execution's cutover without an epoch bound. app_image_changed: ${{ steps.promote.outputs.app_image_changed }} + app_image_digest: ${{ steps.promote.outputs.app_image_digest }} steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: @@ -751,31 +699,8 @@ jobs: ECR_TAG="staging" fi - # Detect whether the APP deploy tag actually moves to a new digest. If - # this commit's app image is byte-identical to the currently-deployed one - # (e.g. a commit that doesn't touch the app image — docs/CI-only), the - # retag is a no-op, ECR fires no push event, and no ECS app deploy runs. - # promote-trigger reads this to promote immediately instead of waiting for - # a cutover that will never happen. - # - # Read digests via the ECR API, which cleanly returns "None" for an absent - # tag (first deploy → changed) vs a non-zero exit on a real read error. On - # a read error we fall through to changed=true (wait) — the safe direction - # (old tasks stay current, job fails visibly) rather than promoting early. APP_REPO="${{ secrets.ECR_APP }}" - ecr_digest() { aws ecr batch-get-image --repository-name "$1" --image-ids imageTag="$2" --query 'images[0].imageId.imageDigest' --output text 2>/dev/null; } - NEW_APP_DIGEST="$(ecr_digest "$APP_REPO" "${{ github.sha }}")" || NEW_APP_DIGEST="__ERR__" - PREV_APP_DIGEST="$(ecr_digest "$APP_REPO" "${ECR_TAG}")" || PREV_APP_DIGEST="__ERR__" - if [ "$NEW_APP_DIGEST" = "__ERR__" ] || [ "$NEW_APP_DIGEST" = "None" ] || [ -z "$NEW_APP_DIGEST" ]; then - echo "ERROR: could not resolve the new app image digest for ${{ github.sha }}" >&2 - exit 1 - fi - if [ "$PREV_APP_DIGEST" != "__ERR__" ] && [ "$NEW_APP_DIGEST" = "$PREV_APP_DIGEST" ]; then - echo "app_image_changed=false" >> "$GITHUB_OUTPUT" - echo "ℹ️ App deploy tag ${ECR_TAG} already points at ${NEW_APP_DIGEST}; no ECS app deploy will be triggered." - else - echo "app_image_changed=true" >> "$GITHUB_OUTPUT" - fi + PREV_APP_DIGEST=$(bash .github/scripts/get-ecr-image-digest.sh "$APP_REPO" "$ECR_TAG" --allow-missing) # Verify every sha image exists before moving any deploy tag, so a # missing/expired image aborts the whole promotion up front. @@ -791,10 +716,17 @@ jobs: "${REGISTRY}/${repo}:${{ github.sha }}" done - # Main/staging: promote the skip-promoted Trigger.dev version at the exact moment - # the ECS app deploy shifts traffic (CodeDeploy AllowTraffic on every target), so - # tasks and app cut over in lockstep. The promote-images retag is what triggers - # the ECS pipeline; this job correlates it via the app image digest + retag epoch + APP_DIGEST=$(bash .github/scripts/get-ecr-image-digest.sh "$APP_REPO" "$ECR_TAG") + echo "app_image_digest=$APP_DIGEST" >> "$GITHUB_OUTPUT" + if [ "$APP_DIGEST" = "$PREV_APP_DIGEST" ]; then + echo "app_image_changed=false" >> "$GITHUB_OUTPUT" + else + echo "app_image_changed=true" >> "$GITHUB_OUTPUT" + fi + + # Main/staging: promote the parked Trigger.dev version after observing the ECS + # traffic cutover (CodeDeploy AllowTraffic on every target). The image retag + # triggers the ECS pipeline; this job correlates it via the digest + retag epoch # (rejecting a stale execution reusing the digest) and promotes at cutover. # Skipped when promote-images skipped the tag move (stale run) — tasks then # correctly stay on the old version. If the app deploy fails or never cuts over, @@ -802,17 +734,15 @@ jobs: promote-trigger: name: Promote Trigger.dev needs: [promote-images, deploy-trigger] - # Require both upstreams to have SUCCEEDED explicitly (not just promoted==true): - # a job if without a status-check function keeps the implicit success() gate, but - # spelling it out removes any doubt that a failed promote-images/deploy-trigger - # can't reach this job and promote tasks with no app retag/cutover. + # Explicit results also suppress skip propagation from optional ancestors. if: >- + !cancelled() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') && needs.promote-images.result == 'success' && needs.deploy-trigger.result == 'success' && needs.promote-images.outputs.promoted == 'true' - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Must exceed the poll script's OVERALL_TIMEOUT (70 min, covering a prod deploy # queued behind a ~50-min bake) PLUS runner setup + the final promote step, so # the Actions timeout never kills the job before the script's own deadline. @@ -827,7 +757,7 @@ jobs: - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.13 + bun-version: 1.4.1 - name: Cache Bun dependencies uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 @@ -841,13 +771,7 @@ jobs: ${{ runner.os }}-bun- - name: Install dependencies - run: bun install --frozen-lockfile - - - name: Download app image digest - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - name: app-image-digest - path: digest + run: bun install --frozen-lockfile --ignore-scripts - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 @@ -860,17 +784,21 @@ jobs: # MaxSessionDuration to be >= this value (roles are managed outside the repo). role-duration-seconds: 5400 - # Skip the cutover wait when the app image didn't change (no ECS deploy was - # triggered) — otherwise the poll would hang until timeout. Promotion still - # runs below, immediately, since there is no app cutover to align with. + # An unchanged tag may belong to a failed or still-running earlier deploy. + # Verify its latest cutover rather than treating tag equality as success. - name: Wait for ECS traffic cutover - if: needs.promote-images.outputs.app_image_changed == 'true' env: + APP_IMAGE_CHANGED: ${{ needs.promote-images.outputs.app_image_changed }} + DIGEST: ${{ needs.promote-images.outputs.app_image_digest }} PIPELINE: sim-${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}-us-east-1-app-deployment RETAG_EPOCH: ${{ needs.promote-images.outputs.retag_epoch }} run: | set -eo pipefail - DIGEST=$(cat digest/app-image-digest.txt) + case "$APP_IMAGE_CHANGED" in + true) ;; + false) RETAG_EPOCH=0 ;; + *) echo "ERROR: invalid app image change metadata" >&2; exit 1 ;; + esac bash .github/scripts/wait-for-ecs-cutover.sh "$PIPELINE" "$DIGEST" "$RETAG_EPOCH" - name: Promote Trigger.dev version @@ -891,7 +819,7 @@ jobs: exit 1 fi echo "Promoting Trigger.dev version $VERSION ($TRIGGER_ENV)" - bunx trigger.dev@4.4.3 promote "$VERSION" --env "$TRIGGER_ENV" + bunx trigger.dev@4.5.12 promote "$VERSION" --env "$TRIGGER_ENV" # Build ARM64 images for GHCR (main branch only, runs in parallel with # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64 diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index d7122f1e889..abf812907db 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -347,6 +347,9 @@ jobs: - name: Lint code run: bun run lint:check + - name: Test Trigger deployment gates + run: python3 .github/scripts/test-trigger-deploy.py + # Every zero-argument `check:*` script, run concurrently. The list is derived in # scripts/run-audits.ts, which also writes the per-audit timing table to the job # summary and annotates failures. Audits needing a base ref stay separate below. From ec2f0bc7617561a8b49fe453ed3e746c0ce56884 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 16 Sep 2026 13:16:50 -0700 Subject: [PATCH 13/15] fix(ci): bind Trigger promotion to the latest app tag move --- .github/actions/docker-build/action.yml | 7 --- .github/scripts/promote-app-image.sh | 20 +++++++ .github/scripts/test-trigger-deploy.py | 68 ++++++++++++++++++++++- .github/scripts/wait-for-ecs-cutover.sh | 24 ++++++-- .github/workflows/ci.yml | 74 +++++++++---------------- 5 files changed, 129 insertions(+), 64 deletions(-) create mode 100644 .github/scripts/promote-app-image.sh diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 2be32c5591c..72d90e8e8bb 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -30,11 +30,6 @@ inputs: bypass an input `default:` entirely. required: false -outputs: - digest: - description: The image digest returned by the selected build provider. - value: ${{ steps.build-blacksmith.outputs.digest || steps.build-github.outputs.digest }} - # Registry logins must precede this action. provenance/sbom stay off: attestation # manifests break `imagetools create` retagging in promote-images. runs: @@ -69,7 +64,6 @@ runs: cache-key: ${{ steps.cache-key.outputs.value }} - name: Build and push (Blacksmith) - id: build-blacksmith if: inputs.provider == '' || inputs.provider == 'blacksmith' uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 with: @@ -175,7 +169,6 @@ runs: # No cache-to: type=gha — it shares the 10 GB repo quota with the cache mounts. - name: Build and push (GitHub) - id: build-github if: inputs.provider != '' && inputs.provider != 'blacksmith' uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: diff --git a/.github/scripts/promote-app-image.sh b/.github/scripts/promote-app-image.sh new file mode 100644 index 00000000000..39cb88f23ff --- /dev/null +++ b/.github/scripts/promote-app-image.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Capture the cutover lower bound at the app tag move, after the image is built. +set -euo pipefail +REGISTRY="${1:?registry required}" +REPOSITORY="${2:?repository required}" +SOURCE_TAG="${3:?source tag required}" +DEPLOY_TAG="${4:?deploy tag required}" +: "${GITHUB_OUTPUT:?GitHub output file required}" +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PREVIOUS=$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG" --allow-missing) +EPOCH=$(date +%s) +docker buildx imagetools create -t "$REGISTRY/$REPOSITORY:$DEPLOY_TAG" "$REGISTRY/$REPOSITORY:$SOURCE_TAG" +DIGEST=$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG") +CHANGED=true +if [ "$DIGEST" = "$PREVIOUS" ]; then CHANGED=false; fi +{ + echo "retag_epoch=$EPOCH" + echo "app_image_digest=$DIGEST" + echo "app_image_changed=$CHANGED" +} >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/test-trigger-deploy.py b/.github/scripts/test-trigger-deploy.py index 6041631acd6..0def19bd959 100644 --- a/.github/scripts/test-trigger-deploy.py +++ b/.github/scripts/test-trigger-deploy.py @@ -48,7 +48,17 @@ def run_script(self, script, args, responses): if response.get('error'): sys.stderr.write(response['error']) sys.exit(254) +if response.get('advance_clock'): + clock = root / 'clock' + value = int(clock.read_text()) if clock.exists() else 1000 + clock.write_text(str(value + response['advance_clock'])) print(response.get('text', json.dumps(response.get('json')))) +''') + (root / 'docker').write_text('''#!/usr/bin/env python3 +import os, pathlib, sys +root = pathlib.Path(os.environ['FIXTURE_DIR']) +with (root / 'calls').open('a') as stream: + stream.write('docker ' + ' '.join(sys.argv[1:]) + '\\n') ''') (root / 'date').write_text('''#!/usr/bin/env python3 import os, pathlib @@ -58,15 +68,17 @@ def run_script(self, script, args, responses): print(value) ''') (root / 'sleep').write_text('#!/bin/sh\nexit 0\n') - for name in ('aws', 'date', 'sleep'): + for name in ('aws', 'date', 'sleep', 'docker'): (root / name).chmod(0o755) result = subprocess.run( ['bash', str(SCRIPTS / script), *args], env={**os.environ, 'PATH': f'{root}:{os.environ["PATH"]}', - 'FIXTURE_DIR': str(root), 'POLL_INTERVAL': '1', 'OVERALL_TIMEOUT': '12'}, + 'FIXTURE_DIR': str(root), 'POLL_INTERVAL': '1', 'OVERALL_TIMEOUT': '12', + 'GITHUB_OUTPUT': str(root / 'outputs')}, capture_output=True, text=True, timeout=10, ) calls = (root / 'calls').read_text() if (root / 'calls').exists() else '' + result.github_output = (root / 'outputs').read_text() if (root / 'outputs').exists() else '' return result, calls def poll(self, updates=None, since='1000'): @@ -100,6 +112,32 @@ def test_chooses_newest_matching_execution(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertIn('--pipeline-execution-id execution-current', calls) + def test_changed_image_rejects_newer_different_execution(self): + result, calls = self.poll({'list-pipeline-executions': {'json': [ + execution(), execution(1001, digest=OTHER_DIGEST, identifier='execution-newer')]}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('deployment was superseded', result.stderr) + self.assertNotIn('get-pipeline-execution ', calls) + + def test_rechecks_latest_digest_after_cutover(self): + result, calls = self.poll({'list-pipeline-executions': [ + {'json': [execution()]}, + {'json': [execution(), execution(1001, digest=OTHER_DIGEST, identifier='execution-newer')]}, + ]}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('deployment was superseded', result.stderr) + self.assertIn('get-deployment-target ', calls) + self.assertNotIn('Traffic cutover complete', result.stdout) + + def test_rechecks_execution_identity_for_same_digest_after_cutover(self): + result, _ = self.poll({'list-pipeline-executions': [ + {'json': [execution()]}, + {'json': [execution(), execution(1001, identifier='execution-newer')]}, + ]}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('newer pipeline execution appeared', result.stdout) + self.assertNotIn('Traffic cutover complete', result.stdout) + def test_iso_timestamps(self): result, _ = self.poll({'list-pipeline-executions': {'json': [ execution('1970-01-01T00:16:40+00:00')]}}) @@ -182,6 +220,32 @@ def test_ecr_response_failures_are_not_missing_images(self): result, _ = self.run_script('get-ecr-image-digest.sh', ['app', 'deploy', '--allow-missing'], {'batch-get-image': response}) self.assertNotEqual(result.returncode, 0) + def test_tag_move_uses_push_boundary_and_final_manifest_digest(self): + result, calls = self.run_script('promote-app-image.sh', ['registry', 'app', 'commit-dev', 'dev'], { + 'batch-get-image': [ + {'advance_clock': 30, 'json': {'images': [{'imageId': {'imageDigest': DIGEST}}], 'failures': []}}, + {'json': {'images': [{'imageId': {'imageDigest': OTHER_DIGEST}}], 'failures': []}}, + ]}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('retag_epoch=1030', result.github_output) + self.assertIn(f'app_image_digest={OTHER_DIGEST}', result.github_output) + self.assertIn('app_image_changed=true', result.github_output) + self.assertEqual([line.split()[0] for line in calls.splitlines()], ['ecr', 'docker', 'ecr']) + self.assertIn('registry/app:commit-dev', calls) + + def test_tag_move_aborts_before_docker_when_ecr_read_fails(self): + result, calls = self.run_script('promote-app-image.sh', ['registry', 'app', 'commit', 'deploy'], { + 'batch-get-image': {'error': 'AccessDeniedException'}}) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('docker', calls) + self.assertEqual(result.github_output, '') + + def test_same_digest_tag_move_reports_unchanged(self): + result, _ = self.run_script('promote-app-image.sh', ['registry', 'app', 'commit', 'deploy'], { + 'batch-get-image': {'json': {'images': [{'imageId': {'imageDigest': DIGEST}}], 'failures': []}}}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('app_image_changed=false', result.github_output) + if __name__ == '__main__': unittest.main() diff --git a/.github/scripts/wait-for-ecs-cutover.sh b/.github/scripts/wait-for-ecs-cutover.sh index a925c061ad9..b18758378fa 100755 --- a/.github/scripts/wait-for-ecs-cutover.sh +++ b/.github/scripts/wait-for-ecs-cutover.sh @@ -30,13 +30,12 @@ aws_read() { aws --cli-connect-timeout 10 --cli-read-timeout 30 "$@" } -EXECUTION_ID='' -while [ -z "$EXECUTION_ID" ]; do - check_deadline 'the matching pipeline execution' +find_execution() { + local executions executions=$(aws_read codepipeline list-pipeline-executions \ --pipeline-name "$PIPELINE" --max-items 30 \ --query 'pipelineExecutionSummaries' --output json) - EXECUTION_ID=$(printf '%s\n' "$executions" | SINCE="$SINCE_EPOCH" DIGEST="$DIGEST" python3 -c ' + printf '%s\n' "$executions" | SINCE="$SINCE_EPOCH" DIGEST="$DIGEST" python3 -c ' import datetime, json, os, sys since = int(os.environ["SINCE"]) def epoch(execution): @@ -52,9 +51,17 @@ if since == 0: raise SystemExit("ERROR: unchanged app tag does not match the latest pipeline execution; cutover is unverified") selected = executions[0] else: - selected = next((e for e in executions if epoch(e) >= since and matches(e)), None) + selected = executions[0] if executions and epoch(executions[0]) >= since else None + if selected and not matches(selected): + raise SystemExit("ERROR: latest pipeline execution does not match this app digest; deployment was superseded or its source is unverified") print(selected["pipelineExecutionId"] if selected else "") -') +' +} + +EXECUTION_ID='' +while [ -z "$EXECUTION_ID" ]; do + check_deadline 'the matching pipeline execution' + EXECUTION_ID=$(find_execution) if [ -z "$EXECUTION_ID" ]; then log 'No matching execution since this push; waiting' sleep "$POLL_INTERVAL" @@ -112,6 +119,11 @@ while true; do esac done if [ "$all_ok" = 1 ]; then + LATEST_EXECUTION_ID=$(find_execution) + if [ "$LATEST_EXECUTION_ID" != "$EXECUTION_ID" ]; then + log 'ERROR: a newer pipeline execution appeared during cutover; not promoting' + exit 1 + fi log 'Traffic cutover complete on every ECS target' exit 0 fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25efcb0bd8e..b8d9c64982b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,48 +215,34 @@ jobs: env: ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }} - # Capture the previous tag and timestamp before the build pushes :dev. - # Only a missing tag is allowed; failed reads abort before deployment. - - name: Capture pre-build :dev state - id: prevdigest - if: matrix.ecr_repo_secret == 'ECR_APP' - run: | - echo "epoch=$(date +%s)" >> "$GITHUB_OUTPUT" - PREV=$(bash .github/scripts/get-ecr-image-digest.sh "${{ steps.ecr-repo.outputs.name }}" dev --allow-missing) - echo "digest=${PREV}" >> "$GITHUB_OUTPUT" - - name: Build and push - id: build uses: ./.github/actions/docker-build with: provider: ${{ vars.CI_PROVIDER }} file: ${{ matrix.dockerfile }} platforms: linux/amd64 - tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev + tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:${{ matrix.ecr_repo_secret == 'ECR_APP' && format('{0}-dev', github.sha) || 'dev' }} max-cache-size-mb: ${{ matrix.cache_mb }} - # App leg only: publish the metadata promote-trigger-dev needs to correlate - # this push to its dev ECS deploy and decide whether to wait. Dev has no - # promote-images job, so this stands in for its retag_epoch/app_image_changed - # outputs. The epoch and prev digest come from the pre-build step above. + - name: Promote dev app image + id: appdeploy + if: matrix.ecr_repo_secret == 'ECR_APP' + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ steps.ecr-repo.outputs.name }} + run: bash .github/scripts/promote-app-image.sh "$REGISTRY" "$REPOSITORY" "${GITHUB_SHA}-dev" dev + - name: Publish dev cutover metadata if: matrix.ecr_repo_secret == 'ECR_APP' + env: + DIGEST: ${{ steps.appdeploy.outputs.app_image_digest }} + EPOCH: ${{ steps.appdeploy.outputs.retag_epoch }} + CHANGED: ${{ steps.appdeploy.outputs.app_image_changed }} run: | mkdir -p dev-meta - NEW="${{ steps.build.outputs.digest }}" - PREV="${{ steps.prevdigest.outputs.digest }}" - if [ -z "$NEW" ]; then - echo "ERROR: build did not report an image digest" >&2 - exit 1 - fi - echo "$NEW" > dev-meta/digest.txt - echo "${{ steps.prevdigest.outputs.epoch }}" > dev-meta/retag_epoch.txt - if [ "$NEW" = "$PREV" ]; then - echo "false" > dev-meta/app_image_changed.txt - echo "ℹ️ :dev already points at ${NEW}; no ECS dev deploy will be triggered." - else - echo "true" > dev-meta/app_image_changed.txt - fi + echo "$DIGEST" > dev-meta/digest.txt + echo "$EPOCH" > dev-meta/retag_epoch.txt + echo "$CHANGED" > dev-meta/app_image_changed.txt - name: Upload dev cutover metadata if: matrix.ecr_repo_secret == 'ECR_APP' @@ -325,8 +311,8 @@ jobs: fi # Dev: promote the skip-promoted preview version at the dev ECS traffic cutover. - # Dev has no promote-images gate (build-dev pushes :dev directly), so the digest, - # trigger epoch, and app-image-changed signal come from build-dev's artifact. + # The dev app build moves :dev only after building its commit-tagged image, + # then passes the tag digest and retag timestamp through an artifact. # trigger.dev supports promoting a specific preview branch: promote --env preview # --branch dev-sim. promote-trigger-dev: @@ -686,11 +672,6 @@ jobs: ${{ secrets.ECR_REALTIME }} ${{ secrets.ECR_PII }} run: | - # Record the retag time BEFORE moving any tag — this is when the ECS - # pipeline for this push is triggered. promote-trigger uses it to - # reject an older pipeline execution reusing the same image digest. - echo "retag_epoch=$(date +%s)" >> "$GITHUB_OUTPUT" - REGISTRY="${{ steps.login-ecr.outputs.registry }}" if [ "${{ github.ref }}" = "refs/heads/main" ]; then @@ -700,7 +681,6 @@ jobs: fi APP_REPO="${{ secrets.ECR_APP }}" - PREV_APP_DIGEST=$(bash .github/scripts/get-ecr-image-digest.sh "$APP_REPO" "$ECR_TAG" --allow-missing) # Verify every sha image exists before moving any deploy tag, so a # missing/expired image aborts the whole promotion up front. @@ -711,19 +691,15 @@ jobs: for repo in $ECR_REPOS; do echo "🚀 Promoting ${repo}:${{ github.sha }} to ${ECR_TAG}" - docker buildx imagetools create \ - -t "${REGISTRY}/${repo}:${ECR_TAG}" \ - "${REGISTRY}/${repo}:${{ github.sha }}" + if [ "$repo" = "$APP_REPO" ]; then + bash .github/scripts/promote-app-image.sh "$REGISTRY" "$APP_REPO" "$GITHUB_SHA" "$ECR_TAG" + else + docker buildx imagetools create \ + -t "${REGISTRY}/${repo}:${ECR_TAG}" \ + "${REGISTRY}/${repo}:${{ github.sha }}" + fi done - APP_DIGEST=$(bash .github/scripts/get-ecr-image-digest.sh "$APP_REPO" "$ECR_TAG") - echo "app_image_digest=$APP_DIGEST" >> "$GITHUB_OUTPUT" - if [ "$APP_DIGEST" = "$PREV_APP_DIGEST" ]; then - echo "app_image_changed=false" >> "$GITHUB_OUTPUT" - else - echo "app_image_changed=true" >> "$GITHUB_OUTPUT" - fi - # Main/staging: promote the parked Trigger.dev version after observing the ECS # traffic cutover (CodeDeploy AllowTraffic on every target). The image retag # triggers the ECS pipeline; this job correlates it via the digest + retag epoch From d7dd8f62106cc5fb73d3dc91d46b462973745818 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 16 Sep 2026 13:24:36 -0700 Subject: [PATCH 14/15] test(ci): reject exhausted deployment response fixtures --- .github/scripts/test-trigger-deploy.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/scripts/test-trigger-deploy.py b/.github/scripts/test-trigger-deploy.py index 0def19bd959..2183d357da7 100644 --- a/.github/scripts/test-trigger-deploy.py +++ b/.github/scripts/test-trigger-deploy.py @@ -41,10 +41,13 @@ def run_script(self, script, args, responses): if key not in responses: raise SystemExit('Unexpected AWS call: ' + key) response = responses[key] +# Objects model steady state; lists are finite, ordered expectations. if isinstance(response, list): - response = response.pop(0) - responses[key] = response if not responses[key] else responses[key] + if not response: + raise SystemExit('Unexpected extra AWS call: ' + key) + next_response = response.pop(0) (root / 'responses.json').write_text(json.dumps(responses)) + response = next_response if response.get('error'): sys.stderr.write(response['error']) sys.exit(254) @@ -143,6 +146,12 @@ def test_iso_timestamps(self): execution('1970-01-01T00:16:40+00:00')]}}) self.assertEqual(result.returncode, 0, result.stderr) + def test_scripted_responses_reject_unexpected_extra_calls(self): + result, _ = self.poll({'list-pipeline-executions': [{'json': [execution()]}]}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('Unexpected extra AWS call: list-pipeline-executions', result.stderr) + self.assertNotIn('Traffic cutover complete', result.stdout) + def test_access_denial_fails_immediately(self): result, calls = self.poll({'list-pipeline-executions': {'error': 'AccessDeniedException'}}) self.assertNotEqual(result.returncode, 0) From 72cea20534a68343572efe5fa32321d367afa2f4 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 16 Sep 2026 13:29:43 -0700 Subject: [PATCH 15/15] test(ci): require cutover checks for every ECS target --- .github/scripts/test-trigger-deploy.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/scripts/test-trigger-deploy.py b/.github/scripts/test-trigger-deploy.py index 2183d357da7..1ea31f4961a 100644 --- a/.github/scripts/test-trigger-deploy.py +++ b/.github/scripts/test-trigger-deploy.py @@ -98,10 +98,14 @@ def poll(self, updates=None, since='1000'): return self.run_script('wait-for-ecs-cutover.sh', ['app-pipeline', DIGEST, since], responses) def test_waits_for_every_target(self): - result, calls = self.poll({'get-deployment-target:target-two': [ - {'text': 'InProgress'}, {'text': 'Succeeded'}]}) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(calls.count('--target-id target-two'), 2) + targets = ('target-one', 'target-two') + for pending_target in targets: + with self.subTest(pending_target=pending_target): + result, calls = self.poll({f'get-deployment-target:{pending_target}': [ + {'text': 'InProgress'}, {'text': 'Succeeded'}]}) + self.assertEqual(result.returncode, 0, result.stderr) + for target in targets: + self.assertEqual(calls.count(f'--target-id {target}'), 2) def test_rejects_stale_execution_inside_former_clock_skew_window(self): result, calls = self.poll({'list-pipeline-executions': {'json': [execution(start=999)]}})