diff --git a/.github/scripts/get-ecr-image-digest.sh b/.github/scripts/get-ecr-image-digest.sh new file mode 100644 index 00000000000..f8de53f925e --- /dev/null +++ b/.github/scripts/get-ecr-image-digest.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Read one ECR tag. Only ImageNotFound is optional; AWS and response errors fail. +set -euo pipefail +REPOSITORY="${1:?repository required}" +TAG="${2:?tag required}" +ALLOW_MISSING="${3:-}" +if [ -n "$ALLOW_MISSING" ] && [ "$ALLOW_MISSING" != '--allow-missing' ]; then + echo 'ERROR: expected --allow-missing or no third argument' >&2 + exit 1 +fi +export AWS_PAGER='' +aws ecr batch-get-image --repository-name "$REPOSITORY" --image-ids imageTag="$TAG" --output json | + ALLOW_MISSING="$ALLOW_MISSING" python3 -c ' +import json, os, re, sys +response = json.load(sys.stdin) +images, failures = response["images"], response["failures"] +if failures: + if not images and len(failures) == 1 and failures[0]["failureCode"] == "ImageNotFound" and os.environ["ALLOW_MISSING"]: + print("") + sys.exit(0) + raise SystemExit("ERROR: ECR image lookup failed: " + ", ".join(f["failureCode"] for f in failures)) +if len(images) != 1: + raise SystemExit("ERROR: expected exactly one ECR image") +digest = images[0]["imageId"]["imageDigest"] +if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): + raise SystemExit("ERROR: invalid ECR image digest") +print(digest) +' diff --git a/.github/scripts/promote-app-image.sh b/.github/scripts/promote-app-image.sh new file mode 100644 index 00000000000..39cb88f23ff --- /dev/null +++ b/.github/scripts/promote-app-image.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Capture the cutover lower bound at the app tag move, after the image is built. +set -euo pipefail +REGISTRY="${1:?registry required}" +REPOSITORY="${2:?repository required}" +SOURCE_TAG="${3:?source tag required}" +DEPLOY_TAG="${4:?deploy tag required}" +: "${GITHUB_OUTPUT:?GitHub output file required}" +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PREVIOUS=$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG" --allow-missing) +EPOCH=$(date +%s) +docker buildx imagetools create -t "$REGISTRY/$REPOSITORY:$DEPLOY_TAG" "$REGISTRY/$REPOSITORY:$SOURCE_TAG" +DIGEST=$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG") +CHANGED=true +if [ "$DIGEST" = "$PREVIOUS" ]; then CHANGED=false; fi +{ + echo "retag_epoch=$EPOCH" + echo "app_image_digest=$DIGEST" + echo "app_image_changed=$CHANGED" +} >> "$GITHUB_OUTPUT" diff --git a/.github/scripts/test-trigger-deploy.py b/.github/scripts/test-trigger-deploy.py new file mode 100644 index 00000000000..1ea31f4961a --- /dev/null +++ b/.github/scripts/test-trigger-deploy.py @@ -0,0 +1,264 @@ +"""Exercise the deployment gates with scripted AWS responses; no live mutations.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +SCRIPTS = Path(__file__).resolve().parent +DIGEST = 'sha256:' + 'a' * 64 +OTHER_DIGEST = 'sha256:' + 'b' * 64 + + +def execution(start=1000, digest=DIGEST, identifier='execution-current'): + return { + 'startTime': start, + 'pipelineExecutionId': identifier, + 'sourceRevisions': [{'actionName': 'ECR_Source', 'revisionId': digest}], + } + + +class DeploymentGateTests(unittest.TestCase): + def run_script(self, script, args, responses): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture = root / 'responses.json' + fixture.write_text(json.dumps(responses)) + (root / 'aws').write_text('''#!/usr/bin/env python3 +import json, os, pathlib, sys +root = pathlib.Path(os.environ['FIXTURE_DIR']) +args = sys.argv[1:] +if args[0] == '--cli-connect-timeout': + args = args[4:] +service, operation = args[:2] +key = operation +if operation == 'get-deployment-target': + key += ':' + args[args.index('--target-id') + 1] +with (root / 'calls').open('a') as stream: + stream.write(' '.join(args) + '\\n') +responses = json.loads((root / 'responses.json').read_text()) +if key not in responses: + raise SystemExit('Unexpected AWS call: ' + key) +response = responses[key] +# Objects model steady state; lists are finite, ordered expectations. +if isinstance(response, list): + if not response: + raise SystemExit('Unexpected extra AWS call: ' + key) + next_response = response.pop(0) + (root / 'responses.json').write_text(json.dumps(responses)) + response = next_response +if response.get('error'): + sys.stderr.write(response['error']) + sys.exit(254) +if response.get('advance_clock'): + clock = root / 'clock' + value = int(clock.read_text()) if clock.exists() else 1000 + clock.write_text(str(value + response['advance_clock'])) +print(response.get('text', json.dumps(response.get('json')))) +''') + (root / 'docker').write_text('''#!/usr/bin/env python3 +import os, pathlib, sys +root = pathlib.Path(os.environ['FIXTURE_DIR']) +with (root / 'calls').open('a') as stream: + stream.write('docker ' + ' '.join(sys.argv[1:]) + '\\n') +''') + (root / 'date').write_text('''#!/usr/bin/env python3 +import os, pathlib +path = pathlib.Path(os.environ['FIXTURE_DIR']) / 'clock' +value = int(path.read_text()) if path.exists() else 1000 +path.write_text(str(value + 1)) +print(value) +''') + (root / 'sleep').write_text('#!/bin/sh\nexit 0\n') + for name in ('aws', 'date', 'sleep', 'docker'): + (root / name).chmod(0o755) + result = subprocess.run( + ['bash', str(SCRIPTS / script), *args], + env={**os.environ, 'PATH': f'{root}:{os.environ["PATH"]}', + 'FIXTURE_DIR': str(root), 'POLL_INTERVAL': '1', 'OVERALL_TIMEOUT': '12', + 'GITHUB_OUTPUT': str(root / 'outputs')}, + capture_output=True, text=True, timeout=10, + ) + calls = (root / 'calls').read_text() if (root / 'calls').exists() else '' + result.github_output = (root / 'outputs').read_text() if (root / 'outputs').exists() else '' + return result, calls + + def poll(self, updates=None, since='1000'): + responses = { + 'list-pipeline-executions': {'json': [execution()]}, + 'get-pipeline-execution': {'text': 'InProgress'}, + 'list-action-executions': {'text': 'd-current'}, + 'get-deployment': {'text': 'InProgress'}, + 'list-deployment-targets': {'text': 'target-one\ttarget-two'}, + 'get-deployment-target:target-one': {'text': 'Succeeded'}, + 'get-deployment-target:target-two': {'text': 'Succeeded'}, + } + responses.update(updates or {}) + return self.run_script('wait-for-ecs-cutover.sh', ['app-pipeline', DIGEST, since], responses) + + def test_waits_for_every_target(self): + targets = ('target-one', 'target-two') + for pending_target in targets: + with self.subTest(pending_target=pending_target): + result, calls = self.poll({f'get-deployment-target:{pending_target}': [ + {'text': 'InProgress'}, {'text': 'Succeeded'}]}) + self.assertEqual(result.returncode, 0, result.stderr) + for target in targets: + self.assertEqual(calls.count(f'--target-id {target}'), 2) + + def test_rejects_stale_execution_inside_former_clock_skew_window(self): + result, calls = self.poll({'list-pipeline-executions': {'json': [execution(start=999)]}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('timed out', result.stdout) + self.assertNotIn('get-pipeline-execution ', calls) + + def test_chooses_newest_matching_execution(self): + result, calls = self.poll({'list-pipeline-executions': {'json': [ + execution(1000, identifier='execution-old'), execution(1001)]}}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('--pipeline-execution-id execution-current', calls) + + def test_changed_image_rejects_newer_different_execution(self): + result, calls = self.poll({'list-pipeline-executions': {'json': [ + execution(), execution(1001, digest=OTHER_DIGEST, identifier='execution-newer')]}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('deployment was superseded', result.stderr) + self.assertNotIn('get-pipeline-execution ', calls) + + def test_rechecks_latest_digest_after_cutover(self): + result, calls = self.poll({'list-pipeline-executions': [ + {'json': [execution()]}, + {'json': [execution(), execution(1001, digest=OTHER_DIGEST, identifier='execution-newer')]}, + ]}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('deployment was superseded', result.stderr) + self.assertIn('get-deployment-target ', calls) + self.assertNotIn('Traffic cutover complete', result.stdout) + + def test_rechecks_execution_identity_for_same_digest_after_cutover(self): + result, _ = self.poll({'list-pipeline-executions': [ + {'json': [execution()]}, + {'json': [execution(), execution(1001, identifier='execution-newer')]}, + ]}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('newer pipeline execution appeared', result.stdout) + self.assertNotIn('Traffic cutover complete', result.stdout) + + def test_iso_timestamps(self): + result, _ = self.poll({'list-pipeline-executions': {'json': [ + execution('1970-01-01T00:16:40+00:00')]}}) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_scripted_responses_reject_unexpected_extra_calls(self): + result, _ = self.poll({'list-pipeline-executions': [{'json': [execution()]}]}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('Unexpected extra AWS call: list-pipeline-executions', result.stderr) + self.assertNotIn('Traffic cutover complete', result.stdout) + + def test_access_denial_fails_immediately(self): + result, calls = self.poll({'list-pipeline-executions': {'error': 'AccessDeniedException'}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('AccessDeniedException', result.stderr) + self.assertEqual(len(calls.splitlines()), 1) + + def test_credentials_expiring_during_target_poll_fail(self): + result, _ = self.poll({'get-deployment-target:target-two': {'error': 'ExpiredToken'}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('ExpiredToken', result.stderr) + + def test_failed_and_superseded_pipeline_never_reach_deployment(self): + for status in ('Failed', 'Stopped', 'Superseded'): + with self.subTest(status=status): + result, calls = self.poll({'get-pipeline-execution': {'text': status}}) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('get-deployment ', calls) + + def test_waits_for_queued_deploy_action(self): + result, calls = self.poll({'list-action-executions': [{'text': 'None'}, {'text': 'd-current'}]}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(calls.count('list-action-executions '), 2) + + def test_failed_deployment_never_accepts_old_cutover(self): + result, calls = self.poll({'get-deployment': {'text': 'Failed'}}) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('get-deployment-target ', calls) + + def test_empty_targets_cannot_satisfy_gate(self): + result, _ = self.poll({'list-deployment-targets': {'text': ''}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('timed out', result.stdout) + + def test_failed_target_fails_immediately(self): + result, _ = self.poll({'get-deployment-target:target-two': {'text': 'Failed'}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn('cutover status Failed', result.stdout) + + def test_unchanged_image_verifies_existing_cutover(self): + result, calls = self.poll({'list-pipeline-executions': {'json': [execution(start=900)]}}, since='0') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('get-deployment-target ', calls) + + def test_unchanged_image_rejects_latest_different_deploy(self): + result, calls = self.poll({'list-pipeline-executions': {'json': [ + execution(start=900), execution(start=999, digest=OTHER_DIGEST)]}}, since='0') + self.assertNotEqual(result.returncode, 0) + self.assertIn('cutover is unverified', result.stderr) + self.assertNotIn('get-deployment ', calls) + + def test_unchanged_image_rejects_failed_previous_deploy(self): + result, _ = self.poll({'get-deployment': {'text': 'Failed'}}, since='0') + self.assertNotEqual(result.returncode, 0) + + def test_invalid_metadata_fails_before_aws(self): + result, calls = self.poll(since='corrupted') + self.assertNotEqual(result.returncode, 0) + self.assertEqual(calls, '') + + def test_ecr_digest_and_missing_tag(self): + result, _ = self.run_script('get-ecr-image-digest.sh', ['app', 'deploy'], { + 'batch-get-image': {'json': {'images': [{'imageId': {'imageDigest': DIGEST}}], 'failures': []}}}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), DIGEST) + missing = {'batch-get-image': {'json': {'images': [], 'failures': [{'failureCode': 'ImageNotFound'}]}}} + result, _ = self.run_script('get-ecr-image-digest.sh', ['app', 'deploy', '--allow-missing'], missing) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), '') + result, _ = self.run_script('get-ecr-image-digest.sh', ['app', 'deploy'], missing) + self.assertNotEqual(result.returncode, 0) + + def test_ecr_response_failures_are_not_missing_images(self): + for response in ({'error': 'AccessDeniedException'}, {'json': {'images': [], 'failures': [{'failureCode': 'KmsError'}]}}, {'json': {'images': [], 'failures': []}}): + with self.subTest(response=response): + result, _ = self.run_script('get-ecr-image-digest.sh', ['app', 'deploy', '--allow-missing'], {'batch-get-image': response}) + self.assertNotEqual(result.returncode, 0) + + def test_tag_move_uses_push_boundary_and_final_manifest_digest(self): + result, calls = self.run_script('promote-app-image.sh', ['registry', 'app', 'commit-dev', 'dev'], { + 'batch-get-image': [ + {'advance_clock': 30, 'json': {'images': [{'imageId': {'imageDigest': DIGEST}}], 'failures': []}}, + {'json': {'images': [{'imageId': {'imageDigest': OTHER_DIGEST}}], 'failures': []}}, + ]}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('retag_epoch=1030', result.github_output) + self.assertIn(f'app_image_digest={OTHER_DIGEST}', result.github_output) + self.assertIn('app_image_changed=true', result.github_output) + self.assertEqual([line.split()[0] for line in calls.splitlines()], ['ecr', 'docker', 'ecr']) + self.assertIn('registry/app:commit-dev', calls) + + def test_tag_move_aborts_before_docker_when_ecr_read_fails(self): + result, calls = self.run_script('promote-app-image.sh', ['registry', 'app', 'commit', 'deploy'], { + 'batch-get-image': {'error': 'AccessDeniedException'}}) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn('docker', calls) + self.assertEqual(result.github_output, '') + + def test_same_digest_tag_move_reports_unchanged(self): + result, _ = self.run_script('promote-app-image.sh', ['registry', 'app', 'commit', 'deploy'], { + 'batch-get-image': {'json': {'images': [{'imageId': {'imageDigest': DIGEST}}], 'failures': []}}}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('app_image_changed=false', result.github_output) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/wait-for-ecs-cutover.sh b/.github/scripts/wait-for-ecs-cutover.sh new file mode 100755 index 00000000000..b18758378fa --- /dev/null +++ b/.github/scripts/wait-for-ecs-cutover.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Resolve a pushed app digest to CodePipeline -> CodeDeploy -> every ECS target's +# AllowTraffic event. An unchanged tag uses since-epoch=0 to verify the latest +# pipeline execution instead of assuming the tagged image is already serving. +# Usage: wait-for-ecs-cutover.sh +set -euo pipefail + +PIPELINE="${1:?pipeline name required}" +DIGEST="${2:?image digest required}" +SINCE_EPOCH="${3:?since-epoch required}" +POLL_INTERVAL="${POLL_INTERVAL:-15}" +OVERALL_TIMEOUT="${OVERALL_TIMEOUT:-4200}" +if ! [[ "$PIPELINE" =~ ^[A-Za-z0-9.@_-]+$ && "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ && "$SINCE_EPOCH" =~ ^[0-9]+$ && "$POLL_INTERVAL" =~ ^[1-9][0-9]*$ && "$OVERALL_TIMEOUT" =~ ^[1-9][0-9]*$ ]]; then + echo 'ERROR: invalid pipeline, digest, epoch, or polling budget' >&2 + exit 1 +fi +export AWS_PAGER='' +export AWS_RETRY_MODE=standard +export AWS_MAX_ATTEMPTS=3 + +deadline=$(( $(date +%s) + OVERALL_TIMEOUT )) +log() { echo "[wait-for-ecs-cutover] $*"; } +check_deadline() { + if [ "$(date +%s)" -ge "$deadline" ]; then + log "ERROR: timed out after ${OVERALL_TIMEOUT}s waiting for $1" + exit 1 + fi +} +aws_read() { + aws --cli-connect-timeout 10 --cli-read-timeout 30 "$@" +} + +find_execution() { + local executions + executions=$(aws_read codepipeline list-pipeline-executions \ + --pipeline-name "$PIPELINE" --max-items 30 \ + --query 'pipelineExecutionSummaries' --output json) + printf '%s\n' "$executions" | SINCE="$SINCE_EPOCH" DIGEST="$DIGEST" python3 -c ' +import datetime, json, os, sys +since = int(os.environ["SINCE"]) +def epoch(execution): + value = execution["startTime"] + if isinstance(value, (int, float)): + return value + return datetime.datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() +def matches(execution): + return any(r["actionName"] == "ECR_Source" and r.get("revisionId") == os.environ["DIGEST"] for r in execution.get("sourceRevisions", [])) +executions = sorted(json.load(sys.stdin), key=epoch, reverse=True) +if since == 0: + if not executions or not matches(executions[0]): + raise SystemExit("ERROR: unchanged app tag does not match the latest pipeline execution; cutover is unverified") + selected = executions[0] +else: + selected = executions[0] if executions and epoch(executions[0]) >= since else None + if selected and not matches(selected): + raise SystemExit("ERROR: latest pipeline execution does not match this app digest; deployment was superseded or its source is unverified") +print(selected["pipelineExecutionId"] if selected else "") +' +} + +EXECUTION_ID='' +while [ -z "$EXECUTION_ID" ]; do + check_deadline 'the matching pipeline execution' + EXECUTION_ID=$(find_execution) + if [ -z "$EXECUTION_ID" ]; then + log 'No matching execution since this push; waiting' + sleep "$POLL_INTERVAL" + fi +done +log "Matched pipeline execution: $EXECUTION_ID" + +DEPLOYMENT_ID='' +while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = 'None' ]; do + check_deadline 'the CodeDeploy deployment (the Deploy stage may be queued)' + status=$(aws_read codepipeline get-pipeline-execution \ + --pipeline-name "$PIPELINE" --pipeline-execution-id "$EXECUTION_ID" \ + --query 'pipelineExecution.status' --output text) + case "$status" in + Failed|Stopped|Stopping|Superseded|Cancelled) + log "ERROR: pipeline execution ended in $status; not promoting"; exit 1 ;; + InProgress|Succeeded) ;; + *) log "ERROR: unexpected pipeline status: $status"; exit 1 ;; + esac + DEPLOYMENT_ID=$(aws_read codepipeline list-action-executions \ + --pipeline-name "$PIPELINE" --filter pipelineExecutionId="$EXECUTION_ID" \ + --query "actionExecutionDetails[?stageName=='Deploy'].output.executionResult.externalExecutionId | [0]" \ + --output text) + if [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = 'None' ]; then + if [ "$status" = 'Succeeded' ]; then + log 'ERROR: successful pipeline has no CodeDeploy deployment'; exit 1 + fi + sleep "$POLL_INTERVAL" + fi +done +log "CodeDeploy deployment: $DEPLOYMENT_ID" + +while true; do + check_deadline 'AllowTraffic on every ECS target' + status=$(aws_read deploy get-deployment --deployment-id "$DEPLOYMENT_ID" \ + --query 'deploymentInfo.status' --output text) + case "$status" in + Failed|Stopped) log "ERROR: deployment ended in $status; not promoting"; exit 1 ;; + Created|Queued|InProgress|Baking|Ready|Succeeded) ;; + *) log "ERROR: unexpected deployment status: $status"; exit 1 ;; + esac + target_ids=$(aws_read deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \ + --query 'targetIds' --output text) + if [ -n "$target_ids" ] && [ "$target_ids" != 'None' ]; then + all_ok=1 + for target in $target_ids; do + cutover=$(aws_read deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$target" \ + --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \ + --output text) + case "$cutover" in + Succeeded) ;; + Failed|Skipped|Unknown) log "ERROR: target $target cutover status $cutover"; exit 1 ;; + Pending|InProgress|None|'') all_ok=0 ;; + *) log "ERROR: unexpected cutover status: $cutover"; exit 1 ;; + esac + done + if [ "$all_ok" = 1 ]; then + LATEST_EXECUTION_ID=$(find_execution) + if [ "$LATEST_EXECUTION_ID" != "$EXECUTION_ID" ]; then + log 'ERROR: a newer pipeline execution appeared during cutover; not promoting' + exit 1 + fi + log 'Traffic cutover complete on every ECS target' + exit 0 + fi + fi + log 'Traffic cutover is not complete; waiting' + sleep "$POLL_INTERVAL" +done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75d45ac5e46..b8d9c64982b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -221,18 +221,210 @@ jobs: provider: ${{ vars.CI_PROVIDER }} file: ${{ matrix.dockerfile }} platforms: linux/amd64 - tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev + tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:${{ matrix.ecr_repo_secret == 'ECR_APP' && format('{0}-dev', github.sha) || 'dev' }} max-cache-size-mb: ${{ matrix.cache_mb }} - # Dev: deploy Trigger.dev background tasks to the preview "dev-sim" branch. - # Gated after migrate-dev for the same reason as build-dev — the new task - # code runs against the dev DB, so the schema must be pushed first. + - name: Promote dev app image + id: appdeploy + if: matrix.ecr_repo_secret == 'ECR_APP' + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ steps.ecr-repo.outputs.name }} + run: bash .github/scripts/promote-app-image.sh "$REGISTRY" "$REPOSITORY" "${GITHUB_SHA}-dev" dev + + - name: Publish dev cutover metadata + if: matrix.ecr_repo_secret == 'ECR_APP' + env: + DIGEST: ${{ steps.appdeploy.outputs.app_image_digest }} + EPOCH: ${{ steps.appdeploy.outputs.retag_epoch }} + CHANGED: ${{ steps.appdeploy.outputs.app_image_changed }} + run: | + mkdir -p dev-meta + echo "$DIGEST" > dev-meta/digest.txt + echo "$EPOCH" > dev-meta/retag_epoch.txt + echo "$CHANGED" > dev-meta/app_image_changed.txt + + - name: Upload dev cutover metadata + if: matrix.ecr_repo_secret == 'ECR_APP' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: dev-cutover-meta + path: dev-meta/ + retention-days: 1 + + # Dev: build & upload the Trigger.dev task version WITHOUT promoting it + # (--skip-promotion) to the preview "dev-sim" branch. promote-trigger-dev flips + # it at the dev ECS traffic cutover. Gated after migrate-dev so the schema is + # pushed before the new task version can run against the dev DB. deploy-trigger-dev: name: Deploy Trigger.dev (Dev) needs: [migrate-dev] if: github.event_name == 'push' && github.ref == 'refs/heads/dev' runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 15 + outputs: + version: ${{ steps.deploy.outputs.deploymentVersion }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.1 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile --ignore-scripts + + - name: Deploy to Trigger.dev (skip promotion) + id: deploy + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + bunx trigger.dev@4.5.12 deploy --env preview --branch dev-sim --skip-promotion + + - name: Validate deployment version output + env: + VERSION: ${{ steps.deploy.outputs.deploymentVersion }} + run: | + if ! [[ "$VERSION" =~ ^[0-9]{8}\.[0-9]+$ ]]; then + echo "ERROR: Trigger.dev did not report a valid deploymentVersion output" >&2 + exit 1 + fi + + # Dev: promote the skip-promoted preview version at the dev ECS traffic cutover. + # The dev app build moves :dev only after building its commit-tagged image, + # then passes the tag digest and retag timestamp through an artifact. + # trigger.dev supports promoting a specific preview branch: promote --env preview + # --branch dev-sim. + promote-trigger-dev: + name: Promote Trigger.dev (Dev) + needs: [build-dev, deploy-trigger-dev] + # Run as long as the task upload succeeded, even if a NON-app build-dev leg + # (realtime/pii/migrations) failed: the app leg pushes :dev independently and + # may have already triggered the ECS deploy, so an unrelated image failure must + # not strand the app on the old task version. The app-metadata artifact (only + # the app leg uploads it) is the real signal that an app deploy happened. + if: >- + !cancelled() && + github.event_name == 'push' && github.ref == 'refs/heads/dev' && + needs.deploy-trigger-dev.result == 'success' + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # Dev bake is 5 min and dev deploys don't queue behind a bake (serialized by the + # ci- group), so a 20-min poll is ample; the 40-min job leaves ~20 min for + # setup + promote above it (mirrors the prod 90-vs-70 margin), and the 40-min + # session outlasts the poll. + timeout-minutes: 40 + permissions: + contents: read + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.1 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile --ignore-scripts + + - name: Download dev cutover metadata + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: dev-cutover-meta + path: dev-meta + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ secrets.DEV_AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.DEV_AWS_REGION }} + role-duration-seconds: 2400 + + - name: Wait for ECS traffic cutover + env: + OVERALL_TIMEOUT: "1200" + run: | + set -eo pipefail + CHANGED=$(cat dev-meta/app_image_changed.txt) + DIGEST=$(cat dev-meta/digest.txt) + EPOCH=$(cat dev-meta/retag_epoch.txt) + case "$CHANGED" in + true) ;; + false) EPOCH=0 ;; + *) echo "ERROR: invalid app image change metadata" >&2; exit 1 ;; + esac + bash .github/scripts/wait-for-ecs-cutover.sh sim-dev-us-east-1-app-deployment "$DIGEST" "$EPOCH" + + - name: Promote Trigger.dev version + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + VERSION: ${{ needs.deploy-trigger-dev.outputs.version }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + if [ -z "$VERSION" ]; then + echo "ERROR: no deployed version passed from deploy-trigger-dev" >&2 + exit 1 + fi + echo "Promoting Trigger.dev version $VERSION (preview / dev-sim)" + bunx trigger.dev@4.5.12 promote "$VERSION" --env preview --branch dev-sim + + # Main/staging: build & upload the Trigger.dev task version WITHOUT promoting it + # (--skip-promotion). New runs keep executing the OLD promoted version until + # promote-trigger flips it at the ECS traffic cutover — so the app cutting over + # never changes which task version runs until promote-trigger (which depends on + # this job) promotes the version uploaded here. Runs in parallel with the build; + # intentionally NOT gating the app deploy on it, to avoid coupling every app / + # realtime / pii / migration deploy to trigger.dev availability. + deploy-trigger: + name: Deploy Trigger.dev + needs: [migrate] + if: >- + !cancelled() && + needs.migrate.result == 'success' && + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + timeout-minutes: 15 + outputs: + version: ${{ steps.deploy.outputs.deploymentVersion }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -256,17 +448,29 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile --ignore-scripts - - name: Deploy to Trigger.dev + - name: Deploy to Trigger.dev (skip promotion) + id: deploy working-directory: ./apps/sim env: - TRIGGER_ACCESS_TOKEN: ${{ secrets.DEV_TRIGGER_ACCESS_TOKEN }} + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + TRIGGER_ENV: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }} run: | + set -eo pipefail if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then - echo "ERROR: DEV_TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + bunx trigger.dev@4.5.12 deploy --env "$TRIGGER_ENV" --skip-promotion + + - name: Validate deployment version output + env: + VERSION: ${{ steps.deploy.outputs.deploymentVersion }} + run: | + if ! [[ "$VERSION" =~ ^[0-9]{8}\.[0-9]+$ ]]; then + echo "ERROR: Trigger.dev did not report a valid deploymentVersion output" >&2 exit 1 fi - bunx trigger.dev@4.5.12 deploy --env preview --branch dev-sim # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR. # Runs in parallel with tests — only immutable sha tags are pushed here, and @@ -415,7 +619,22 @@ jobs: permissions: contents: read id-token: write + outputs: + # Whether the deploy tag was actually moved (false on a stale-run guard + # skip). promote-trigger keys off this so tasks are never promoted when + # the app itself wasn't. + promoted: ${{ steps.guard.outputs.fresh }} + # Epoch when the deploy tag was retagged (this push's ECS pipeline trigger). + # promote-trigger passes it to the poll script so a stale pipeline execution + # reusing the same image digest can't satisfy the cutover gate. + retag_epoch: ${{ steps.promote.outputs.retag_epoch }} + # Unchanged tags verify the latest execution's cutover without an epoch bound. + app_image_changed: ${{ steps.promote.outputs.app_image_changed }} + app_image_digest: ${{ steps.promote.outputs.app_image_digest }} steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: @@ -444,6 +663,7 @@ jobs: fi - name: Promote images to deploy tags + id: promote if: steps.guard.outputs.fresh == 'true' env: ECR_REPOS: >- @@ -460,6 +680,8 @@ jobs: ECR_TAG="staging" fi + APP_REPO="${{ secrets.ECR_APP }}" + # Verify every sha image exists before moving any deploy tag, so a # missing/expired image aborts the whole promotion up front. for repo in $ECR_REPOS; do @@ -469,11 +691,112 @@ jobs: for repo in $ECR_REPOS; do echo "🚀 Promoting ${repo}:${{ github.sha }} to ${ECR_TAG}" - docker buildx imagetools create \ - -t "${REGISTRY}/${repo}:${ECR_TAG}" \ - "${REGISTRY}/${repo}:${{ github.sha }}" + if [ "$repo" = "$APP_REPO" ]; then + bash .github/scripts/promote-app-image.sh "$REGISTRY" "$APP_REPO" "$GITHUB_SHA" "$ECR_TAG" + else + docker buildx imagetools create \ + -t "${REGISTRY}/${repo}:${ECR_TAG}" \ + "${REGISTRY}/${repo}:${{ github.sha }}" + fi done + # Main/staging: promote the parked Trigger.dev version after observing the ECS + # traffic cutover (CodeDeploy AllowTraffic on every target). The image retag + # triggers the ECS pipeline; this job correlates it via the digest + retag epoch + # (rejecting a stale execution reusing the digest) and promotes at cutover. + # Skipped when promote-images skipped the tag move (stale run) — tasks then + # correctly stay on the old version. If the app deploy fails or never cuts over, + # promote never fires and this job fails visibly. + promote-trigger: + name: Promote Trigger.dev + needs: [promote-images, deploy-trigger] + # Explicit results also suppress skip propagation from optional ancestors. + if: >- + !cancelled() && + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') && + needs.promote-images.result == 'success' && + needs.deploy-trigger.result == 'success' && + needs.promote-images.outputs.promoted == 'true' + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # Must exceed the poll script's OVERALL_TIMEOUT (70 min, covering a prod deploy + # queued behind a ~50-min bake) PLUS runner setup + the final promote step, so + # the Actions timeout never kills the job before the script's own deadline. + timeout-minutes: 90 + permissions: + contents: read + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.1 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile --ignore-scripts + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }} + aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }} + # The poll can run up to ~70 min (prod deploy queued behind a bake), which + # outlasts the default 1h session. Hold the session for the full job so AWS + # calls don't start failing mid-poll. Requires the deploy role's + # MaxSessionDuration to be >= this value (roles are managed outside the repo). + role-duration-seconds: 5400 + + # An unchanged tag may belong to a failed or still-running earlier deploy. + # Verify its latest cutover rather than treating tag equality as success. + - name: Wait for ECS traffic cutover + env: + APP_IMAGE_CHANGED: ${{ needs.promote-images.outputs.app_image_changed }} + DIGEST: ${{ needs.promote-images.outputs.app_image_digest }} + PIPELINE: sim-${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}-us-east-1-app-deployment + RETAG_EPOCH: ${{ needs.promote-images.outputs.retag_epoch }} + run: | + set -eo pipefail + case "$APP_IMAGE_CHANGED" in + true) ;; + false) RETAG_EPOCH=0 ;; + *) echo "ERROR: invalid app image change metadata" >&2; exit 1 ;; + esac + bash .github/scripts/wait-for-ecs-cutover.sh "$PIPELINE" "$DIGEST" "$RETAG_EPOCH" + + - name: Promote Trigger.dev version + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + TRIGGER_ENV: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }} + VERSION: ${{ needs.deploy-trigger.outputs.version }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + if [ -z "$VERSION" ]; then + echo "ERROR: no deployed version passed from deploy-trigger" >&2 + exit 1 + fi + echo "Promoting Trigger.dev version $VERSION ($TRIGGER_ENV)" + bunx trigger.dev@4.5.12 promote "$VERSION" --env "$TRIGGER_ENV" + # Build ARM64 images for GHCR (main branch only, runs in parallel with # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64 # are applied by create-ghcr-manifests after the gate, so a failing run diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index d7122f1e889..abf812907db 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -347,6 +347,9 @@ jobs: - name: Lint code run: bun run lint:check + - name: Test Trigger deployment gates + run: python3 .github/scripts/test-trigger-deploy.py + # Every zero-argument `check:*` script, run concurrently. The list is derived in # scripts/run-audits.ts, which also writes the per-audit timing table to the job # summary and annotates failures. Audits needing a base ref stay separate below.