Skip to content

Commit ec2f0bc

Browse files
TheodoreSpeaksTheodoreSpeaks
committed
fix(ci): bind Trigger promotion to the latest app tag move
1 parent da56970 commit ec2f0bc

5 files changed

Lines changed: 129 additions & 64 deletions

File tree

‎.github/actions/docker-build/action.yml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,6 @@ inputs:
3030
bypass an input `default:` entirely.
3131
required: false
3232

33-
outputs:
34-
digest:
35-
description: The image digest returned by the selected build provider.
36-
value: ${{ steps.build-blacksmith.outputs.digest || steps.build-github.outputs.digest }}
37-
3833
# Registry logins must precede this action. provenance/sbom stay off: attestation
3934
# manifests break `imagetools create` retagging in promote-images.
4035
runs:
@@ -69,7 +64,6 @@ runs:
6964
cache-key: ${{ steps.cache-key.outputs.value }}
7065

7166
- name: Build and push (Blacksmith)
72-
id: build-blacksmith
7367
if: inputs.provider == '' || inputs.provider == 'blacksmith'
7468
uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
7569
with:
@@ -175,7 +169,6 @@ runs:
175169

176170
# No cache-to: type=gha — it shares the 10 GB repo quota with the cache mounts.
177171
- name: Build and push (GitHub)
178-
id: build-github
179172
if: inputs.provider != '' && inputs.provider != 'blacksmith'
180173
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
181174
with:

‎.github/scripts/promote-app-image.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
# Capture the cutover lower bound at the app tag move, after the image is built.
3+
set -euo pipefail
4+
REGISTRY="${1:?registry required}"
5+
REPOSITORY="${2:?repository required}"
6+
SOURCE_TAG="${3:?source tag required}"
7+
DEPLOY_TAG="${4:?deploy tag required}"
8+
: "${GITHUB_OUTPUT:?GitHub output file required}"
9+
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
10+
PREVIOUS=$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG" --allow-missing)
11+
EPOCH=$(date +%s)
12+
docker buildx imagetools create -t "$REGISTRY/$REPOSITORY:$DEPLOY_TAG" "$REGISTRY/$REPOSITORY:$SOURCE_TAG"
13+
DIGEST=$(bash "$SCRIPT_DIR/get-ecr-image-digest.sh" "$REPOSITORY" "$DEPLOY_TAG")
14+
CHANGED=true
15+
if [ "$DIGEST" = "$PREVIOUS" ]; then CHANGED=false; fi
16+
{
17+
echo "retag_epoch=$EPOCH"
18+
echo "app_image_digest=$DIGEST"
19+
echo "app_image_changed=$CHANGED"
20+
} >> "$GITHUB_OUTPUT"

‎.github/scripts/test-trigger-deploy.py‎

Lines changed: 66 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,17 @@ def run_script(self, script, args, responses):
4848
if response.get('error'):
4949
sys.stderr.write(response['error'])
5050
sys.exit(254)
51+
if response.get('advance_clock'):
52+
clock = root / 'clock'
53+
value = int(clock.read_text()) if clock.exists() else 1000
54+
clock.write_text(str(value + response['advance_clock']))
5155
print(response.get('text', json.dumps(response.get('json'))))
56+
''')
57+
(root / 'docker').write_text('''#!/usr/bin/env python3
58+
import os, pathlib, sys
59+
root = pathlib.Path(os.environ['FIXTURE_DIR'])
60+
with (root / 'calls').open('a') as stream:
61+
stream.write('docker ' + ' '.join(sys.argv[1:]) + '\\n')
5262
''')
5363
(root / 'date').write_text('''#!/usr/bin/env python3
5464
import os, pathlib
@@ -58,15 +68,17 @@ def run_script(self, script, args, responses):
5868
print(value)
5969
''')
6070
(root / 'sleep').write_text('#!/bin/sh\nexit 0\n')
61-
for name in ('aws', 'date', 'sleep'):
71+
for name in ('aws', 'date', 'sleep', 'docker'):
6272
(root / name).chmod(0o755)
6373
result = subprocess.run(
6474
['bash', str(SCRIPTS / script), *args],
6575
env={**os.environ, 'PATH': f'{root}:{os.environ["PATH"]}',
66-
'FIXTURE_DIR': str(root), 'POLL_INTERVAL': '1', 'OVERALL_TIMEOUT': '12'},
76+
'FIXTURE_DIR': str(root), 'POLL_INTERVAL': '1', 'OVERALL_TIMEOUT': '12',
77+
'GITHUB_OUTPUT': str(root / 'outputs')},
6778
capture_output=True, text=True, timeout=10,
6879
)
6980
calls = (root / 'calls').read_text() if (root / 'calls').exists() else ''
81+
result.github_output = (root / 'outputs').read_text() if (root / 'outputs').exists() else ''
7082
return result, calls
7183

7284
def poll(self, updates=None, since='1000'):
@@ -100,6 +112,32 @@ def test_chooses_newest_matching_execution(self):
100112
self.assertEqual(result.returncode, 0, result.stderr)
101113
self.assertIn('--pipeline-execution-id execution-current', calls)
102114

115+
def test_changed_image_rejects_newer_different_execution(self):
116+
result, calls = self.poll({'list-pipeline-executions': {'json': [
117+
execution(), execution(1001, digest=OTHER_DIGEST, identifier='execution-newer')]}})
118+
self.assertNotEqual(result.returncode, 0)
119+
self.assertIn('deployment was superseded', result.stderr)
120+
self.assertNotIn('get-pipeline-execution ', calls)
121+
122+
def test_rechecks_latest_digest_after_cutover(self):
123+
result, calls = self.poll({'list-pipeline-executions': [
124+
{'json': [execution()]},
125+
{'json': [execution(), execution(1001, digest=OTHER_DIGEST, identifier='execution-newer')]},
126+
]})
127+
self.assertNotEqual(result.returncode, 0)
128+
self.assertIn('deployment was superseded', result.stderr)
129+
self.assertIn('get-deployment-target ', calls)
130+
self.assertNotIn('Traffic cutover complete', result.stdout)
131+
132+
def test_rechecks_execution_identity_for_same_digest_after_cutover(self):
133+
result, _ = self.poll({'list-pipeline-executions': [
134+
{'json': [execution()]},
135+
{'json': [execution(), execution(1001, identifier='execution-newer')]},
136+
]})
137+
self.assertNotEqual(result.returncode, 0)
138+
self.assertIn('newer pipeline execution appeared', result.stdout)
139+
self.assertNotIn('Traffic cutover complete', result.stdout)
140+
103141
def test_iso_timestamps(self):
104142
result, _ = self.poll({'list-pipeline-executions': {'json': [
105143
execution('1970-01-01T00:16:40+00:00')]}})
@@ -182,6 +220,32 @@ def test_ecr_response_failures_are_not_missing_images(self):
182220
result, _ = self.run_script('get-ecr-image-digest.sh', ['app', 'deploy', '--allow-missing'], {'batch-get-image': response})
183221
self.assertNotEqual(result.returncode, 0)
184222

223+
def test_tag_move_uses_push_boundary_and_final_manifest_digest(self):
224+
result, calls = self.run_script('promote-app-image.sh', ['registry', 'app', 'commit-dev', 'dev'], {
225+
'batch-get-image': [
226+
{'advance_clock': 30, 'json': {'images': [{'imageId': {'imageDigest': DIGEST}}], 'failures': []}},
227+
{'json': {'images': [{'imageId': {'imageDigest': OTHER_DIGEST}}], 'failures': []}},
228+
]})
229+
self.assertEqual(result.returncode, 0, result.stderr)
230+
self.assertIn('retag_epoch=1030', result.github_output)
231+
self.assertIn(f'app_image_digest={OTHER_DIGEST}', result.github_output)
232+
self.assertIn('app_image_changed=true', result.github_output)
233+
self.assertEqual([line.split()[0] for line in calls.splitlines()], ['ecr', 'docker', 'ecr'])
234+
self.assertIn('registry/app:commit-dev', calls)
235+
236+
def test_tag_move_aborts_before_docker_when_ecr_read_fails(self):
237+
result, calls = self.run_script('promote-app-image.sh', ['registry', 'app', 'commit', 'deploy'], {
238+
'batch-get-image': {'error': 'AccessDeniedException'}})
239+
self.assertNotEqual(result.returncode, 0)
240+
self.assertNotIn('docker', calls)
241+
self.assertEqual(result.github_output, '')
242+
243+
def test_same_digest_tag_move_reports_unchanged(self):
244+
result, _ = self.run_script('promote-app-image.sh', ['registry', 'app', 'commit', 'deploy'], {
245+
'batch-get-image': {'json': {'images': [{'imageId': {'imageDigest': DIGEST}}], 'failures': []}}})
246+
self.assertEqual(result.returncode, 0, result.stderr)
247+
self.assertIn('app_image_changed=false', result.github_output)
248+
185249

186250
if __name__ == '__main__':
187251
unittest.main()

‎.github/scripts/wait-for-ecs-cutover.sh‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,12 @@ aws_read() {
3030
aws --cli-connect-timeout 10 --cli-read-timeout 30 "$@"
3131
}
3232

33-
EXECUTION_ID=''
34-
while [ -z "$EXECUTION_ID" ]; do
35-
check_deadline 'the matching pipeline execution'
33+
find_execution() {
34+
local executions
3635
executions=$(aws_read codepipeline list-pipeline-executions \
3736
--pipeline-name "$PIPELINE" --max-items 30 \
3837
--query 'pipelineExecutionSummaries' --output json)
39-
EXECUTION_ID=$(printf '%s\n' "$executions" | SINCE="$SINCE_EPOCH" DIGEST="$DIGEST" python3 -c '
38+
printf '%s\n' "$executions" | SINCE="$SINCE_EPOCH" DIGEST="$DIGEST" python3 -c '
4039
import datetime, json, os, sys
4140
since = int(os.environ["SINCE"])
4241
def epoch(execution):
@@ -52,9 +51,17 @@ if since == 0:
5251
raise SystemExit("ERROR: unchanged app tag does not match the latest pipeline execution; cutover is unverified")
5352
selected = executions[0]
5453
else:
55-
selected = next((e for e in executions if epoch(e) >= since and matches(e)), None)
54+
selected = executions[0] if executions and epoch(executions[0]) >= since else None
55+
if selected and not matches(selected):
56+
raise SystemExit("ERROR: latest pipeline execution does not match this app digest; deployment was superseded or its source is unverified")
5657
print(selected["pipelineExecutionId"] if selected else "")
57-
')
58+
'
59+
}
60+
61+
EXECUTION_ID=''
62+
while [ -z "$EXECUTION_ID" ]; do
63+
check_deadline 'the matching pipeline execution'
64+
EXECUTION_ID=$(find_execution)
5865
if [ -z "$EXECUTION_ID" ]; then
5966
log 'No matching execution since this push; waiting'
6067
sleep "$POLL_INTERVAL"
@@ -112,6 +119,11 @@ while true; do
112119
esac
113120
done
114121
if [ "$all_ok" = 1 ]; then
122+
LATEST_EXECUTION_ID=$(find_execution)
123+
if [ "$LATEST_EXECUTION_ID" != "$EXECUTION_ID" ]; then
124+
log 'ERROR: a newer pipeline execution appeared during cutover; not promoting'
125+
exit 1
126+
fi
115127
log 'Traffic cutover complete on every ECS target'
116128
exit 0
117129
fi

‎.github/workflows/ci.yml‎

Lines changed: 25 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -215,48 +215,34 @@ jobs:
215215
env:
216216
ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }}
217217

218-
# Capture the previous tag and timestamp before the build pushes :dev.
219-
# Only a missing tag is allowed; failed reads abort before deployment.
220-
- name: Capture pre-build :dev state
221-
id: prevdigest
222-
if: matrix.ecr_repo_secret == 'ECR_APP'
223-
run: |
224-
echo "epoch=$(date +%s)" >> "$GITHUB_OUTPUT"
225-
PREV=$(bash .github/scripts/get-ecr-image-digest.sh "${{ steps.ecr-repo.outputs.name }}" dev --allow-missing)
226-
echo "digest=${PREV}" >> "$GITHUB_OUTPUT"
227-
228218
- name: Build and push
229-
id: build
230219
uses: ./.github/actions/docker-build
231220
with:
232221
provider: ${{ vars.CI_PROVIDER }}
233222
file: ${{ matrix.dockerfile }}
234223
platforms: linux/amd64
235-
tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev
224+
tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:${{ matrix.ecr_repo_secret == 'ECR_APP' && format('{0}-dev', github.sha) || 'dev' }}
236225
max-cache-size-mb: ${{ matrix.cache_mb }}
237226

238-
# App leg only: publish the metadata promote-trigger-dev needs to correlate
239-
# this push to its dev ECS deploy and decide whether to wait. Dev has no
240-
# promote-images job, so this stands in for its retag_epoch/app_image_changed
241-
# outputs. The epoch and prev digest come from the pre-build step above.
227+
- name: Promote dev app image
228+
id: appdeploy
229+
if: matrix.ecr_repo_secret == 'ECR_APP'
230+
env:
231+
REGISTRY: ${{ steps.login-ecr.outputs.registry }}
232+
REPOSITORY: ${{ steps.ecr-repo.outputs.name }}
233+
run: bash .github/scripts/promote-app-image.sh "$REGISTRY" "$REPOSITORY" "${GITHUB_SHA}-dev" dev
234+
242235
- name: Publish dev cutover metadata
243236
if: matrix.ecr_repo_secret == 'ECR_APP'
237+
env:
238+
DIGEST: ${{ steps.appdeploy.outputs.app_image_digest }}
239+
EPOCH: ${{ steps.appdeploy.outputs.retag_epoch }}
240+
CHANGED: ${{ steps.appdeploy.outputs.app_image_changed }}
244241
run: |
245242
mkdir -p dev-meta
246-
NEW="${{ steps.build.outputs.digest }}"
247-
PREV="${{ steps.prevdigest.outputs.digest }}"
248-
if [ -z "$NEW" ]; then
249-
echo "ERROR: build did not report an image digest" >&2
250-
exit 1
251-
fi
252-
echo "$NEW" > dev-meta/digest.txt
253-
echo "${{ steps.prevdigest.outputs.epoch }}" > dev-meta/retag_epoch.txt
254-
if [ "$NEW" = "$PREV" ]; then
255-
echo "false" > dev-meta/app_image_changed.txt
256-
echo "ℹ️ :dev already points at ${NEW}; no ECS dev deploy will be triggered."
257-
else
258-
echo "true" > dev-meta/app_image_changed.txt
259-
fi
243+
echo "$DIGEST" > dev-meta/digest.txt
244+
echo "$EPOCH" > dev-meta/retag_epoch.txt
245+
echo "$CHANGED" > dev-meta/app_image_changed.txt
260246
261247
- name: Upload dev cutover metadata
262248
if: matrix.ecr_repo_secret == 'ECR_APP'
@@ -325,8 +311,8 @@ jobs:
325311
fi
326312
327313
# Dev: promote the skip-promoted preview version at the dev ECS traffic cutover.
328-
# Dev has no promote-images gate (build-dev pushes :dev directly), so the digest,
329-
# trigger epoch, and app-image-changed signal come from build-dev's artifact.
314+
# The dev app build moves :dev only after building its commit-tagged image,
315+
# then passes the tag digest and retag timestamp through an artifact.
330316
# trigger.dev supports promoting a specific preview branch: promote --env preview
331317
# --branch dev-sim.
332318
promote-trigger-dev:
@@ -686,11 +672,6 @@ jobs:
686672
${{ secrets.ECR_REALTIME }}
687673
${{ secrets.ECR_PII }}
688674
run: |
689-
# Record the retag time BEFORE moving any tag — this is when the ECS
690-
# pipeline for this push is triggered. promote-trigger uses it to
691-
# reject an older pipeline execution reusing the same image digest.
692-
echo "retag_epoch=$(date +%s)" >> "$GITHUB_OUTPUT"
693-
694675
REGISTRY="${{ steps.login-ecr.outputs.registry }}"
695676
696677
if [ "${{ github.ref }}" = "refs/heads/main" ]; then
@@ -700,7 +681,6 @@ jobs:
700681
fi
701682
702683
APP_REPO="${{ secrets.ECR_APP }}"
703-
PREV_APP_DIGEST=$(bash .github/scripts/get-ecr-image-digest.sh "$APP_REPO" "$ECR_TAG" --allow-missing)
704684
705685
# Verify every sha image exists before moving any deploy tag, so a
706686
# missing/expired image aborts the whole promotion up front.
@@ -711,19 +691,15 @@ jobs:
711691
712692
for repo in $ECR_REPOS; do
713693
echo "🚀 Promoting ${repo}:${{ github.sha }} to ${ECR_TAG}"
714-
docker buildx imagetools create \
715-
-t "${REGISTRY}/${repo}:${ECR_TAG}" \
716-
"${REGISTRY}/${repo}:${{ github.sha }}"
694+
if [ "$repo" = "$APP_REPO" ]; then
695+
bash .github/scripts/promote-app-image.sh "$REGISTRY" "$APP_REPO" "$GITHUB_SHA" "$ECR_TAG"
696+
else
697+
docker buildx imagetools create \
698+
-t "${REGISTRY}/${repo}:${ECR_TAG}" \
699+
"${REGISTRY}/${repo}:${{ github.sha }}"
700+
fi
717701
done
718702
719-
APP_DIGEST=$(bash .github/scripts/get-ecr-image-digest.sh "$APP_REPO" "$ECR_TAG")
720-
echo "app_image_digest=$APP_DIGEST" >> "$GITHUB_OUTPUT"
721-
if [ "$APP_DIGEST" = "$PREV_APP_DIGEST" ]; then
722-
echo "app_image_changed=false" >> "$GITHUB_OUTPUT"
723-
else
724-
echo "app_image_changed=true" >> "$GITHUB_OUTPUT"
725-
fi
726-
727703
# Main/staging: promote the parked Trigger.dev version after observing the ECS
728704
# traffic cutover (CodeDeploy AllowTraffic on every target). The image retag
729705
# triggers the ECS pipeline; this job correlates it via the digest + retag epoch

0 commit comments

Comments
 (0)