From 9a593852b1f75640ac9372b81e3416775a22c7a7 Mon Sep 17 00:00:00 2001 From: jmgasper Date: Mon, 28 Sep 2026 13:38:38 +1000 Subject: [PATCH 1/3] Inject service and global SSM appvars into Forms ECS tasks --- .circleci/config.yml | 2 +- deploy/README.md | 50 +++++++++++- deploy/appvars.py | 174 +++++++++++++++++++++++++++++++++++++++++ deploy/release.py | 17 +++- deploy/service.yaml | 2 +- deploy/test_appvars.py | 100 +++++++++++++++++++++++ docs/operations.md | 5 ++ 7 files changed, 342 insertions(+), 8 deletions(-) create mode 100644 deploy/appvars.py create mode 100644 deploy/test_appvars.py diff --git a/.circleci/config.yml b/.circleci/config.yml index c018910..1f5ec63 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ commands: command: | sudo apt-get update -qq sudo apt-get install -y -qq jq python3-pip - pip install boto3 awscli + pip install boto3 awscli PyYAML jobs: deploy: diff --git a/deploy/README.md b/deploy/README.md index bb5c26c..483e7e7 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -42,7 +42,8 @@ verification job. `develop` builds and deploys dev; `master` builds and deploys production. Both use Topcoder's `org-global` context and pinned `tc-deploy-scripts` credential helper. Forms retains its runtime and migration images and `release.py` deployment process: target-database migrations still run before -runtime promotion. Deployments are serialized separately per environment. The +runtime promotion. Each release also refreshes the runtime SSM appvar references +from the service and global paths, as described below. Deployments are serialized separately per environment. The CircleCI project must be connected in the Topcoder organization to use that shared context. @@ -71,7 +72,8 @@ digests and AWS task/stack identifiers. On an observation timeout, inspect the reported task/stack operation before retrying; do not start a second migration while the first remains active. Infrastructure edits are applied separately using CloudFormation with the current ImageTag and environment parameters preserved; -normal app releases retain the existing stack template and parameters. +normal app releases preserve the existing stack infrastructure and parameters while +refreshing the runtime appvar references and their SSM execution permissions. ## Dev sample @@ -80,3 +82,47 @@ normal app releases retain the existing stack template and parameters. It refuses production and conflicting published definitions. The corresponding CMS seed lives in `payload-cms/scripts/seed-forms-test.ts`. The website resolves the CMS page at `/forms-test` and fetches its schema from the public Forms API at runtime. + +## Runtime appvar injection + +Forms uses the same ECS secret-reference mapping as the other v6 services' +`master_deploy.sh -j /config/${APPNAME}/appvar,/config/common/global-appvar`: + +- Enumerate all direct parameters from `/config/forms-api-v6/appvar` first, then + `/config/common/global-appvar`, including every paginated result. The mapping follows the pinned + [v1.4.20 deployment suite](https://github.com/topcoder-platform/tc-deploy-scripts/blob/v1.4.20/master_deploy.sh) + used by `bus-api-v6`. +- Inject each leaf name as the environment variable, using its SSM ARN in the + task definition's `Secrets` / `valueFrom`. No decrypted values enter deployment + artifacts, logs, Docker images, or plaintext ECS environment entries. +- Existing literal task environment settings take precedence; service-specific + appvars take precedence over identically named globals. There is no appvar allowlist. +- ECS resolves the values at task startup. The execution role can read both exact + prefixes. Custom KMS keys still require the existing `SecretsKmsKeyArn` grant. + +`deploy/appvars.py` implements this mapping for the existing CloudFormation-owned +Forms service. `release.py` refreshes it on every successful release instead of +reusing a fixed list of secrets. Migration tasks still receive only their explicit +`DATABASE_URL` secret; runtime appvars do not change migration credentials. + +For a configuration-only rollout with the existing image, load the target AWS +credentials and use Python with `boto3` and `PyYAML` installed: + +```sh +python3 deploy/appvars.py dev # read-only plan and CloudFormation validation +python3 deploy/appvars.py dev --apply # update bindings/IAM and roll the same image +``` + +All existing stack parameters, including ImageTag and DesiredCount, are retained. +Only the runtime secrets list and its appvar IAM policy change. Parameter additions +and removals are picked up when generating the template. Changing an existing SSM +value alone requires a new release or an ECS force-new-deployment to refresh tasks; +an unchanged configuration plan does not force a restart. + +For Kafka delivery, create `BUSAPI_URL=https://api.topcoder-dev.com/v6` under the +Forms dev appvar path. The Forms `AUTH0_CLIENT_ID` / `AUTH0_CLIENT_SECRET` combine +with shared `AUTH0_URL`, `AUTH0_AUDIENCE`, and optional `AUTH0_PROXY_SERVER_URL` / +`TOKEN_CACHE_TIME`. Production needs its corresponding URL and authorized credentials. +Do not decrypt or copy shared values into the service path to perform injection. + +Deployment mapping regression tests: `python3 -m unittest discover -s deploy -p 'test_*.py'`. diff --git a/deploy/appvars.py b/deploy/appvars.py new file mode 100644 index 0000000..7a9b6e9 --- /dev/null +++ b/deploy/appvars.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Inject Forms and global SSM appvars as ECS secret references, matching tc-deploy-scripts -j. + +CLI: appvars.py dev|production [--apply]. Defaults to a read-only configuration plan. +Uses inherited AWS credentials. Never decrypts parameters or writes their values. +The apply mode updates only the running stack's appvar bindings and execution-role +SSM permissions, preserving its image, parameters, networking, and migration setup. +""" +import argparse +import copy +import json +import re +import time + +import boto3 +import yaml + +GLOBAL_APPVARS = '/config/common/global-appvar' + + +class CloudFormationLoader(yaml.SafeLoader): + """Read CloudFormation YAML intrinsics as JSON objects without evaluating tags. + + Used when AWS returns an existing YAML template rather than a JSON mapping. + Only standard intrinsic tags are accepted; unknown tags raise ValueError. + """ + + +def intrinsic(loader, tag, node): + """Convert one CloudFormation YAML tag to its JSON representation. + + Takes the safe loader, tag suffix, and YAML node; returns an intrinsic mapping. + Raises ValueError for unsupported tags and propagates malformed YAML errors. + """ + if tag not in {'Ref', 'Condition', 'Base64', 'GetAtt', 'GetAZs', 'ImportValue', + 'Join', 'Select', 'Split', 'Sub', 'FindInMap', 'If', 'Equals', + 'And', 'Or', 'Not', 'Cidr', 'Transform', 'Length', 'ToJsonString'}: + raise ValueError('Unsupported CloudFormation intrinsic: ' + tag) + if isinstance(node, yaml.ScalarNode): + value = loader.construct_scalar(node) + elif isinstance(node, yaml.SequenceNode): + value = loader.construct_sequence(node) + else: + value = loader.construct_mapping(node) + if tag == 'GetAtt' and isinstance(value, str): + value = value.split('.', 1) + return {tag if tag in {'Ref', 'Condition'} else 'Fn::' + tag: value} + + +CloudFormationLoader.add_multi_constructor('!', intrinsic) + + +def load_template(body): + """Return an independent JSON-compatible stack template from an AWS template body. + + Accepts the dict or YAML/JSON text returned by get_template. Raises ValueError + for missing resources, and propagates safe YAML parsing errors. + """ + template = copy.deepcopy(body) if isinstance(body, dict) else yaml.load(body, Loader=CloudFormationLoader) + if not isinstance(template, dict) or not isinstance(template.get('Resources'), dict): + raise ValueError('CloudFormation template has no resources.') + return template + + +def appvar_secrets(ssm, prefix, partition, region, account, environment_names=()): + """Discover all direct SSM appvars with the same precedence as master_deploy.sh -j. + + Takes the SSM client, service path, AWS ARN components, and existing literal + environment names. Returns ECS name/valueFrom entries: literal environment + first, service parameters next, global parameters last. Pagination is consumed + without decryption. Raises ValueError for unsafe variable names; AWS errors + propagate. Values are never placed in the generated task definition. + """ + seen = set(environment_names) + secrets = [] + for path in [prefix.rstrip('/'), GLOBAL_APPVARS]: + for page in ssm.get_paginator('get_parameters_by_path').paginate( + Path=path, Recursive=False, WithDecryption=False): + for parameter in page['Parameters']: + name = parameter['Name'].removeprefix(path + '/') + if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', name): + raise ValueError('Appvar name is not a direct environment variable: ' + parameter['Name']) + if name not in seen: + seen.add(name) + secrets.append({'name': name, 'valueFrom': + f'arn:{partition}:ssm:{region}:{account}:parameter{parameter["Name"]}'}) + return sorted(secrets, key=lambda item: item['name']) + + +def configure_template(body, ssm, prefix, partition, region, account): + """Refresh the Forms runtime secrets and execution-role grants in a stack template. + + Takes the current template, SSM client, and deployment identifiers. Returns a + new template, preserving unrelated resources, runtime image, and literal env. + Rebuilds the secret list so removed parameters are no longer injected. Raises + KeyError for an unexpected Forms stack layout, ValueError for invalid appvars, + and propagates SSM read failures. Used by releases and configuration-only rolls. + """ + template = load_template(body) + resources = template['Resources'] + containers = resources['TaskDefinition']['Properties']['ContainerDefinitions'] + container = next(item for item in containers if item['Name'] == 'forms-api-v6') + entries = appvar_secrets(ssm, prefix, partition, region, account, + [item['Name'] for item in container.get('Environment', [])]) + container['Secrets'] = [{'Name': item['name'], 'ValueFrom': item['valueFrom']} for item in entries] + policies = resources['ExecutionRole']['Properties']['Policies'] + policy_name = 'forms-runtime-appvars' + policies[:] = [policy for policy in policies if policy['PolicyName'] != policy_name] + policies.append({'PolicyName': policy_name, 'PolicyDocument': { + 'Version': '2012-10-17', 'Statement': [{ + 'Effect': 'Allow', 'Action': ['ssm:GetParameters'], + 'Resource': [f'arn:{partition}:ssm:{region}:{account}:parameter{path.rstrip("/")}/*' + for path in [prefix, GLOBAL_APPVARS]], + }], + }}) + return template + + +def main(): + """Plan or apply appvar injection to the selected existing Forms stack. + + Reads CLI arguments and inherited AWS credentials; returns None after showing + names only or completing a configuration roll. Raises on account mismatch, + unstable stack, invalid template, AWS errors, deployment rollback, or timeout. + Does not submit forms, replay events, run migrations, or change appvar values. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('environment', choices=['dev', 'production']) + parser.add_argument('--apply', action='store_true') + args = parser.parse_args() + session = boto3.Session(region_name='us-east-1') + identity = session.client('sts').get_caller_identity() + account = identity['Account'] + if (account == '811668436784') != (args.environment == 'dev'): + raise RuntimeError('AWS account does not match the selected environment.') + cfn = session.client('cloudformation') + stack_name = 'forms-api-v6-' + args.environment + stack = cfn.describe_stacks(StackName=stack_name)['Stacks'][0] + if stack['StackStatus'] not in {'CREATE_COMPLETE', 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_COMPLETE'}: + raise RuntimeError('Stack is not stable; inspect the existing operation.') + settings = {item['ParameterKey']: item['ParameterValue'] for item in stack['Parameters']} + if settings['Environment'] != args.environment or settings['BootstrapOnly'] != 'false': + raise RuntimeError('Stack environment or bootstrap state does not permit a configuration roll.') + original = cfn.get_template(StackName=stack_name, TemplateStage='Original')['TemplateBody'] + template = configure_template(original, session.client('ssm'), settings['ParameterPrefix'], + identity['Arn'].split(':')[1], session.region_name, account) + container = template['Resources']['TaskDefinition']['Properties']['ContainerDefinitions'][0] + print('Runtime SSM appvars: ' + ', '.join(item['Name'] for item in container['Secrets'])) + body = json.dumps(template) + cfn.validate_template(TemplateBody=body) + if not args.apply: + print('Template validated. Re-run with --apply to roll the existing service image with these bindings.') + return + if template == load_template(original): + print('Appvar bindings and execution permissions already match; no stack change required.') + return + cfn.update_stack(StackName=stack_name, TemplateBody=body, + Parameters=[{'ParameterKey': item['ParameterKey'], 'UsePreviousValue': True} + for item in stack['Parameters']], Capabilities=['CAPABILITY_IAM']) + deadline = time.monotonic() + 1800 + while time.monotonic() < deadline: + state = cfn.describe_stacks(StackName=stack_name)['Stacks'][0]['StackStatus'] + if state == 'UPDATE_COMPLETE': + print('Appvar configuration deployed: ' + stack_name) + return + if state not in {'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS'}: + raise RuntimeError('Configuration roll failed or rolled back: ' + state) + print('Waiting for appvar configuration roll...', flush=True) + time.sleep(15) + raise RuntimeError('Timed out observing the stack; inspect its existing operation before retrying.') + + +if __name__ == '__main__': + main() diff --git a/deploy/release.py b/deploy/release.py index 0eb90aa..9c55054 100644 --- a/deploy/release.py +++ b/deploy/release.py @@ -3,7 +3,8 @@ CLI: release.py dev|production IMAGE_TAG [RUNTIME_IMAGE] [MIGRATION_IMAGE]. Requires inherited AWS credentials, boto3, and Docker. Pushes immutable ECR tags, -executes migrations as a private one-shot ECS task, then updates CloudFormation. +discovers service/global SSM appvars, executes migrations as a private one-shot ECS +task, then updates CloudFormation with fresh ECS secret references. Raises on failures; runtime promotion never occurs after failed migrations. """ import base64 @@ -18,6 +19,8 @@ import boto3 +from appvars import configure_template + def wait_until(description, check, seconds=1800): """Poll a live deployment check every 15 seconds, returning its truthy result. @@ -36,7 +39,7 @@ def wait_until(description, check, seconds=1800): def main(): - """Validate release arguments and environment, migrate, promote, and save evidence. + """Validate release arguments, discover appvars, migrate, promote, and save evidence. Returns None on success; AWS, Docker, failed migration, and failed deployment errors propagate. Credentials are passed through stdin/environment, never argv. @@ -48,7 +51,8 @@ def main(): raise RuntimeError('Invalid immutable release tag.') local_runtime, local_migration = sys.argv[3:] or ['forms-api-v6:candidate', 'forms-api-v6:migrate-candidate'] session = boto3.Session(region_name=os.environ.get('AWS_REGION', 'us-east-1')) - account = session.client('sts').get_caller_identity()['Account'] + identity = session.client('sts').get_caller_identity() + account = identity['Account'] if (account == '811668436784') != (environment == 'dev'): raise RuntimeError('AWS account does not match the selected deployment environment.') cfn, ecs, ecr = [session.client(name) for name in ['cloudformation', 'ecs', 'ecr']] @@ -59,6 +63,11 @@ def main(): settings = {x['ParameterKey']: x['ParameterValue'] for x in stack['Parameters']} if settings['Environment'] != environment or settings['BootstrapOnly'] != 'false': raise RuntimeError('Stack environment or bootstrap state does not permit deployment.') + template = configure_template( + cfn.get_template(StackName=stack_name, TemplateStage='Original')['TemplateBody'], + session.client('ssm'), settings['ParameterPrefix'], identity['Arn'].split(':')[1], + session.region_name, account) + cfn.validate_template(TemplateBody=json.dumps(template)) output = {x['OutputKey']: x['OutputValue'] for x in stack['Outputs']} repository = output['RepositoryUri'] authorization = ecr.get_authorization_token()['authorizationData'][0] @@ -107,7 +116,7 @@ def migration_finished(): parameters = [({'ParameterKey': x['ParameterKey'], 'ParameterValue': tag} if x['ParameterKey'] == 'ImageTag' else {'ParameterKey': x['ParameterKey'], 'ParameterValue': str(max(1, int(settings['DesiredCount'])))} if x['ParameterKey'] == 'DesiredCount' else {'ParameterKey': x['ParameterKey'], 'UsePreviousValue': True}) for x in stack['Parameters']] - result = cfn.update_stack(StackName=stack_name, UsePreviousTemplate=True, + result = cfn.update_stack(StackName=stack_name, TemplateBody=json.dumps(template), Parameters=parameters, Capabilities=['CAPABILITY_IAM']) print('Updating stack: ' + result['StackId'], flush=True) diff --git a/deploy/service.yaml b/deploy/service.yaml index d13513e..b71a1c1 100644 --- a/deploy/service.yaml +++ b/deploy/service.yaml @@ -150,7 +150,7 @@ Resources: Action: ssm:GetParameters Resource: - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter${ParameterPrefix}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/config/common/global-appvar/AUTH_SECRET + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/config/common/global-appvar/* - !If - HasSecretsKmsKey - Effect: Allow diff --git a/deploy/test_appvars.py b/deploy/test_appvars.py new file mode 100644 index 0000000..c842713 --- /dev/null +++ b/deploy/test_appvars.py @@ -0,0 +1,100 @@ +"""Regression tests for the v6 SSM-to-ECS appvar mapping and template preservation.""" +import copy +from pathlib import Path +import unittest +from unittest.mock import Mock + +from appvars import appvar_secrets, configure_template, load_template + +PREFIX = '/config/forms-api-v6/appvar' +GLOBAL = '/config/common/global-appvar' + + +def ssm_fixture(pages): + """Return an SSM mock serving path-indexed pages; unknown paths raise KeyError.""" + client = Mock() + client.get_paginator.return_value.paginate.side_effect = lambda **kwargs: iter(pages[kwargs['Path']]) + return client + + +def parameter(path, name): + """Return one SSM fixture with a sentinel value that must never reach a template.""" + return {'Name': path + '/' + name, 'Value': 'DO_NOT_EMBED_PARAMETER_VALUES'} + + +class AppvarTests(unittest.TestCase): + """Exercise pagination, v6 precedence, safe references, and CloudFormation changes.""" + + def test_service_precedence_and_all_pages(self): + """Map every page, preferring literal env then service values over globals.""" + client = ssm_fixture({ + PREFIX: [{'Parameters': [parameter(PREFIX, 'AUTH0_CLIENT_ID'), parameter(PREFIX, 'PORT')]}, + {'Parameters': [parameter(PREFIX, 'AUTH0_CLIENT_SECRET')]}], + GLOBAL: [{'Parameters': [parameter(GLOBAL, 'AUTH0_CLIENT_ID'), parameter(GLOBAL, 'AUTH0_URL')]}, + {'Parameters': [parameter(GLOBAL, 'AUTH_SECRET')]}], + }) + result = appvar_secrets(client, PREFIX, 'aws', 'us-east-1', '123', ['PORT']) + refs = {entry['name']: entry['valueFrom'] for entry in result} + self.assertEqual(set(refs), {'AUTH0_CLIENT_ID', 'AUTH0_CLIENT_SECRET', 'AUTH0_URL', 'AUTH_SECRET'}) + self.assertEqual(refs['AUTH0_CLIENT_ID'], 'arn:aws:ssm:us-east-1:123:parameter' + PREFIX + '/AUTH0_CLIENT_ID') + self.assertEqual(refs['AUTH0_URL'], 'arn:aws:ssm:us-east-1:123:parameter' + GLOBAL + '/AUTH0_URL') + self.assertNotIn('DO_NOT_EMBED_PARAMETER_VALUES', str(result)) + for call in client.get_paginator.return_value.paginate.call_args_list: + self.assertFalse(call.kwargs['Recursive']) + self.assertFalse(call.kwargs['WithDecryption']) + + def test_template_preserves_runtime_and_other_resources(self): + """Refresh only runtime secret bindings and a narrowly scoped IAM policy.""" + original = load_template(Path(__file__).with_name('service.yaml').read_text()) + snapshot = copy.deepcopy(original) + client = ssm_fixture({PREFIX: [{'Parameters': [parameter(PREFIX, 'DATABASE_URL'), parameter(PREFIX, 'BUSAPI_URL')]}], + GLOBAL: [{'Parameters': [parameter(GLOBAL, 'AUTH_SECRET')]}]}) + result = configure_template(original, client, PREFIX, 'aws', 'us-east-1', '123') + self.assertEqual(original, snapshot) + self.assertEqual(result['Parameters'], original['Parameters']) + for name in original['Resources']: + if name not in {'TaskDefinition', 'ExecutionRole'}: + self.assertEqual(result['Resources'][name], original['Resources'][name]) + container = result['Resources']['TaskDefinition']['Properties']['ContainerDefinitions'][0] + before = original['Resources']['TaskDefinition']['Properties']['ContainerDefinitions'][0] + self.assertEqual({k: v for k, v in container.items() if k != 'Secrets'}, + {k: v for k, v in before.items() if k != 'Secrets'}) + self.assertEqual({item['Name'] for item in container['Secrets']}, {'DATABASE_URL', 'BUSAPI_URL', 'AUTH_SECRET'}) + policy = result['Resources']['ExecutionRole']['Properties']['Policies'][-1] + self.assertEqual(policy['PolicyDocument']['Statement'][0]['Action'], ['ssm:GetParameters']) + self.assertEqual(policy['PolicyDocument']['Statement'][0]['Resource'], [ + 'arn:aws:ssm:us-east-1:123:parameter' + PREFIX + '/*', + 'arn:aws:ssm:us-east-1:123:parameter' + GLOBAL + '/*']) + self.assertNotIn('DO_NOT_EMBED_PARAMETER_VALUES', str(result)) + + def test_refresh_is_idempotent_and_removes_deleted_parameters(self): + """Remove obsolete references without accumulating permissions on later releases.""" + original = Path(__file__).with_name('service.yaml').read_text() + client = ssm_fixture({PREFIX: [{'Parameters': [parameter(PREFIX, 'DATABASE_URL')]}], GLOBAL: [{'Parameters': []}]}) + once = configure_template(original, client, PREFIX, 'aws', 'us-east-1', '123') + twice = configure_template(once, client, PREFIX, 'aws', 'us-east-1', '123') + self.assertEqual(once, twice) + names = [entry['Name'] for entry in twice['Resources']['TaskDefinition']['Properties']['ContainerDefinitions'][0]['Secrets']] + self.assertEqual(names, ['DATABASE_URL']) + + def test_read_errors_and_unsafe_names_stop_generation(self): + """Fail before deployment when appvar discovery fails or returns unsafe names.""" + client = Mock() + client.get_paginator.side_effect = RuntimeError('SSM unavailable') + with self.assertRaises(RuntimeError): + appvar_secrets(client, PREFIX, 'aws', 'us-east-1', '123') + client = ssm_fixture({PREFIX: [{'Parameters': [parameter(PREFIX, 'nested/SECRET')]}]}) + with self.assertRaises(ValueError): + appvar_secrets(client, PREFIX, 'aws', 'us-east-1', '123') + + def test_cloudformation_intrinsics_are_not_evaluated(self): + """Preserve scalar/sequence/mapping intrinsics and reject unsafe YAML objects.""" + result = load_template('Resources:\n Arn: !GetAtt Role.Arn\n Choice: !If [IsDev, dev, prod]\n Text: !Sub "${Name}"\n') + self.assertEqual(result['Resources']['Arn'], {'Fn::GetAtt': ['Role', 'Arn']}) + self.assertEqual(result['Resources']['Choice'], {'Fn::If': ['IsDev', 'dev', 'prod']}) + with self.assertRaises(ValueError): + load_template('Resources: !Run arbitrary') + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/operations.md b/docs/operations.md index 464a40e..7381311 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -23,6 +23,11 @@ The Prisma client uses the PostgreSQL driver adapter and generated TypeScript co ## Outbound Bus API +ECS releases inject all service appvars from `/config/forms-api-v6/appvar` and +shared appvars from `/config/common/global-appvar` as SSM secret references, with +service values taking precedence. See [deployment injection](../deploy/README.md#runtime-appvar-injection) +for configuration-only rolls and required execution-role permissions. + Ordinary submissions need no Bus API configuration. To accept `kafka=true` submissions successfully, configure: | Variable | Meaning | From 75cbfc110ad504651f809e2fc2a87c00a2f443ef Mon Sep 17 00:00:00 2001 From: jmgasper Date: Mon, 28 Sep 2026 13:40:34 +1000 Subject: [PATCH 2/3] Run appvar deployment regression checks in CI --- .circleci/config.yml | 3 +++ .github/workflows/ci.yml | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f5ec63..a0ff254 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,6 +35,9 @@ jobs: - checkout - setup_remote_docker - install-dependencies + - run: + name: Verify ECS appvar injection + command: python3 -m unittest discover -s deploy -p 'test_*.py' - run: name: Build runtime and one-shot migration images command: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c44bbc5..7b29311 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,11 @@ jobs: TEST_DATABASE_URL: postgresql://forms:forms_ci@localhost:5432/forms_test?schema=forms steps: - uses: actions/checkout@v4 + - name: Verify ECS appvar injection + run: | + python3 -m venv /tmp/forms-deploy-checks + /tmp/forms-deploy-checks/bin/pip install boto3 PyYAML + /tmp/forms-deploy-checks/bin/python -m unittest discover -s deploy -p 'test_*.py' - name: Select project Node and install pnpm shell: bash run: | From 16a138837a52ec913a780c9b843c4f1b7f582607 Mon Sep 17 00:00:00 2001 From: jmgasper Date: Mon, 28 Sep 2026 15:39:45 +1000 Subject: [PATCH 3/3] Use the shared ECS deployment suite for Forms appvars --- .circleci/config.yml | 21 +++-- .github/workflows/ci.yml | 5 -- deploy/README.md | 133 +++++++++++++++--------------- deploy/appvars.py | 174 --------------------------------------- deploy/release.py | 74 +++++------------ deploy/service.yaml | 2 +- deploy/test_appvars.py | 100 ---------------------- 7 files changed, 100 insertions(+), 409 deletions(-) delete mode 100644 deploy/appvars.py delete mode 100644 deploy/test_appvars.py diff --git a/.circleci/config.yml b/.circleci/config.yml index a0ff254..0456752 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ commands: command: | sudo apt-get update -qq sudo apt-get install -y -qq jq python3-pip - pip install boto3 awscli PyYAML + pip install boto3 awscli jobs: deploy: @@ -35,26 +35,31 @@ jobs: - checkout - setup_remote_docker - install-dependencies - - run: - name: Verify ECS appvar injection - command: python3 -m unittest discover -s deploy -p 'test_*.py' - run: name: Build runtime and one-shot migration images command: | docker buildx build --load --target migrate -t forms-api-v6:migrate-candidate . - docker buildx build --load --target runtime -t forms-api-v6:candidate . + docker buildx build --load --target runtime -t forms-api-v6:latest . - run: - name: Authenticate, migrate, and deploy through CloudFormation + name: Migrate and deploy using the Topcoder deployment suite no_output_timeout: 35m command: | git clone --quiet --depth 1 --branch v1.4.20 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript test "$(git -C ../buildscript rev-parse HEAD)" = 5f3745c35463ce0e2475c12269aa7a2b446456e5 - cp ../buildscript/awsconfiguration.sh . + cp ../buildscript/{master_deploy,buildenv,awsconfiguration,psvar-processor}.sh . ./awsconfiguration.sh << parameters.deploy_env >> set +x source awsenvconf - rm -f awsenvconf awsconfiguration.sh + ./psvar-processor.sh -t appenv -p "/config/${APPNAME}/deployvar" + source deployvar_env python3 -u deploy/release.py << parameters.environment >> "<< parameters.environment >>-${CIRCLE_SHA1}-${CIRCLE_BUILD_NUM}" + ./master_deploy.sh \ + -d ECS \ + -e << parameters.deploy_env >> \ + -t latest \ + -j "/config/${APPNAME}/appvar,/config/common/global-appvar" \ + -i "$APPNAME" \ + -p FARGATE - store_artifacts: path: deploy/release-<< parameters.environment >>.json destination: release diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b29311..c44bbc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,11 +19,6 @@ jobs: TEST_DATABASE_URL: postgresql://forms:forms_ci@localhost:5432/forms_test?schema=forms steps: - uses: actions/checkout@v4 - - name: Verify ECS appvar injection - run: | - python3 -m venv /tmp/forms-deploy-checks - /tmp/forms-deploy-checks/bin/pip install boto3 PyYAML - /tmp/forms-deploy-checks/bin/python -m unittest discover -s deploy -p 'test_*.py' - name: Select project Node and install pnpm shell: bash run: | diff --git a/deploy/README.md b/deploy/README.md index 483e7e7..4f2eed3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -34,46 +34,70 @@ The copy refuses a nonempty target. The source is retained and fenced against fu ## Releases -`.circleci/config.yml` follows the other v6 services' Docker build flow: checkout, -remote Docker setup, deployment dependency installation, image build, and release. -Node, pnpm, dependency installation, Prisma generation, and TypeScript compilation -run inside the Docker build. There is no CircleCI PostgreSQL service or separate -verification job. `develop` builds and deploys dev; `master` builds and deploys -production. Both use Topcoder's `org-global` context and pinned `tc-deploy-scripts` -credential helper. Forms retains its runtime and migration images and -`release.py` deployment process: target-database migrations still run before -runtime promotion. Each release also refreshes the runtime SSM appvar references -from the service and global paths, as described below. Deployments are serialized separately per environment. The -CircleCI project must be connected in the Topcoder organization to use that shared -context. - -Local checks and the same release command can run from an authorized workstation: +`.circleci/config.yml` uses the same pinned `tc-deploy-scripts` v1.4.20 flow as +`bus-api-v6`: `awsconfiguration.sh` loads credentials, `psvar-processor.sh` loads +`/config/forms-api-v6/deployvar`, and `master_deploy.sh` publishes the runtime image +and updates ECS with service/global SSM secret references. `develop` deploys dev; +`master` deploys production. Both use the Topcoder `org-global` context and are +serialized separately per environment. + +Docker builds the runtime as `forms-api-v6:latest` and a separate migration image. +The existing `release.py` now only pushes and runs the migration image in the +service's private subnets, using `MIGRATION_DATABASE_URL` as its sole secret. +It clones the service's current task definition and records the migration image +digest and task ARN in `deploy/release-.json`. A nonzero migration +exit stops the job before `master_deploy.sh` can deploy the runtime. On timeout, +inspect the reported migration task before retrying. + +The runtime deployment command is the standard shared invocation: + +```sh +./master_deploy.sh -d ECS -e DEV -t latest \ + -j "/config/${APPNAME}/appvar,/config/common/global-appvar" \ + -i "$APPNAME" -p FARGATE +``` + +The shared script tags the runtime image with `CIRCLE_BUILD_NUM`, registers the +ECS task definition, updates the existing service, and checks rollout status. +Its Fargate template uses the shared `ecsTaskExecutionRole` and writes logs to +`/aws/ecs/` with the deployment environment as stream prefix. Ensure +that role can read both SSM prefixes (and decrypt any custom KMS key). +The service's ALB readiness checks and deployment circuit breaker remain in place. +Database migrations must remain compatible with the previous runtime; service +rollback does not undo schema or data migrations. + +Provision `/config/forms-api-v6/deployvar` before the first release in each account: + +| Parameter | Value | +| --- | --- | +| `AWS_REPOSITORY`, `AWS_ECS_SERVICE`, `AWS_ECS_TASK_FAMILY`, `AWS_ECS_CONTAINER_NAME` | `forms-api-v6` | +| `AWS_ECS_CLUSTER` | Existing cluster, e.g. `topcoder-infrastructure` | +| `AWS_ECS_PORTS` | `3000:3000:tcp` | +| `AWS_ECS_FARGATE_CPU`, `AWS_ECS_FARGATE_MEMORY` | `512`, `1024` for the current Forms task size | +| `AWS_ECS_CONTAINER_CPU`, `AWS_ECS_CONTAINER_MEMORY_RESERVATION` | `0`, `512` | +| `AWS_ECS_READONLY_ROOTFILESYSTEM` | `true` | +| `AWS_ECS_TASK_ROLE_ARN` | Existing Forms task role **name**, without its ARN prefix | +| `AWS_ECS_CONTAINER_HEALTH_CMD` | Readiness command, with double quotes escaped for `psvar-processor.sh`'s shell export format | + +Move runtime environment settings into service appvars: `NODE_ENV=production`, +`PORT=3000`, `AWS_REGION=us-east-1`, and the environment's existing `CORS_ORIGINS` +and `TRUST_PROXY_CIDRS`. The shared template obtains runtime settings from SSM, +not from the previous CloudFormation task definition. Production requires its own +origins, credentials, task role, and deployvars. + +Local verification: ```sh nvm use pnpm lint && pnpm build && pnpm test -docker build --target migrate -t forms-api-v6:migrate-candidate . -docker build --target runtime -t forms-api-v6:candidate . -python3 -u deploy/release.py dev dev-UNIQUE_RELEASE_TAG ``` -Use `--network=host` for local Docker builds if the workstation bridge cannot -resolve the registry. `release.py` requires boto3 and Docker and checks account and -stack environment before pushing. Runtime and migration images receive immutable -release tags. The migration task runs in the same private subnets as the service, -uses the configured schema-owner login, and applies only the schema-scoped migrations. -Only exit code zero permits CloudFormation promotion. ECS keeps the previous task -healthy during rollout and uses its deployment circuit breaker to roll back failed -runtime starts. Database migrations must remain compatible with the previous -runtime; the service rollback does not undo data/schema migrations. - -The script writes `release-dev.json` or `release-production.json` containing image -digests and AWS task/stack identifiers. On an observation timeout, inspect the -reported task/stack operation before retrying; do not start a second migration -while the first remains active. Infrastructure edits are applied separately using -CloudFormation with the current ImageTag and environment parameters preserved; -normal app releases preserve the existing stack infrastructure and parameters while -refreshing the runtime appvar references and their SSM execution permissions. +CloudFormation remains responsible for infrastructure. Application releases now +update the ECS service directly, as in the other v6 services; the stack's ImageTag +and TaskDefinition output no longer track the active application release. When +changing infrastructure, preserve the live service task definition to avoid +restoring the stack's older task definition. A newly bootstrapped service with +DesiredCount=0 must be scaled up after its migrations and first runtime deployment. ## Dev sample @@ -85,44 +109,17 @@ CMS page at `/forms-test` and fetches its schema from the public Forms API at ru ## Runtime appvar injection -Forms uses the same ECS secret-reference mapping as the other v6 services' -`master_deploy.sh -j /config/${APPNAME}/appvar,/config/common/global-appvar`: - -- Enumerate all direct parameters from `/config/forms-api-v6/appvar` first, then - `/config/common/global-appvar`, including every paginated result. The mapping follows the pinned - [v1.4.20 deployment suite](https://github.com/topcoder-platform/tc-deploy-scripts/blob/v1.4.20/master_deploy.sh) - used by `bus-api-v6`. -- Inject each leaf name as the environment variable, using its SSM ARN in the - task definition's `Secrets` / `valueFrom`. No decrypted values enter deployment - artifacts, logs, Docker images, or plaintext ECS environment entries. -- Existing literal task environment settings take precedence; service-specific - appvars take precedence over identically named globals. There is no appvar allowlist. -- ECS resolves the values at task startup. The execution role can read both exact - prefixes. Custom KMS keys still require the existing `SecretsKmsKeyArn` grant. - -`deploy/appvars.py` implements this mapping for the existing CloudFormation-owned -Forms service. `release.py` refreshes it on every successful release instead of -reusing a fixed list of secrets. Migration tasks still receive only their explicit -`DATABASE_URL` secret; runtime appvars do not change migration credentials. - -For a configuration-only rollout with the existing image, load the target AWS -credentials and use Python with `boto3` and `PyYAML` installed: +Forms invokes the shared `master_deploy.sh` directly with +`-j /config/forms-api-v6/appvar,/config/common/global-appvar`. It injects SSM ARN +references into the runtime task's `secrets` list; service-specific names take +precedence over matching globals. ECS resolves the values at task startup. +There is no Forms-specific appvar mapping script or extra Python/YAML dependency. -```sh -python3 deploy/appvars.py dev # read-only plan and CloudFormation validation -python3 deploy/appvars.py dev --apply # update bindings/IAM and roll the same image -``` - -All existing stack parameters, including ImageTag and DesiredCount, are retained. -Only the runtime secrets list and its appvar IAM policy change. Parameter additions -and removals are picked up when generating the template. Changing an existing SSM -value alone requires a new release or an ECS force-new-deployment to refresh tasks; -an unchanged configuration plan does not force a restart. +New releases pick up parameter additions and removals. After changing only an +existing parameter's value, force a new ECS deployment to refresh running tasks. For Kafka delivery, create `BUSAPI_URL=https://api.topcoder-dev.com/v6` under the Forms dev appvar path. The Forms `AUTH0_CLIENT_ID` / `AUTH0_CLIENT_SECRET` combine with shared `AUTH0_URL`, `AUTH0_AUDIENCE`, and optional `AUTH0_PROXY_SERVER_URL` / `TOKEN_CACHE_TIME`. Production needs its corresponding URL and authorized credentials. Do not decrypt or copy shared values into the service path to perform injection. - -Deployment mapping regression tests: `python3 -m unittest discover -s deploy -p 'test_*.py'`. diff --git a/deploy/appvars.py b/deploy/appvars.py deleted file mode 100644 index 7a9b6e9..0000000 --- a/deploy/appvars.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 -"""Inject Forms and global SSM appvars as ECS secret references, matching tc-deploy-scripts -j. - -CLI: appvars.py dev|production [--apply]. Defaults to a read-only configuration plan. -Uses inherited AWS credentials. Never decrypts parameters or writes their values. -The apply mode updates only the running stack's appvar bindings and execution-role -SSM permissions, preserving its image, parameters, networking, and migration setup. -""" -import argparse -import copy -import json -import re -import time - -import boto3 -import yaml - -GLOBAL_APPVARS = '/config/common/global-appvar' - - -class CloudFormationLoader(yaml.SafeLoader): - """Read CloudFormation YAML intrinsics as JSON objects without evaluating tags. - - Used when AWS returns an existing YAML template rather than a JSON mapping. - Only standard intrinsic tags are accepted; unknown tags raise ValueError. - """ - - -def intrinsic(loader, tag, node): - """Convert one CloudFormation YAML tag to its JSON representation. - - Takes the safe loader, tag suffix, and YAML node; returns an intrinsic mapping. - Raises ValueError for unsupported tags and propagates malformed YAML errors. - """ - if tag not in {'Ref', 'Condition', 'Base64', 'GetAtt', 'GetAZs', 'ImportValue', - 'Join', 'Select', 'Split', 'Sub', 'FindInMap', 'If', 'Equals', - 'And', 'Or', 'Not', 'Cidr', 'Transform', 'Length', 'ToJsonString'}: - raise ValueError('Unsupported CloudFormation intrinsic: ' + tag) - if isinstance(node, yaml.ScalarNode): - value = loader.construct_scalar(node) - elif isinstance(node, yaml.SequenceNode): - value = loader.construct_sequence(node) - else: - value = loader.construct_mapping(node) - if tag == 'GetAtt' and isinstance(value, str): - value = value.split('.', 1) - return {tag if tag in {'Ref', 'Condition'} else 'Fn::' + tag: value} - - -CloudFormationLoader.add_multi_constructor('!', intrinsic) - - -def load_template(body): - """Return an independent JSON-compatible stack template from an AWS template body. - - Accepts the dict or YAML/JSON text returned by get_template. Raises ValueError - for missing resources, and propagates safe YAML parsing errors. - """ - template = copy.deepcopy(body) if isinstance(body, dict) else yaml.load(body, Loader=CloudFormationLoader) - if not isinstance(template, dict) or not isinstance(template.get('Resources'), dict): - raise ValueError('CloudFormation template has no resources.') - return template - - -def appvar_secrets(ssm, prefix, partition, region, account, environment_names=()): - """Discover all direct SSM appvars with the same precedence as master_deploy.sh -j. - - Takes the SSM client, service path, AWS ARN components, and existing literal - environment names. Returns ECS name/valueFrom entries: literal environment - first, service parameters next, global parameters last. Pagination is consumed - without decryption. Raises ValueError for unsafe variable names; AWS errors - propagate. Values are never placed in the generated task definition. - """ - seen = set(environment_names) - secrets = [] - for path in [prefix.rstrip('/'), GLOBAL_APPVARS]: - for page in ssm.get_paginator('get_parameters_by_path').paginate( - Path=path, Recursive=False, WithDecryption=False): - for parameter in page['Parameters']: - name = parameter['Name'].removeprefix(path + '/') - if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', name): - raise ValueError('Appvar name is not a direct environment variable: ' + parameter['Name']) - if name not in seen: - seen.add(name) - secrets.append({'name': name, 'valueFrom': - f'arn:{partition}:ssm:{region}:{account}:parameter{parameter["Name"]}'}) - return sorted(secrets, key=lambda item: item['name']) - - -def configure_template(body, ssm, prefix, partition, region, account): - """Refresh the Forms runtime secrets and execution-role grants in a stack template. - - Takes the current template, SSM client, and deployment identifiers. Returns a - new template, preserving unrelated resources, runtime image, and literal env. - Rebuilds the secret list so removed parameters are no longer injected. Raises - KeyError for an unexpected Forms stack layout, ValueError for invalid appvars, - and propagates SSM read failures. Used by releases and configuration-only rolls. - """ - template = load_template(body) - resources = template['Resources'] - containers = resources['TaskDefinition']['Properties']['ContainerDefinitions'] - container = next(item for item in containers if item['Name'] == 'forms-api-v6') - entries = appvar_secrets(ssm, prefix, partition, region, account, - [item['Name'] for item in container.get('Environment', [])]) - container['Secrets'] = [{'Name': item['name'], 'ValueFrom': item['valueFrom']} for item in entries] - policies = resources['ExecutionRole']['Properties']['Policies'] - policy_name = 'forms-runtime-appvars' - policies[:] = [policy for policy in policies if policy['PolicyName'] != policy_name] - policies.append({'PolicyName': policy_name, 'PolicyDocument': { - 'Version': '2012-10-17', 'Statement': [{ - 'Effect': 'Allow', 'Action': ['ssm:GetParameters'], - 'Resource': [f'arn:{partition}:ssm:{region}:{account}:parameter{path.rstrip("/")}/*' - for path in [prefix, GLOBAL_APPVARS]], - }], - }}) - return template - - -def main(): - """Plan or apply appvar injection to the selected existing Forms stack. - - Reads CLI arguments and inherited AWS credentials; returns None after showing - names only or completing a configuration roll. Raises on account mismatch, - unstable stack, invalid template, AWS errors, deployment rollback, or timeout. - Does not submit forms, replay events, run migrations, or change appvar values. - """ - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('environment', choices=['dev', 'production']) - parser.add_argument('--apply', action='store_true') - args = parser.parse_args() - session = boto3.Session(region_name='us-east-1') - identity = session.client('sts').get_caller_identity() - account = identity['Account'] - if (account == '811668436784') != (args.environment == 'dev'): - raise RuntimeError('AWS account does not match the selected environment.') - cfn = session.client('cloudformation') - stack_name = 'forms-api-v6-' + args.environment - stack = cfn.describe_stacks(StackName=stack_name)['Stacks'][0] - if stack['StackStatus'] not in {'CREATE_COMPLETE', 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_COMPLETE'}: - raise RuntimeError('Stack is not stable; inspect the existing operation.') - settings = {item['ParameterKey']: item['ParameterValue'] for item in stack['Parameters']} - if settings['Environment'] != args.environment or settings['BootstrapOnly'] != 'false': - raise RuntimeError('Stack environment or bootstrap state does not permit a configuration roll.') - original = cfn.get_template(StackName=stack_name, TemplateStage='Original')['TemplateBody'] - template = configure_template(original, session.client('ssm'), settings['ParameterPrefix'], - identity['Arn'].split(':')[1], session.region_name, account) - container = template['Resources']['TaskDefinition']['Properties']['ContainerDefinitions'][0] - print('Runtime SSM appvars: ' + ', '.join(item['Name'] for item in container['Secrets'])) - body = json.dumps(template) - cfn.validate_template(TemplateBody=body) - if not args.apply: - print('Template validated. Re-run with --apply to roll the existing service image with these bindings.') - return - if template == load_template(original): - print('Appvar bindings and execution permissions already match; no stack change required.') - return - cfn.update_stack(StackName=stack_name, TemplateBody=body, - Parameters=[{'ParameterKey': item['ParameterKey'], 'UsePreviousValue': True} - for item in stack['Parameters']], Capabilities=['CAPABILITY_IAM']) - deadline = time.monotonic() + 1800 - while time.monotonic() < deadline: - state = cfn.describe_stacks(StackName=stack_name)['Stacks'][0]['StackStatus'] - if state == 'UPDATE_COMPLETE': - print('Appvar configuration deployed: ' + stack_name) - return - if state not in {'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS'}: - raise RuntimeError('Configuration roll failed or rolled back: ' + state) - print('Waiting for appvar configuration roll...', flush=True) - time.sleep(15) - raise RuntimeError('Timed out observing the stack; inspect its existing operation before retrying.') - - -if __name__ == '__main__': - main() diff --git a/deploy/release.py b/deploy/release.py index 9c55054..389575d 100644 --- a/deploy/release.py +++ b/deploy/release.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -"""Deploy prebuilt runtime and migration images using the existing Forms stack. +"""Run the migration gate before the shared Topcoder ECS deployment script. -CLI: release.py dev|production IMAGE_TAG [RUNTIME_IMAGE] [MIGRATION_IMAGE]. -Requires inherited AWS credentials, boto3, and Docker. Pushes immutable ECR tags, -discovers service/global SSM appvars, executes migrations as a private one-shot ECS -task, then updates CloudFormation with fresh ECS secret references. -Raises on failures; runtime promotion never occurs after failed migrations. +CLI: release.py dev|production IMAGE_TAG [MIGRATION_IMAGE]. +Requires inherited AWS credentials, boto3, and Docker. Pushes the prebuilt migration +image to ECR and runs it as a private one-shot ECS task. Raises on failures so +CircleCI stops before master_deploy.sh promotes the runtime image. """ import base64 import copy @@ -19,8 +18,6 @@ import boto3 -from appvars import configure_template - def wait_until(description, check, seconds=1800): """Poll a live deployment check every 15 seconds, returning its truthy result. @@ -39,20 +36,19 @@ def wait_until(description, check, seconds=1800): def main(): - """Validate release arguments, discover appvars, migrate, promote, and save evidence. + """Validate CLI arguments and environment, run migrations, and save evidence. - Returns None on success; AWS, Docker, failed migration, and failed deployment + Returns None on success; AWS, Docker, and failed migration errors propagate. Credentials are passed through stdin/environment, never argv. """ - if len(sys.argv) not in (3, 5) or sys.argv[1] not in ('dev', 'production'): + if len(sys.argv) not in (3, 4) or sys.argv[1] not in ('dev', 'production'): raise RuntimeError(__doc__) environment, tag = sys.argv[1:3] if not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9_.-]{0,110}', tag): raise RuntimeError('Invalid immutable release tag.') - local_runtime, local_migration = sys.argv[3:] or ['forms-api-v6:candidate', 'forms-api-v6:migrate-candidate'] + local_migration = sys.argv[3] if len(sys.argv) == 4 else 'forms-api-v6:migrate-candidate' session = boto3.Session(region_name=os.environ.get('AWS_REGION', 'us-east-1')) - identity = session.client('sts').get_caller_identity() - account = identity['Account'] + account = session.client('sts').get_caller_identity()['Account'] if (account == '811668436784') != (environment == 'dev'): raise RuntimeError('AWS account does not match the selected deployment environment.') cfn, ecs, ecr = [session.client(name) for name in ['cloudformation', 'ecs', 'ecr']] @@ -63,38 +59,31 @@ def main(): settings = {x['ParameterKey']: x['ParameterValue'] for x in stack['Parameters']} if settings['Environment'] != environment or settings['BootstrapOnly'] != 'false': raise RuntimeError('Stack environment or bootstrap state does not permit deployment.') - template = configure_template( - cfn.get_template(StackName=stack_name, TemplateStage='Original')['TemplateBody'], - session.client('ssm'), settings['ParameterPrefix'], identity['Arn'].split(':')[1], - session.region_name, account) - cfn.validate_template(TemplateBody=json.dumps(template)) output = {x['OutputKey']: x['OutputValue'] for x in stack['Outputs']} repository = output['RepositoryUri'] authorization = ecr.get_authorization_token()['authorizationData'][0] user, password = base64.b64decode(authorization['authorizationToken']).decode().split(':', 1) subprocess.run(['docker', 'login', '--username', user, '--password-stdin', authorization['proxyEndpoint']], input=password, text=True, check=True, stdout=subprocess.DEVNULL) - images = [] - for local, suffix in [(local_runtime, ''), (local_migration, '-migrate')]: - destination = f'{repository}:{tag}{suffix}' - subprocess.run(['docker', 'tag', local, destination], check=True) - subprocess.run(['docker', 'push', destination], check=True) - digest = ecr.describe_images(repositoryName='forms-api-v6', imageIds=[{'imageTag': tag + suffix}])['imageDetails'][0]['imageDigest'] - images.append(f'{repository}@{digest}') - current = ecs.describe_task_definition(taskDefinition=output['TaskDefinitionArn'])['taskDefinition'] + destination = f'{repository}:{tag}-migrate' + subprocess.run(['docker', 'tag', local_migration, destination], check=True) + subprocess.run(['docker', 'push', destination], check=True) + digest = ecr.describe_images(repositoryName='forms-api-v6', imageIds=[{'imageTag': tag + '-migrate'}])['imageDetails'][0]['imageDigest'] + migration_image = f'{repository}@{digest}' + service = ecs.describe_services(cluster=settings['ClusterName'], services=[output['ServiceName']])['services'][0] + current = ecs.describe_task_definition(taskDefinition=service['taskDefinition'])['taskDefinition'] allowed = {'family', 'taskRoleArn', 'executionRoleArn', 'networkMode', 'containerDefinitions', 'volumes', 'placementConstraints', 'requiresCompatibilities', 'cpu', 'memory', 'runtimePlatform', 'ephemeralStorage'} migration = {k: copy.deepcopy(v) for k, v in current.items() if k in allowed} migration['family'] = 'forms-api-v6-migrate' container = migration['containerDefinitions'][0] - container['image'] = images[1] + container['image'] = migration_image container['readonlyRootFilesystem'] = False container.pop('healthCheck', None) container['portMappings'] = [] container['secrets'] = [{'name': 'DATABASE_URL', 'valueFrom': settings['ParameterPrefix'] + '/MIGRATION_DATABASE_URL'}] container['command'] = ['/bin/sh', '-c', 'pnpm migrate:deploy'] migration_definition = ecs.register_task_definition(**migration)['taskDefinition']['taskDefinitionArn'] - service = ecs.describe_services(cluster=settings['ClusterName'], services=[output['ServiceName']])['services'][0] result = ecs.run_task(cluster=settings['ClusterName'], taskDefinition=migration_definition, launchType='FARGATE', networkConfiguration=service['networkConfiguration'], startedBy='forms-release') @@ -112,30 +101,9 @@ def migration_finished(): completed = wait_until('migration task ' + migration_arn, migration_finished) if any(c.get('exitCode') != 0 for c in completed['containers']): - raise RuntimeError('Migration failed; inspect /aws/ecs/forms-api-v6-' + environment + '. Runtime was not promoted.') - parameters = [({'ParameterKey': x['ParameterKey'], 'ParameterValue': tag} if x['ParameterKey'] == 'ImageTag' - else {'ParameterKey': x['ParameterKey'], 'ParameterValue': str(max(1, int(settings['DesiredCount'])))} if x['ParameterKey'] == 'DesiredCount' - else {'ParameterKey': x['ParameterKey'], 'UsePreviousValue': True}) for x in stack['Parameters']] - result = cfn.update_stack(StackName=stack_name, TemplateBody=json.dumps(template), - Parameters=parameters, Capabilities=['CAPABILITY_IAM']) - print('Updating stack: ' + result['StackId'], flush=True) - - def stack_finished(): - """Read the same stack operation; return completed stack or raise on rollback.""" - state = cfn.describe_stacks(StackName=stack_name)['Stacks'][0] - if state['StackStatus'] == 'UPDATE_COMPLETE': - return state - if state['StackStatus'] not in ('UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS'): - raise RuntimeError('Stack promotion failed or rolled back: ' + state['StackStatus']) - return None - - wait_until('CloudFormation promotion', stack_finished) - live = ecs.describe_services(cluster=settings['ClusterName'], services=[output['ServiceName']])['services'][0] - definition = ecs.describe_task_definition(taskDefinition=live['taskDefinition'])['taskDefinition'] - if definition['containerDefinitions'][0]['image'] != f'{repository}:{tag}' or live['runningCount'] < 1: - raise RuntimeError('ECS did not retain the requested release.') - evidence = {'environment': environment, 'tag': tag, 'runtimeImage': images[0], 'migrationImage': images[1], - 'migrationTask': migration_arn, 'taskDefinition': live['taskDefinition'], 'stack': stack_name} + raise RuntimeError('Migration failed; inspect the migration task logs. Runtime was not promoted.') + evidence = {'environment': environment, 'tag': tag, 'migrationImage': migration_image, + 'migrationTask': migration_arn, 'stack': stack_name} Path('deploy/release-' + environment + '.json').write_text(json.dumps(evidence, indent=2) + '\n') print(json.dumps(evidence, indent=2)) diff --git a/deploy/service.yaml b/deploy/service.yaml index b71a1c1..d13513e 100644 --- a/deploy/service.yaml +++ b/deploy/service.yaml @@ -150,7 +150,7 @@ Resources: Action: ssm:GetParameters Resource: - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter${ParameterPrefix}/* - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/config/common/global-appvar/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/config/common/global-appvar/AUTH_SECRET - !If - HasSecretsKmsKey - Effect: Allow diff --git a/deploy/test_appvars.py b/deploy/test_appvars.py deleted file mode 100644 index c842713..0000000 --- a/deploy/test_appvars.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Regression tests for the v6 SSM-to-ECS appvar mapping and template preservation.""" -import copy -from pathlib import Path -import unittest -from unittest.mock import Mock - -from appvars import appvar_secrets, configure_template, load_template - -PREFIX = '/config/forms-api-v6/appvar' -GLOBAL = '/config/common/global-appvar' - - -def ssm_fixture(pages): - """Return an SSM mock serving path-indexed pages; unknown paths raise KeyError.""" - client = Mock() - client.get_paginator.return_value.paginate.side_effect = lambda **kwargs: iter(pages[kwargs['Path']]) - return client - - -def parameter(path, name): - """Return one SSM fixture with a sentinel value that must never reach a template.""" - return {'Name': path + '/' + name, 'Value': 'DO_NOT_EMBED_PARAMETER_VALUES'} - - -class AppvarTests(unittest.TestCase): - """Exercise pagination, v6 precedence, safe references, and CloudFormation changes.""" - - def test_service_precedence_and_all_pages(self): - """Map every page, preferring literal env then service values over globals.""" - client = ssm_fixture({ - PREFIX: [{'Parameters': [parameter(PREFIX, 'AUTH0_CLIENT_ID'), parameter(PREFIX, 'PORT')]}, - {'Parameters': [parameter(PREFIX, 'AUTH0_CLIENT_SECRET')]}], - GLOBAL: [{'Parameters': [parameter(GLOBAL, 'AUTH0_CLIENT_ID'), parameter(GLOBAL, 'AUTH0_URL')]}, - {'Parameters': [parameter(GLOBAL, 'AUTH_SECRET')]}], - }) - result = appvar_secrets(client, PREFIX, 'aws', 'us-east-1', '123', ['PORT']) - refs = {entry['name']: entry['valueFrom'] for entry in result} - self.assertEqual(set(refs), {'AUTH0_CLIENT_ID', 'AUTH0_CLIENT_SECRET', 'AUTH0_URL', 'AUTH_SECRET'}) - self.assertEqual(refs['AUTH0_CLIENT_ID'], 'arn:aws:ssm:us-east-1:123:parameter' + PREFIX + '/AUTH0_CLIENT_ID') - self.assertEqual(refs['AUTH0_URL'], 'arn:aws:ssm:us-east-1:123:parameter' + GLOBAL + '/AUTH0_URL') - self.assertNotIn('DO_NOT_EMBED_PARAMETER_VALUES', str(result)) - for call in client.get_paginator.return_value.paginate.call_args_list: - self.assertFalse(call.kwargs['Recursive']) - self.assertFalse(call.kwargs['WithDecryption']) - - def test_template_preserves_runtime_and_other_resources(self): - """Refresh only runtime secret bindings and a narrowly scoped IAM policy.""" - original = load_template(Path(__file__).with_name('service.yaml').read_text()) - snapshot = copy.deepcopy(original) - client = ssm_fixture({PREFIX: [{'Parameters': [parameter(PREFIX, 'DATABASE_URL'), parameter(PREFIX, 'BUSAPI_URL')]}], - GLOBAL: [{'Parameters': [parameter(GLOBAL, 'AUTH_SECRET')]}]}) - result = configure_template(original, client, PREFIX, 'aws', 'us-east-1', '123') - self.assertEqual(original, snapshot) - self.assertEqual(result['Parameters'], original['Parameters']) - for name in original['Resources']: - if name not in {'TaskDefinition', 'ExecutionRole'}: - self.assertEqual(result['Resources'][name], original['Resources'][name]) - container = result['Resources']['TaskDefinition']['Properties']['ContainerDefinitions'][0] - before = original['Resources']['TaskDefinition']['Properties']['ContainerDefinitions'][0] - self.assertEqual({k: v for k, v in container.items() if k != 'Secrets'}, - {k: v for k, v in before.items() if k != 'Secrets'}) - self.assertEqual({item['Name'] for item in container['Secrets']}, {'DATABASE_URL', 'BUSAPI_URL', 'AUTH_SECRET'}) - policy = result['Resources']['ExecutionRole']['Properties']['Policies'][-1] - self.assertEqual(policy['PolicyDocument']['Statement'][0]['Action'], ['ssm:GetParameters']) - self.assertEqual(policy['PolicyDocument']['Statement'][0]['Resource'], [ - 'arn:aws:ssm:us-east-1:123:parameter' + PREFIX + '/*', - 'arn:aws:ssm:us-east-1:123:parameter' + GLOBAL + '/*']) - self.assertNotIn('DO_NOT_EMBED_PARAMETER_VALUES', str(result)) - - def test_refresh_is_idempotent_and_removes_deleted_parameters(self): - """Remove obsolete references without accumulating permissions on later releases.""" - original = Path(__file__).with_name('service.yaml').read_text() - client = ssm_fixture({PREFIX: [{'Parameters': [parameter(PREFIX, 'DATABASE_URL')]}], GLOBAL: [{'Parameters': []}]}) - once = configure_template(original, client, PREFIX, 'aws', 'us-east-1', '123') - twice = configure_template(once, client, PREFIX, 'aws', 'us-east-1', '123') - self.assertEqual(once, twice) - names = [entry['Name'] for entry in twice['Resources']['TaskDefinition']['Properties']['ContainerDefinitions'][0]['Secrets']] - self.assertEqual(names, ['DATABASE_URL']) - - def test_read_errors_and_unsafe_names_stop_generation(self): - """Fail before deployment when appvar discovery fails or returns unsafe names.""" - client = Mock() - client.get_paginator.side_effect = RuntimeError('SSM unavailable') - with self.assertRaises(RuntimeError): - appvar_secrets(client, PREFIX, 'aws', 'us-east-1', '123') - client = ssm_fixture({PREFIX: [{'Parameters': [parameter(PREFIX, 'nested/SECRET')]}]}) - with self.assertRaises(ValueError): - appvar_secrets(client, PREFIX, 'aws', 'us-east-1', '123') - - def test_cloudformation_intrinsics_are_not_evaluated(self): - """Preserve scalar/sequence/mapping intrinsics and reject unsafe YAML objects.""" - result = load_template('Resources:\n Arn: !GetAtt Role.Arn\n Choice: !If [IsDev, dev, prod]\n Text: !Sub "${Name}"\n') - self.assertEqual(result['Resources']['Arn'], {'Fn::GetAtt': ['Role', 'Arn']}) - self.assertEqual(result['Resources']['Choice'], {'Fn::If': ['IsDev', 'dev', 'prod']}) - with self.assertRaises(ValueError): - load_template('Resources: !Run arbitrary') - - -if __name__ == '__main__': - unittest.main()