diff --git a/.circleci/config.yml b/.circleci/config.yml index c018910..0456752 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,19 +39,27 @@ jobs: name: Build runtime and one-shot migration images command: | docker buildx build --load --target migrate -t forms-api-v6:migrate-candidate . - docker buildx build --load --target runtime -t forms-api-v6:candidate . + docker buildx build --load --target runtime -t forms-api-v6:latest . - run: - name: Authenticate, migrate, and deploy through CloudFormation + name: Migrate and deploy using the Topcoder deployment suite no_output_timeout: 35m command: | git clone --quiet --depth 1 --branch v1.4.20 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript test "$(git -C ../buildscript rev-parse HEAD)" = 5f3745c35463ce0e2475c12269aa7a2b446456e5 - cp ../buildscript/awsconfiguration.sh . + cp ../buildscript/{master_deploy,buildenv,awsconfiguration,psvar-processor}.sh . ./awsconfiguration.sh << parameters.deploy_env >> set +x source awsenvconf - rm -f awsenvconf awsconfiguration.sh + ./psvar-processor.sh -t appenv -p "/config/${APPNAME}/deployvar" + source deployvar_env python3 -u deploy/release.py << parameters.environment >> "<< parameters.environment >>-${CIRCLE_SHA1}-${CIRCLE_BUILD_NUM}" + ./master_deploy.sh \ + -d ECS \ + -e << parameters.deploy_env >> \ + -t latest \ + -j "/config/${APPNAME}/appvar,/config/common/global-appvar" \ + -i "$APPNAME" \ + -p FARGATE - store_artifacts: path: deploy/release-<< parameters.environment >>.json destination: release diff --git a/deploy/README.md b/deploy/README.md index bb5c26c..4f2eed3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -34,44 +34,70 @@ The copy refuses a nonempty target. The source is retained and fenced against fu ## Releases -`.circleci/config.yml` follows the other v6 services' Docker build flow: checkout, -remote Docker setup, deployment dependency installation, image build, and release. -Node, pnpm, dependency installation, Prisma generation, and TypeScript compilation -run inside the Docker build. There is no CircleCI PostgreSQL service or separate -verification job. `develop` builds and deploys dev; `master` builds and deploys -production. Both use Topcoder's `org-global` context and pinned `tc-deploy-scripts` -credential helper. Forms retains its runtime and migration images and -`release.py` deployment process: target-database migrations still run before -runtime promotion. Deployments are serialized separately per environment. The -CircleCI project must be connected in the Topcoder organization to use that shared -context. - -Local checks and the same release command can run from an authorized workstation: +`.circleci/config.yml` uses the same pinned `tc-deploy-scripts` v1.4.20 flow as +`bus-api-v6`: `awsconfiguration.sh` loads credentials, `psvar-processor.sh` loads +`/config/forms-api-v6/deployvar`, and `master_deploy.sh` publishes the runtime image +and updates ECS with service/global SSM secret references. `develop` deploys dev; +`master` deploys production. Both use the Topcoder `org-global` context and are +serialized separately per environment. + +Docker builds the runtime as `forms-api-v6:latest` and a separate migration image. +The existing `release.py` now only pushes and runs the migration image in the +service's private subnets, using `MIGRATION_DATABASE_URL` as its sole secret. +It clones the service's current task definition and records the migration image +digest and task ARN in `deploy/release-.json`. A nonzero migration +exit stops the job before `master_deploy.sh` can deploy the runtime. On timeout, +inspect the reported migration task before retrying. + +The runtime deployment command is the standard shared invocation: + +```sh +./master_deploy.sh -d ECS -e DEV -t latest \ + -j "/config/${APPNAME}/appvar,/config/common/global-appvar" \ + -i "$APPNAME" -p FARGATE +``` + +The shared script tags the runtime image with `CIRCLE_BUILD_NUM`, registers the +ECS task definition, updates the existing service, and checks rollout status. +Its Fargate template uses the shared `ecsTaskExecutionRole` and writes logs to +`/aws/ecs/` with the deployment environment as stream prefix. Ensure +that role can read both SSM prefixes (and decrypt any custom KMS key). +The service's ALB readiness checks and deployment circuit breaker remain in place. +Database migrations must remain compatible with the previous runtime; service +rollback does not undo schema or data migrations. + +Provision `/config/forms-api-v6/deployvar` before the first release in each account: + +| Parameter | Value | +| --- | --- | +| `AWS_REPOSITORY`, `AWS_ECS_SERVICE`, `AWS_ECS_TASK_FAMILY`, `AWS_ECS_CONTAINER_NAME` | `forms-api-v6` | +| `AWS_ECS_CLUSTER` | Existing cluster, e.g. `topcoder-infrastructure` | +| `AWS_ECS_PORTS` | `3000:3000:tcp` | +| `AWS_ECS_FARGATE_CPU`, `AWS_ECS_FARGATE_MEMORY` | `512`, `1024` for the current Forms task size | +| `AWS_ECS_CONTAINER_CPU`, `AWS_ECS_CONTAINER_MEMORY_RESERVATION` | `0`, `512` | +| `AWS_ECS_READONLY_ROOTFILESYSTEM` | `true` | +| `AWS_ECS_TASK_ROLE_ARN` | Existing Forms task role **name**, without its ARN prefix | +| `AWS_ECS_CONTAINER_HEALTH_CMD` | Readiness command, with double quotes escaped for `psvar-processor.sh`'s shell export format | + +Move runtime environment settings into service appvars: `NODE_ENV=production`, +`PORT=3000`, `AWS_REGION=us-east-1`, and the environment's existing `CORS_ORIGINS` +and `TRUST_PROXY_CIDRS`. The shared template obtains runtime settings from SSM, +not from the previous CloudFormation task definition. Production requires its own +origins, credentials, task role, and deployvars. + +Local verification: ```sh nvm use pnpm lint && pnpm build && pnpm test -docker build --target migrate -t forms-api-v6:migrate-candidate . -docker build --target runtime -t forms-api-v6:candidate . -python3 -u deploy/release.py dev dev-UNIQUE_RELEASE_TAG ``` -Use `--network=host` for local Docker builds if the workstation bridge cannot -resolve the registry. `release.py` requires boto3 and Docker and checks account and -stack environment before pushing. Runtime and migration images receive immutable -release tags. The migration task runs in the same private subnets as the service, -uses the configured schema-owner login, and applies only the schema-scoped migrations. -Only exit code zero permits CloudFormation promotion. ECS keeps the previous task -healthy during rollout and uses its deployment circuit breaker to roll back failed -runtime starts. Database migrations must remain compatible with the previous -runtime; the service rollback does not undo data/schema migrations. - -The script writes `release-dev.json` or `release-production.json` containing image -digests and AWS task/stack identifiers. On an observation timeout, inspect the -reported task/stack operation before retrying; do not start a second migration -while the first remains active. Infrastructure edits are applied separately using -CloudFormation with the current ImageTag and environment parameters preserved; -normal app releases retain the existing stack template and parameters. +CloudFormation remains responsible for infrastructure. Application releases now +update the ECS service directly, as in the other v6 services; the stack's ImageTag +and TaskDefinition output no longer track the active application release. When +changing infrastructure, preserve the live service task definition to avoid +restoring the stack's older task definition. A newly bootstrapped service with +DesiredCount=0 must be scaled up after its migrations and first runtime deployment. ## Dev sample @@ -80,3 +106,20 @@ normal app releases retain the existing stack template and parameters. It refuses production and conflicting published definitions. The corresponding CMS seed lives in `payload-cms/scripts/seed-forms-test.ts`. The website resolves the CMS page at `/forms-test` and fetches its schema from the public Forms API at runtime. + +## Runtime appvar injection + +Forms invokes the shared `master_deploy.sh` directly with +`-j /config/forms-api-v6/appvar,/config/common/global-appvar`. It injects SSM ARN +references into the runtime task's `secrets` list; service-specific names take +precedence over matching globals. ECS resolves the values at task startup. +There is no Forms-specific appvar mapping script or extra Python/YAML dependency. + +New releases pick up parameter additions and removals. After changing only an +existing parameter's value, force a new ECS deployment to refresh running tasks. + +For Kafka delivery, create `BUSAPI_URL=https://api.topcoder-dev.com/v6` under the +Forms dev appvar path. The Forms `AUTH0_CLIENT_ID` / `AUTH0_CLIENT_SECRET` combine +with shared `AUTH0_URL`, `AUTH0_AUDIENCE`, and optional `AUTH0_PROXY_SERVER_URL` / +`TOKEN_CACHE_TIME`. Production needs its corresponding URL and authorized credentials. +Do not decrypt or copy shared values into the service path to perform injection. diff --git a/deploy/release.py b/deploy/release.py index 0eb90aa..389575d 100644 --- a/deploy/release.py +++ b/deploy/release.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -"""Deploy prebuilt runtime and migration images using the existing Forms stack. +"""Run the migration gate before the shared Topcoder ECS deployment script. -CLI: release.py dev|production IMAGE_TAG [RUNTIME_IMAGE] [MIGRATION_IMAGE]. -Requires inherited AWS credentials, boto3, and Docker. Pushes immutable ECR tags, -executes migrations as a private one-shot ECS task, then updates CloudFormation. -Raises on failures; runtime promotion never occurs after failed migrations. +CLI: release.py dev|production IMAGE_TAG [MIGRATION_IMAGE]. +Requires inherited AWS credentials, boto3, and Docker. Pushes the prebuilt migration +image to ECR and runs it as a private one-shot ECS task. Raises on failures so +CircleCI stops before master_deploy.sh promotes the runtime image. """ import base64 import copy @@ -36,17 +36,17 @@ def wait_until(description, check, seconds=1800): def main(): - """Validate release arguments and environment, migrate, promote, and save evidence. + """Validate CLI arguments and environment, run migrations, and save evidence. - Returns None on success; AWS, Docker, failed migration, and failed deployment + Returns None on success; AWS, Docker, and failed migration errors propagate. Credentials are passed through stdin/environment, never argv. """ - if len(sys.argv) not in (3, 5) or sys.argv[1] not in ('dev', 'production'): + if len(sys.argv) not in (3, 4) or sys.argv[1] not in ('dev', 'production'): raise RuntimeError(__doc__) environment, tag = sys.argv[1:3] if not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9_.-]{0,110}', tag): raise RuntimeError('Invalid immutable release tag.') - local_runtime, local_migration = sys.argv[3:] or ['forms-api-v6:candidate', 'forms-api-v6:migrate-candidate'] + local_migration = sys.argv[3] if len(sys.argv) == 4 else 'forms-api-v6:migrate-candidate' session = boto3.Session(region_name=os.environ.get('AWS_REGION', 'us-east-1')) account = session.client('sts').get_caller_identity()['Account'] if (account == '811668436784') != (environment == 'dev'): @@ -65,27 +65,25 @@ def main(): user, password = base64.b64decode(authorization['authorizationToken']).decode().split(':', 1) subprocess.run(['docker', 'login', '--username', user, '--password-stdin', authorization['proxyEndpoint']], input=password, text=True, check=True, stdout=subprocess.DEVNULL) - images = [] - for local, suffix in [(local_runtime, ''), (local_migration, '-migrate')]: - destination = f'{repository}:{tag}{suffix}' - subprocess.run(['docker', 'tag', local, destination], check=True) - subprocess.run(['docker', 'push', destination], check=True) - digest = ecr.describe_images(repositoryName='forms-api-v6', imageIds=[{'imageTag': tag + suffix}])['imageDetails'][0]['imageDigest'] - images.append(f'{repository}@{digest}') - current = ecs.describe_task_definition(taskDefinition=output['TaskDefinitionArn'])['taskDefinition'] + destination = f'{repository}:{tag}-migrate' + subprocess.run(['docker', 'tag', local_migration, destination], check=True) + subprocess.run(['docker', 'push', destination], check=True) + digest = ecr.describe_images(repositoryName='forms-api-v6', imageIds=[{'imageTag': tag + '-migrate'}])['imageDetails'][0]['imageDigest'] + migration_image = f'{repository}@{digest}' + service = ecs.describe_services(cluster=settings['ClusterName'], services=[output['ServiceName']])['services'][0] + current = ecs.describe_task_definition(taskDefinition=service['taskDefinition'])['taskDefinition'] allowed = {'family', 'taskRoleArn', 'executionRoleArn', 'networkMode', 'containerDefinitions', 'volumes', 'placementConstraints', 'requiresCompatibilities', 'cpu', 'memory', 'runtimePlatform', 'ephemeralStorage'} migration = {k: copy.deepcopy(v) for k, v in current.items() if k in allowed} migration['family'] = 'forms-api-v6-migrate' container = migration['containerDefinitions'][0] - container['image'] = images[1] + container['image'] = migration_image container['readonlyRootFilesystem'] = False container.pop('healthCheck', None) container['portMappings'] = [] container['secrets'] = [{'name': 'DATABASE_URL', 'valueFrom': settings['ParameterPrefix'] + '/MIGRATION_DATABASE_URL'}] container['command'] = ['/bin/sh', '-c', 'pnpm migrate:deploy'] migration_definition = ecs.register_task_definition(**migration)['taskDefinition']['taskDefinitionArn'] - service = ecs.describe_services(cluster=settings['ClusterName'], services=[output['ServiceName']])['services'][0] result = ecs.run_task(cluster=settings['ClusterName'], taskDefinition=migration_definition, launchType='FARGATE', networkConfiguration=service['networkConfiguration'], startedBy='forms-release') @@ -103,30 +101,9 @@ def migration_finished(): completed = wait_until('migration task ' + migration_arn, migration_finished) if any(c.get('exitCode') != 0 for c in completed['containers']): - raise RuntimeError('Migration failed; inspect /aws/ecs/forms-api-v6-' + environment + '. Runtime was not promoted.') - parameters = [({'ParameterKey': x['ParameterKey'], 'ParameterValue': tag} if x['ParameterKey'] == 'ImageTag' - else {'ParameterKey': x['ParameterKey'], 'ParameterValue': str(max(1, int(settings['DesiredCount'])))} if x['ParameterKey'] == 'DesiredCount' - else {'ParameterKey': x['ParameterKey'], 'UsePreviousValue': True}) for x in stack['Parameters']] - result = cfn.update_stack(StackName=stack_name, UsePreviousTemplate=True, - Parameters=parameters, Capabilities=['CAPABILITY_IAM']) - print('Updating stack: ' + result['StackId'], flush=True) - - def stack_finished(): - """Read the same stack operation; return completed stack or raise on rollback.""" - state = cfn.describe_stacks(StackName=stack_name)['Stacks'][0] - if state['StackStatus'] == 'UPDATE_COMPLETE': - return state - if state['StackStatus'] not in ('UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS'): - raise RuntimeError('Stack promotion failed or rolled back: ' + state['StackStatus']) - return None - - wait_until('CloudFormation promotion', stack_finished) - live = ecs.describe_services(cluster=settings['ClusterName'], services=[output['ServiceName']])['services'][0] - definition = ecs.describe_task_definition(taskDefinition=live['taskDefinition'])['taskDefinition'] - if definition['containerDefinitions'][0]['image'] != f'{repository}:{tag}' or live['runningCount'] < 1: - raise RuntimeError('ECS did not retain the requested release.') - evidence = {'environment': environment, 'tag': tag, 'runtimeImage': images[0], 'migrationImage': images[1], - 'migrationTask': migration_arn, 'taskDefinition': live['taskDefinition'], 'stack': stack_name} + raise RuntimeError('Migration failed; inspect the migration task logs. Runtime was not promoted.') + evidence = {'environment': environment, 'tag': tag, 'migrationImage': migration_image, + 'migrationTask': migration_arn, 'stack': stack_name} Path('deploy/release-' + environment + '.json').write_text(json.dumps(evidence, indent=2) + '\n') print(json.dumps(evidence, indent=2)) diff --git a/docs/operations.md b/docs/operations.md index 464a40e..7381311 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -23,6 +23,11 @@ The Prisma client uses the PostgreSQL driver adapter and generated TypeScript co ## Outbound Bus API +ECS releases inject all service appvars from `/config/forms-api-v6/appvar` and +shared appvars from `/config/common/global-appvar` as SSM secret references, with +service values taking precedence. See [deployment injection](../deploy/README.md#runtime-appvar-injection) +for configuration-only rolls and required execution-role permissions. + Ordinary submissions need no Bus API configuration. To accept `kafka=true` submissions successfully, configure: | Variable | Meaning |