From 352bcdd94c8d8cba1f1eb438e58c0b719bac6386 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Sat, 17 Oct 2026 11:56:14 -0400 Subject: [PATCH 1/2] audit-specs: Check ssh.socket instead of ssh running in qemu baselines The qemu image build has been failing supascan since Oct 14 because ssh.service isn't running when the baseline check runs. Ubuntu 24.04 starts sshd through ssh.socket. The build boots the latest noble cloud image, then runs apt-get dist-upgrade. When the archive has a newer openssh-server than the image, the package upgrade stops ssh.service and restarts only ssh.socket. Packer's existing session keeps working, but nothing opens a new connection, so ssh.service stays stopped until the check runs. Whether the check passes depends on whether Ubuntu has shipped an openssh update since the last cloud image. That's also why #2426 had to flip this same spec six weeks ago. Drop the running check for ssh and check that ssh.socket is enabled instead. That holds whether or not openssh gets upgraded during the build. Assisted-by: Claude Opus 5.5 --- audit-specs/baselines/qemu-build-amd64/service.yml | 3 ++- audit-specs/baselines/qemu-build-arm64/service.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/audit-specs/baselines/qemu-build-amd64/service.yml b/audit-specs/baselines/qemu-build-amd64/service.yml index 639d791837..21f8e463c5 100644 --- a/audit-specs/baselines/qemu-build-amd64/service.yml +++ b/audit-specs/baselines/qemu-build-amd64/service.yml @@ -41,7 +41,8 @@ service: running: true ssh: enabled: false - running: true + ssh.socket: + enabled: true atd: enabled: false running: false diff --git a/audit-specs/baselines/qemu-build-arm64/service.yml b/audit-specs/baselines/qemu-build-arm64/service.yml index 639d791837..21f8e463c5 100644 --- a/audit-specs/baselines/qemu-build-arm64/service.yml +++ b/audit-specs/baselines/qemu-build-arm64/service.yml @@ -41,7 +41,8 @@ service: running: true ssh: enabled: false - running: true + ssh.socket: + enabled: true atd: enabled: false running: false From 9483e59e4b8777b905e6bbb55c0d0f858a61ca14 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Fri, 16 Oct 2026 17:44:14 -0400 Subject: [PATCH 2/2] ci: move qemu images builds into nix-build Split publish into its own workflow, mimicing AMI Release Nix. We don't need the publish steps most of the time so better to avoid all the unnecessary stuff in this file. When we do want to publish compress the output with zstd since its nice and fast, compresses from ~5G to ~1G and should be fine. --- .github/workflows/actions.lock | 5 + .github/workflows/nix-build.yml | 7 +- .github/workflows/qemu-image-build.yml | 150 ++++++----------------- .github/workflows/qemu-image-release.yml | 129 +++++++++++++++++++ 4 files changed, 180 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/qemu-image-release.yml diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 0679b0fffd..514f85f9c3 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -84,6 +84,11 @@ workflows: - 'slackapi/slack-github-action@v4.0.0' '.github/workflows/qemu-image-build.yml': - 'actions/checkout@v7.0.1' + - 'actions/upload-artifact@v7.0.2' + - 'aws-actions/configure-aws-credentials@v6.3.0' + '.github/workflows/qemu-image-release.yml': + - 'actions/checkout@v7.0.1' + - 'actions/download-artifact@v8.0.2' - 'aws-actions/amazon-ecr-login@v2.1.7' - 'aws-actions/configure-aws-credentials@v6.3.0' - 'slackapi/slack-github-action@v4.0.0' diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index bba30d27be..4bc2e297fa 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -231,7 +231,7 @@ jobs: - checks-aarch64-darwin - checks-aarch64-linux - checks-x86_64-linux - # must always run, otherwise if any of the needs fail then this is skipped and PR is allowed through + # must always run, otherwise if any of the needs fail then this is skipped and PR is allowed through # not sure what GH's thinking is there that skipped == passed for the branch protection rule... # insert Mr Incredible meme -> pass is pass, skipped is not passed, PR should not merge if: always() @@ -264,6 +264,10 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + qemu: + needs: builds-aarch64-linux + uses: $/.github/workflows/qemu-image-build.yml + check: name: Nix CI ✅ if: always() @@ -272,6 +276,7 @@ jobs: - amis - checks - containers + - qemu - schema steps: - name: Check dependent job results diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index dfacd0aeb2..94524a29e9 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -1,16 +1,27 @@ name: Build QEMU image on: - push: - branches: - - develop - - release/* - paths: - - ansible/postgres_release.yml + workflow_call: + inputs: + arch: + description: Guest architecture (amd64 is experimental and publishes only to staging) + type: string + default: arm64 + publish: + description: Upload the built disk image as an artifact for the release workflow + type: boolean + default: false + outputs: + version: + description: Image tag for this build + value: ${{ jobs.build.outputs.version }} + artifact: + description: Name of the uploaded disk image artifact (empty unless publish was set) + value: ${{ jobs.build.outputs.artifact }} workflow_dispatch: inputs: arch: - description: 'Guest architecture (amd64 is experimental and publishes only to staging)' + description: Guest architecture (amd64 is experimental and publishes only to staging) type: choice default: arm64 options: @@ -19,63 +30,28 @@ on: permissions: contents: read - id-token: write jobs: - prepare: - runs-on: blacksmith-2vcpu-ubuntu-2404 - outputs: - postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} - steps: - - name: Checkout Repo - uses: actions/checkout@v7.0.1 - - - name: Set PostgreSQL versions - only builds pg17 atm - id: set-versions - run: | - VERSIONS=$(yq -e -o=json -I=0 '[.postgres_major[1]]' ansible/vars.yml) - echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT - build: - needs: prepare - strategy: - matrix: - postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} - arch: ["${{ inputs.arch || 'arm64' }}"] - runs-on: ${{ matrix.arch == 'amd64' && 'blacksmith-16vcpu-ubuntu-2404' || 'arm-native-runner' }} + runs-on: ${{ inputs.arch == 'amd64' && 'blacksmith-16vcpu-ubuntu-2404' || 'arm-native-runner' }} env: - TARGET_ARCH: ${{ matrix.arch }} + # only builds pg17 atm + POSTGRES_MAJOR_VERSION: "17" + TARGET_ARCH: ${{ inputs.arch || 'arm64' }} timeout-minutes: 150 + outputs: + version: ${{ steps.process_release_version.outputs.version }} + artifact: ${{ inputs.publish && format('qemu-image-{0}', inputs.arch || 'arm64') || '' }} permissions: - contents: write - packages: write - id-token: write + contents: read steps: - name: Checkout Repo uses: actions/checkout@v7.0.1 - - name: enable KVM support - run: | - sudo chown runner /dev/kvm - sudo chmod 666 /dev/kvm - - - name: Set PostgreSQL version environment variable - run: | - echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - - name: Install Nix uses: $/.github/actions/install-nix - - name: Run checks if triggered manually - if: ${{ github.event_name == 'workflow_dispatch' && matrix.arch == 'arm64' }} - run: | - SUFFIX=$(nix run nixpkgs#yq -- -e '.postgres_release.postgres${{ matrix.postgres_version }}' ansible/postgres_release.yml | sed -E 's/[0-9\.]+(.*)$/\1/') - if [[ -z $SUFFIX ]] ; then - echo "Version must include non-numeric characters if built manually." - exit 1 - fi - - name: Verify QEMU can use KVM shell: bash run: | @@ -99,81 +75,35 @@ jobs: - name: Build QEMU artifact env: GIT_SHA: ${{ steps.resolve-git-sha.outputs.sha }} - run: BUILD_QEMU_IMAGE_HW_VIRT_ONLY=1 nix run .#build-qemu-image "${{ matrix.postgres_version }}" "$TARGET_ARCH" + run: BUILD_QEMU_IMAGE_HW_VIRT_ONLY=1 nix run .#build-qemu-image "$POSTGRES_MAJOR_VERSION" "$TARGET_ARCH" - name: Grab release version id: process_release_version run: | - PG_VERSION=$(nix run nixpkgs#yq -- -er '.postgres_release.postgres${{ matrix.postgres_version }}' ansible/postgres_release.yml) + PG_VERSION=$(nix run nixpkgs#yq -- -er ".postgres_release.postgres$POSTGRES_MAJOR_VERSION" ansible/postgres_release.yml) if [[ "$TARGET_ARCH" == amd64 ]]; then PG_VERSION="$PG_VERSION-amd64-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" fi echo "version=$PG_VERSION" >> $GITHUB_OUTPUT - - name: configure aws credentials - staging - if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/') }} - uses: aws-actions/configure-aws-credentials@v6.3.0 - with: - role-to-assume: ${{ secrets.CONTROL_PLANE_DEV_ROLE }} - aws-region: "us-east-1" - - - name: Login to Amazon ECR Staging - if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/') }} - id: login-ecr-private-dev - uses: aws-actions/amazon-ecr-login@v2.1.7 - - name: Build image env: IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} run: | docker build --platform "linux/$TARGET_ARCH" -f Dockerfile-kubernetes -t "postgres:$IMAGE_TAG" packer-work-qemu-* - - name: Push docker image to Amazon ECR - if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/') }} - env: - REGISTRY: 812073016711.dkr.ecr.us-east-1.amazonaws.com - REPOSITORY: postgres-vm-image - IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} - run: | - docker tag "postgres:$IMAGE_TAG" "$REGISTRY/$REPOSITORY:$IMAGE_TAG" - docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG" - - # TODO (darora): temporarily also push to prod account from here - add a guard to only publish proper tagged releases to prod? - - name: configure aws credentials - prod - if: ${{ matrix.arch == 'arm64' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/')) }} - uses: aws-actions/configure-aws-credentials@v6.3.0 - with: - role-to-assume: ${{ secrets.CONTROL_PLANE_PROD_ROLE }} - aws-region: "us-east-1" - - - name: Login to Amazon ECR Prod - if: ${{ matrix.arch == 'arm64' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/')) }} - id: login-ecr-private-prod - uses: aws-actions/amazon-ecr-login@v2.1.7 - - - name: Push docker image to Amazon ECR Prod - if: ${{ matrix.arch == 'arm64' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/')) }} - env: - REGISTRY: 156470330064.dkr.ecr.us-east-1.amazonaws.com - REPOSITORY: postgres-vm-image - IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} + - name: Compress disk image + if: inputs.publish run: | - docker tag "postgres:$IMAGE_TAG" "$REGISTRY/$REPOSITORY:$IMAGE_TAG" - docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG" + nix run nixpkgs#zstd -- -T0 -o "$RUNNER_TEMP/packer-cloudimg.zst" packer-work-qemu-*/output-cloudimg/packer-cloudimg - - name: Slack Notification on Failure - if: ${{ failure() }} - uses: slackapi/slack-github-action@v4.0.0 + - name: Upload disk image + if: inputs.publish + uses: actions/upload-artifact@v7.0.2 with: - webhook: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} - webhook-type: incoming-webhook - payload: | - username: gha-failure - icon_emoji: ':old_man_yells_at_github_actions:' - channel: C0C16MJ8WHX - text: | - *Building Postgres QEMU artifact failed* - *Actions URL:* <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ github.workflow }}> - *Event:* ${{ github.event_name }} - *Ref:* ${{ github.ref }} - *Commit:* <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> + name: qemu-image-${{ env.TARGET_ARCH }} + path: ${{ runner.temp }}/packer-cloudimg.zst + # already zstd-compressed + compression-level: 0 + retention-days: 3 + if-no-files-found: error diff --git a/.github/workflows/qemu-image-release.yml b/.github/workflows/qemu-image-release.yml new file mode 100644 index 0000000000..55a0b1b200 --- /dev/null +++ b/.github/workflows/qemu-image-release.yml @@ -0,0 +1,129 @@ +name: Release QEMU image + +on: + push: + branches: + - develop + - release/* + paths: + - ansible/postgres_release.yml + workflow_dispatch: + inputs: + arch: + description: Guest architecture (amd64 is experimental and publishes only to staging) + type: choice + default: arm64 + options: + - arm64 + - amd64 + +permissions: + contents: read + +jobs: + build: + uses: $/.github/workflows/qemu-image-build.yml + with: + arch: ${{ inputs.arch || 'arm64' }} + publish: true + + publish: + needs: build + runs-on: ${{ inputs.arch == 'amd64' && 'blacksmith-16vcpu-ubuntu-2404' || 'arm-native-runner' }} + timeout-minutes: 60 + env: + POSTGRES_MAJOR_VERSION: "17" + TARGET_ARCH: ${{ inputs.arch || 'arm64' }} + IMAGE_TAG: ${{ needs.build.outputs.version }} + permissions: + contents: read + id-token: write + + steps: + - name: Checkout Repo + uses: actions/checkout@v7.0.1 + + - name: Install Nix + uses: $/.github/actions/install-nix + + - name: Run checks if publishing manually + if: github.event_name == 'workflow_dispatch' && env.TARGET_ARCH == 'arm64' + run: | + SUFFIX=$(nix run nixpkgs#yq -- -e ".postgres_release.postgres$POSTGRES_MAJOR_VERSION" ansible/postgres_release.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + if [[ -z $SUFFIX ]] ; then + echo "Version must include non-numeric characters if published manually." + exit 1 + fi + + - name: Download disk image + uses: actions/download-artifact@v8.0.2 + with: + name: ${{ needs.build.outputs.artifact }} + path: ${{ runner.temp }}/disk-image + + - name: Build image + run: | + mkdir -p image-context/output-cloudimg + nix run nixpkgs#zstd -- -d -o image-context/output-cloudimg/packer-cloudimg "$RUNNER_TEMP/disk-image/packer-cloudimg.zst" + docker build --platform "linux/$TARGET_ARCH" -f Dockerfile-kubernetes -t "postgres:$IMAGE_TAG" image-context + + - name: configure aws credentials - staging + uses: aws-actions/configure-aws-credentials@v6.3.0 + with: + role-to-assume: ${{ secrets.CONTROL_PLANE_DEV_ROLE }} + aws-region: "us-east-1" + + - name: Login to Amazon ECR Staging + id: login-ecr-private-dev + uses: aws-actions/amazon-ecr-login@v2.1.7 + + - name: Push docker image to Amazon ECR + env: + REGISTRY: 812073016711.dkr.ecr.us-east-1.amazonaws.com + REPOSITORY: postgres-vm-image + run: | + docker tag "postgres:$IMAGE_TAG" "$REGISTRY/$REPOSITORY:$IMAGE_TAG" + docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG" + + # TODO (darora): temporarily also push to prod account from here - add a guard to only publish proper tagged releases to prod? + - name: configure aws credentials - prod + if: env.TARGET_ARCH == 'arm64' + uses: aws-actions/configure-aws-credentials@v6.3.0 + with: + role-to-assume: ${{ secrets.CONTROL_PLANE_PROD_ROLE }} + aws-region: "us-east-1" + + - name: Login to Amazon ECR Prod + if: env.TARGET_ARCH == 'arm64' + id: login-ecr-private-prod + uses: aws-actions/amazon-ecr-login@v2.1.7 + + - name: Push docker image to Amazon ECR Prod + if: env.TARGET_ARCH == 'arm64' + env: + REGISTRY: 156470330064.dkr.ecr.us-east-1.amazonaws.com + REPOSITORY: postgres-vm-image + run: | + docker tag "postgres:$IMAGE_TAG" "$REGISTRY/$REPOSITORY:$IMAGE_TAG" + docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG" + + notify: + needs: [build, publish] + if: failure() + runs-on: blacksmith-2vcpu-ubuntu-2404 + steps: + - name: Slack Notification on Failure + uses: slackapi/slack-github-action@v4.0.0 + with: + webhook: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + webhook-type: incoming-webhook + payload: | + username: gha-failure + icon_emoji: ':old_man_yells_at_github_actions:' + channel: C0C16MJ8WHX + text: | + *Building Postgres QEMU artifact failed* + *Actions URL:* <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}/checks|${{ github.workflow }}> + *Event:* ${{ github.event_name }} + *Ref:* ${{ github.ref }} + *Commit:* <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>