diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 938a8dd..c0185a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,18 +1,41 @@ --- -# The repo has no runtime tests - nothing here talks to a box. What it does -# catch is DRIFT: a generated file that no longer matches its generator, a -# pinned upstream manifest that changed under its checksum, a playbook that -# stopped parsing, a pinset section the role does not know about. Every one of -# those has gone unnoticed here before, which is why this exists. -name: checks +# The repo's one workflow. Most of it never talks to a box: it catches DRIFT, +# a generated file that no longer matches its generator, a pinned upstream +# manifest that changed under its checksum, a playbook that stopped parsing, a +# pinset section the role does not know about. Every one of those has gone +# unnoticed here before, which is why this exists. +# +# The exception is platform.yml (Argo CD and the monitoring stack's firewall, +# see the README), the only playbook that runs from CI: +# +# pull request -> the checks, then a dry run (--check --diff) of +# platform.yml against the boxes if the PR touches it +# merge to main -> the checks, then, if the merge touches platform.yml, a +# dry run of the merged commit, then the real run, once a +# required reviewer has read the dry run and approved it in +# the `production` environment +# manual -> the checks, plus optionally a dry run, or a dry run and +# an apply, of platform.yml, limited to some boxes if wanted +# +# One workflow, so a merge starts one run. Pull requests from forks get no +# secrets: their platform.yml dry run is skipped. +name: ci on: push: branches: [main] pull_request: workflow_dispatch: - # platform.yml runs these first and only applies if they all pass. - workflow_call: + inputs: + platform: + description: "platform.yml: none, check (dry run) or apply" + type: choice + options: [none, check, apply] + default: none + limit: + description: "platform.yml: boxes to run on (ansible --limit), empty for all" + type: string + default: "" permissions: contents: read @@ -176,3 +199,122 @@ jobs: run: | sudo apt-get update && sudo apt-get install -y shellcheck shellcheck scripts/*.sh + + # --------------------------------------------------------------------------- + # platform.yml + # --------------------------------------------------------------------------- + + # Whether this run touches what platform.yml applies. On a pull request, + # against where the branch left main, so other merges to main do not count. + changes: + runs-on: ubuntu-latest + outputs: + platform: ${{ steps.diff.outputs.platform }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - id: diff + env: + EVENT: ${{ github.event_name }} + RANGE: >- + ${{ github.event_name == 'pull_request' + && format('{0}...{1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) + || format('{0}..{1}', github.event.before, github.sha) }} + run: | + set -euo pipefail + if [[ "$EVENT" == workflow_dispatch || "$RANGE" == 0000000000000000000000000000000000000000..* ]]; then + platform=true + elif git diff --name-only "$RANGE" | grep -qE \ + '^(platform\.yml|roles/(argocd|monitoring|ufw_allowlist)/|roles/route_origin/defaults/main\.yml|group_vars/|inventory/|scripts/ci-platform\.sh|\.github/workflows/ci\.yml)'; then + platform=true + else + platform=false + fi + echo "platform=$platform" | tee -a "$GITHUB_OUTPUT" + + # What platform.yml would change, for this exact commit: on a merge, this is + # what the approver reads before approving the apply below. It writes a + # summary on the run's page. Runs only once every check has passed. + platform-dry-run: + needs: [ansible, manifests, generated, shell, changes] + if: >- + needs.changes.outputs.platform == 'true' && ( + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + github.event_name == 'push' || + (github.event_name == 'workflow_dispatch' && inputs.platform != 'none')) + runs-on: ubuntu-latest + # On a pull request, a newer push to the same branch replaces its dry run, + # in a group of its own: GitHub keeps one pending run per group, so sharing + # the apply's group would let a pull request cancel an apply waiting for + # approval. On main (and manual runs) it queues with the apply in + # platform-main, so it never runs while an apply is changing the boxes. + concurrency: + group: ${{ github.event_name == 'pull_request' && format('platform-check-{0}', github.ref) || 'platform-main' }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + env: + # ansible.cfg sets the yaml callback, which no installed collection + # provides here (see the manifests job). + ANSIBLE_STDOUT_CALLBACK: default + ANSIBLE_FORCE_COLOR: "true" + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Ansible and the collections the roles use + run: | + pip install ansible-core + ansible-galaxy collection install community.general ansible.posix kubernetes.core + - name: Dry run platform.yml + env: + ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} + ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }} + CLOUDFLARE_DNS_TOKEN: ${{ secrets.CLOUDFLARE_DNS_TOKEN }} + ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }} + LIMIT: ${{ inputs.limit }} + run: scripts/ci-platform.sh check "$LIMIT" + + # Needs every check and the dry run of this same commit: a commit that fails + # either is never offered for approval. No status function in `if`, so + # GitHub adds success(). + platform-apply: + needs: [ansible, manifests, generated, shell, changes, platform-dry-run] + if: >- + needs.changes.outputs.platform == 'true' && ( + github.event_name == 'push' || + (github.event_name == 'workflow_dispatch' && inputs.platform == 'apply')) + runs-on: ubuntu-latest + # One at a time with main's dry runs, never cancelled half-way. A newer + # merge waiting behind replaces an older one still waiting for approval: + # the latest main wins, and its own dry run is the one the approver reads. + concurrency: + group: platform-main + cancel-in-progress: false + # Required reviewers: nothing reaches the boxes until one approves. + environment: production + env: + ANSIBLE_STDOUT_CALLBACK: default + ANSIBLE_FORCE_COLOR: "true" + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Ansible and the collections the roles use + run: | + pip install ansible-core + ansible-galaxy collection install community.general ansible.posix kubernetes.core + - name: Run platform.yml + env: + ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} + ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }} + CLOUDFLARE_DNS_TOKEN: ${{ secrets.CLOUDFLARE_DNS_TOKEN }} + ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }} + LIMIT: ${{ inputs.limit }} + run: scripts/ci-platform.sh apply "$LIMIT" diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml deleted file mode 100644 index 329860a..0000000 --- a/.github/workflows/platform.yml +++ /dev/null @@ -1,136 +0,0 @@ ---- -# platform.yml from CI: Argo CD's first install, registering the boxes with -# it, and the firewall rules the monitoring stack needs (README, "Argo CD and -# monitoring"). The service playbooks still run from an operator's machine. -# -# pull request -> dry run (--check --diff) against the boxes -# merge to main -> the repo checks (ci.yml), then, only if they all pass, -# the real run, once approved in the `production` -# environment (required reviewers) -# manual -> either, optionally limited to some boxes; apply also -# waits for the checks -# -# Pull requests from forks get no secrets and are skipped. -name: platform - -on: - pull_request: - paths: - - platform.yml - - roles/argocd/** - - roles/monitoring/** - - roles/ufw_allowlist/** - - roles/route_origin/defaults/main.yml - - group_vars/** - - inventory/** - - scripts/ci-platform.sh - - .github/workflows/platform.yml - push: - branches: [main] - paths: - - platform.yml - - roles/argocd/** - - roles/monitoring/** - - roles/ufw_allowlist/** - - roles/route_origin/defaults/main.yml - - group_vars/** - - inventory/** - - scripts/ci-platform.sh - - .github/workflows/platform.yml - workflow_dispatch: - inputs: - mode: - description: "check (dry run) or apply" - type: choice - options: [check, apply] - default: check - limit: - description: "Boxes to run on (ansible --limit), empty for all" - type: string - default: "" - -permissions: - contents: read - -env: - # ansible.cfg sets the yaml callback, which no installed collection provides - # here (see ci.yml). - ANSIBLE_STDOUT_CALLBACK: default - ANSIBLE_FORCE_COLOR: "true" - -jobs: - # The same checks as ci.yml (lint, syntax, manifests, generated files, - # shellcheck), for this commit. The apply needs them, so a commit that fails - # them is never offered for approval. They run a second time on a merge, - # alongside ci.yml's own run: jobs cannot wait on another workflow. - checks: - if: >- - (github.event_name == 'push') || - (github.event_name == 'workflow_dispatch' && inputs.mode == 'apply') - uses: ./.github/workflows/ci.yml - - check: - if: >- - (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || - (github.event_name == 'workflow_dispatch' && inputs.mode == 'check') - runs-on: ubuntu-latest - # A newer push to the same branch replaces its dry run. Kept apart from the - # apply's group: GitHub keeps one pending run per group, so a shared group - # let a pull request's dry run cancel an apply waiting for approval. - concurrency: - group: platform-check-${{ github.ref }} - cancel-in-progress: true - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install Ansible and the collections the roles use - run: | - pip install ansible-core - ansible-galaxy collection install community.general ansible.posix kubernetes.core - - name: Dry run platform.yml - env: - ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} - ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }} - CLOUDFLARE_DNS_TOKEN: ${{ secrets.CLOUDFLARE_DNS_TOKEN }} - ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }} - LIMIT: ${{ inputs.limit }} - run: scripts/ci-platform.sh check "$LIMIT" - - apply: - needs: checks - # No status function here, so GitHub adds success(): every job of the - # checks must have passed. - if: >- - (github.event_name == 'push') || - (github.event_name == 'workflow_dispatch' && inputs.mode == 'apply') - runs-on: ubuntu-latest - # One apply at a time, never cancelled half-way. A newer merge waiting - # behind it replaces an older one still waiting: the latest main wins. - concurrency: - group: platform-apply - cancel-in-progress: false - # Required reviewers: nothing reaches the boxes until one approves. - environment: production - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install Ansible and the collections the roles use - run: | - pip install ansible-core - ansible-galaxy collection install community.general ansible.posix kubernetes.core - - name: Run platform.yml - env: - ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} - ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }} - CLOUDFLARE_DNS_TOKEN: ${{ secrets.CLOUDFLARE_DNS_TOKEN }} - ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }} - LIMIT: ${{ inputs.limit }} - run: scripts/ci-platform.sh apply "$LIMIT" diff --git a/README.md b/README.md index 6399d70..1c3d6a7 100644 --- a/README.md +++ b/README.md @@ -1445,14 +1445,23 @@ for itself: ingest) from Cloudflare's edge and the other boxes, and 8000 on sing-1 (`healthchecks_host`, the hub's dead-man's switch) from lith-1. -**It runs from CI** (`.github/workflows/platform.yml`), the only playbook that -does; the others still run from an operator's machine: +**It runs from CI**, the only playbook that does; the others still run from an +operator's machine. It is part of the repo's one workflow +(`.github/workflows/ci.yml`), so a merge starts a single run: | When | What | |---|---| -| Pull request touching it | `--check --diff` against the boxes (forks get no secrets and are skipped) | -| Merge to `main` | the repo checks (`ci.yml`) for that commit, then, only if they all pass, the real run, once a required reviewer approves it in the `production` environment | -| Actions → platform → Run workflow | `check` or `apply` (apply also waits for the checks), optionally `--limit`ed | +| Pull request touching it | the checks, then `--check --diff` against the boxes (forks get no secrets and are skipped) | +| Merge to `main` touching it | the checks, then a dry run of the merged commit, then the real run, once a required reviewer has read the dry run and approved it in the `production` environment | +| Actions → ci → Run workflow | the checks, plus `platform`: `check` (dry run) or `apply` (dry run, then the approved run), optionally `--limit`ed | + +Each dry run writes what it would change, per box and task, on the run's +summary page; approve only if that is what you expect. + +"Touching it" means `platform.yml`, its roles (`argocd`, `monitoring`, +`ufw_allowlist`), `group_vars/`, `inventory/`, `scripts/ci-platform.sh` or the +workflow itself. A `group_vars` change for another service therefore also +offers an apply; with nothing of its own changed, it changes nothing. It restarts nothing that serves traffic, so it runs every box in one batch: the hub registers the others using what they produced earlier in the same run. @@ -1527,9 +1536,9 @@ same path a deploy takes), so what is checked is what would be applied: | every `pinset.yml` section is in the role's `known_sections` | a section the content role silently ignores | | `shellcheck scripts/*.sh` | the usual shell traps | -`platform.yml` also has its own workflow, which does talk to the boxes (see -"Argo CD and monitoring"). It calls this one (`ci.yml` is also a reusable -workflow) and only applies once every job here has passed. +The same workflow also runs `platform.yml`, the one part that does talk to +the boxes, and only after every check above has passed (see "Argo CD and +monitoring"). The Cloudflare range check is a separate weekly workflow rather than part of this one, because it reaches `api.cloudflare.com`: an outage there should not @@ -1692,13 +1701,12 @@ dns-libp2p.txt bootstrap.libp2p.io records for the legacy identities site.yml base preparation playbook k3s-upgrade.yml k3s upgrade, one minor version at a time, backup per step platform.yml Argo CD (first install) and the monitoring stack's prerequisites, run from CI -.github/workflows/platform.yml platform.yml: dry run on PRs, approved run on merge scripts/ci-platform.sh CI's wrapper for platform.yml: secrets to variables, SSH key, pinned host keys inventory/known_hosts the boxes' SSH host keys, pinned for CI keys/*.pub the SSH keys allowed in as the admin user: takgn, nymd, byo, break-glass, CI (admin_pubkey_file) routing.yml routing service playbook (needle + origin) bootstrap.yml bootstrap node playbook (with cert-manager) content.yml content cluster node playbook (deploy, load the pinset, wait for PINNED, summary) -.github/workflows/ci.yml drift checks: playbook syntax, lint, kustomize build, generated files, shellcheck +.github/workflows/ci.yml drift checks (playbook syntax, lint, kustomize build, generated files, shellcheck), then platform.yml: dry run on PRs, approved run on merge .github/workflows/cloudflare-ranges.yml weekly pinned-vs-published Cloudflare range check ``` diff --git a/group_vars/ipfs_nodes/main.yml b/group_vars/ipfs_nodes/main.yml index 60a556b..d176670 100644 --- a/group_vars/ipfs_nodes/main.yml +++ b/group_vars/ipfs_nodes/main.yml @@ -21,7 +21,7 @@ admin_pubkey_file: # only in 1Password (IPFS account, Compass vault); see AGENTS.md. Never # remove it without putting another emergency key in its place. - "{{ playbook_dir }}/keys/break-glass.pub" - # CI (.github/workflows/platform.yml). Its private key is the + # CI (.github/workflows/ci.yml, platform.yml). Its private key is the # ANSIBLE_SSH_PRIVATE_KEY repository secret. - "{{ playbook_dir }}/keys/github-actions.pub" diff --git a/scripts/ci-platform.sh b/scripts/ci-platform.sh index 28cc8b6..f30de51 100755 --- a/scripts/ci-platform.sh +++ b/scripts/ci-platform.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Run platform.yml from CI (.github/workflows/platform.yml). +# Run platform.yml from CI (the platform-* jobs of .github/workflows/ci.yml). # # scripts/ci-platform.sh check [limit] dry run: --check --diff # scripts/ci-platform.sh apply [limit] the real run @@ -51,4 +51,51 @@ args=(platform.yml -e "@$secrets") [[ $mode == check ]] && args+=(--check --diff) [[ -n $limit ]] && args+=(--limit "$limit") -ansible-playbook "${args[@]}" +if [[ $mode == apply || -z ${GITHUB_STEP_SUMMARY:-} ]]; then + ansible-playbook "${args[@]}" + exit +fi + +# A dry run in CI also writes what it would change to the run's summary page: +# on a merge, that is what the approver reads before approving the apply. +log=$(mktemp) +trap 'rm -f "$secrets" "$log"' EXIT +set +e +ansible-playbook "${args[@]}" 2>&1 | tee "$log" +status=${PIPESTATUS[0]} +set -e +python3 - "$log" "$status" >> "$GITHUB_STEP_SUMMARY" <<'PY' +import collections, re, sys +log, status = sys.argv[1], int(sys.argv[2]) +ansi = re.compile(r"\x1b\[[0-9;]*m") +lines = [ansi.sub("", l.rstrip("\n")) for l in open(log, errors="replace")] +task, changes, failures, recap, in_recap = "", collections.OrderedDict(), [], [], False +for line in lines: + if m := re.match(r"TASK \[(.*)\]", line): + task = m.group(1) + elif m := re.match(r"changed: \[([^\]]+)\]", line): + changes.setdefault((m.group(1).split(" -> ")[0], task), 0) + changes[(m.group(1).split(" -> ")[0], task)] += 1 + elif m := re.match(r"fatal: \[([^\]]+)\]", line): + failures.append(f"{m.group(1)}: {task}") + elif line.startswith("PLAY RECAP"): + in_recap = True + elif in_recap: + if not line.strip(): + in_recap = False + else: + recap.append(line) +print("### platform.yml dry run (`--check --diff`)\n") +if failures: + print("**Failed:** " + "; ".join(failures) + "\n") +if changes: + print("Would change (approve only if this is what you expect):\n") + print("| Box | Task | Changes |\n|---|---|---|") + for (host, name), n in changes.items(): + print(f"| {host} | {name} | {n} |") +elif status == 0: + print("Nothing would change on the boxes.") +print("\n
PLAY RECAP\n\n```\n" + "\n".join(recap) + "\n```\n
\n") +print("Full output, including the diffs: this job's log.") +PY +exit "$status"