diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0366fd..911fcc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: # others and nothing else would say so. - name: Syntax-check every playbook run: | - for playbook in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml; do + for playbook in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml platform.yml; do echo "== $playbook" ansible-playbook --syntax-check "$playbook" done diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml new file mode 100644 index 0000000..96ff9e7 --- /dev/null +++ b/.github/workflows/platform.yml @@ -0,0 +1,115 @@ +--- +# platform.yml from CI: Argo CD's first install, registering the boxes with +# it, and the firewall rules the monitoring stack needs (README, "Argo CD and +# monitoring"). The service playbooks still run from an operator's machine. +# +# pull request -> dry run (--check --diff) against the boxes +# merge to main -> the real run, once approved in the `production` +# environment (required reviewers) +# manual -> either, optionally limited to some boxes +# +# Pull requests from forks get no secrets and are skipped. +name: platform + +on: + pull_request: + paths: + - platform.yml + - roles/argocd/** + - roles/monitoring/** + - roles/ufw_allowlist/** + - roles/route_origin/defaults/main.yml + - group_vars/** + - inventory/** + - scripts/ci-platform.sh + - .github/workflows/platform.yml + push: + branches: [main] + paths: + - platform.yml + - roles/argocd/** + - roles/monitoring/** + - roles/ufw_allowlist/** + - roles/route_origin/defaults/main.yml + - group_vars/** + - inventory/** + - scripts/ci-platform.sh + - .github/workflows/platform.yml + workflow_dispatch: + inputs: + mode: + description: "check (dry run) or apply" + type: choice + options: [check, apply] + default: check + limit: + description: "Boxes to run on (ansible --limit), empty for all" + type: string + default: "" + +permissions: + contents: read + +# One run at a time against the boxes; never cancel one half-way. +concurrency: + group: platform + cancel-in-progress: false + +env: + # ansible.cfg sets the yaml callback, which no installed collection provides + # here (see ci.yml). + ANSIBLE_STDOUT_CALLBACK: default + ANSIBLE_FORCE_COLOR: "true" + +jobs: + check: + if: >- + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'workflow_dispatch' && inputs.mode == 'check') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Ansible and the collections the roles use + run: | + pip install ansible-core + ansible-galaxy collection install community.general ansible.posix kubernetes.core + - name: Dry run platform.yml + env: + ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} + ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }} + CLOUDFLARE_DNS_TOKEN: ${{ secrets.CLOUDFLARE_DNS_TOKEN }} + ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }} + LIMIT: ${{ inputs.limit }} + run: scripts/ci-platform.sh check "$LIMIT" + + apply: + if: >- + (github.event_name == 'push') || + (github.event_name == 'workflow_dispatch' && inputs.mode == 'apply') + runs-on: ubuntu-latest + # Required reviewers: nothing reaches the boxes until one approves. + environment: production + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Ansible and the collections the roles use + run: | + pip install ansible-core + ansible-galaxy collection install community.general ansible.posix kubernetes.core + - name: Run platform.yml + env: + ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} + ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }} + CLOUDFLARE_DNS_TOKEN: ${{ secrets.CLOUDFLARE_DNS_TOKEN }} + ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }} + LIMIT: ${{ inputs.limit }} + run: scripts/ci-platform.sh apply "$LIMIT" diff --git a/AGENTS.md b/AGENTS.md index 473e379..caa43a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,8 +12,33 @@ three at once is a full outage of that service. ## Getting in -SSH is key-only as the hardened admin user `ipni` (`~/.ssh/id_ed25519`); root -login is disabled and the provider KVM console is the out-of-band fallback. +SSH is key-only as the hardened admin user `ipni`; root login is disabled and +the provider KVM console is the out-of-band fallback. The keys that may sign in +are the files in `keys/`, listed in `admin_pubkey_file` (takgn, nymd, byo, the +break-glass key and CI's own key); the hardening role removes any other. + +### Break-glass access + +For when no operator key works (lost laptop, someone away, a bad hardening +run). The private key exists only in 1Password: IPFS account, Compass vault, +item "IPNI boxes: break-glass SSH key (ipni user)", ID +`4ghgjny7l4rawld4spedu7mxz4`. The reference below uses the ID: the title's `:` +is not allowed in a 1Password reference, and the ID survives a rename. Use it +without writing it to disk, through a throwaway agent: + +```bash +eval "$(ssh-agent -s)" +op read --account team-ipfs.1password.com \ + "op://Compass/4ghgjny7l4rawld4spedu7mxz4/private key?ssh-format=openssh" | ssh-add - +ssh -o IdentitiesOnly=yes -o IdentityFile=keys/break-glass.pub ipni@46.166.169.131 +ssh-agent -k +``` + +Then fix whatever locked people out, through the repo, not by hand. If the key +has been used outside a real emergency or may have leaked, replace it: a new +SSH Key item in Compass, its public half in `keys/break-glass.pub`, and a +hardening run. Last resort if SSH itself is down: the provider's KVM console, +with `vault_console_password`. The playbooks set `remote_user: "{{ admin_user }}"` themselves, so `ansible-playbook` needs no user flag. **Ad-hoc `ansible` commands do not**, and @@ -33,7 +58,9 @@ controller; in CI it is `echo ci > .vault_pass`. ## kubectl The k3s API server (6443) is deliberately closed to the internet — ufw denies -it and there is no flag to open it. Two ways in: +it and there is no flag to open it. The one exception is Argo CD on lith-1 +(`argocd_hub`), which `platform.yml` lets reach the other boxes' 6443 from +lith-1's address only. Two ways in for people: ```bash # 1. straight through ssh, no setup @@ -162,7 +189,7 @@ deliberately disables `line-length` and `braces` (see `.ansible-lint`). A pass looks like `Passed: 0 failure(s), 0 warning(s) ... profile ... 'production'`. ```bash -for pb in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml; do +for pb in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml platform.yml; do ansible-playbook --syntax-check "$pb" done ANSIBLE_STDOUT_CALLBACK=default ansible-playbook scripts/render-manifests.yml -e dest=rendered diff --git a/README.md b/README.md index b38b1aa..4cb292c 100644 --- a/README.md +++ b/README.md @@ -111,16 +111,18 @@ with the OS disk. > `/data` existed would otherwise make the in-use data disk look blank. Once a > volume labelled `ipfsdata` exists, the format step is skipped outright. -**`hardening`** — creates the `ipni` admin user with your ed25519 key and +**`hardening`** — creates the `ipni` admin user with the operators' keys and passwordless sudo, then locks SSH down to key-only with root login disabled, enables ufw with a default-deny inbound policy, and configures fail2ban. > **The key files are the single source of truth.** `authorized_key` runs with > `exclusive: true`, so any key added to the `ipni` account by hand is removed -> on the next run. `admin_pubkey_file` in `group_vars` takes either one path or -> a list of them; to give a second operator access, add their key file to that -> list rather than editing the host. Do it **before** the run that would -> otherwise strip their key. +> on the next run. `admin_pubkey_file` in `group_vars` lists committed public +> key files in `keys/` (`takgn.pub`, `nymd.pub`, `byo.pub`, the 1Password-only +> `break-glass.pub`, and CI's `github-actions.pub`), so the result does not +> depend on who runs it. To give someone access, commit +> their public key there and add it to the list rather than editing the host. +> Do it **before** the run that would otherwise strip their key. fail2ban is kept mainly to cut log noise: with password authentication disabled there are few credential guesses left for it to ban. @@ -1421,6 +1423,59 @@ pending entries first, then entries with a CID the cluster does not hold, then the pinned entries with per-peer status and DNSLink drift, followed by the raw `ipfs-cluster-ctl pin ls` and `status`. +## Argo CD and monitoring + +Monitoring (Prometheus, Loki, Alertmanager, Grafana, Alloy on every box) lives +in [ipni/infra](https://github.com/ipni/infra) and is deployed by Argo CD, +which runs on `argocd_hub` (lith-1). This repo does only what Argo CD cannot do +for itself: + +* **First install of Argo CD**, with the same Helm chart and release name + ipni/infra uses, plus the root Application that hands it over. After that + Argo CD manages its own release from ipni/infra; `platform.yml` never touches + it again, so upgrades and settings go there. +* **Registration**: on each other box, a `cluster-admin` service account for + Argo CD, and 6443 opened to lith-1's address only (the one exception to the + closed API server). lith-1 therefore holds an admin token for every box. +* **Secrets that cannot be in Git**: Argo CD's age key, which decrypts the + SOPS-encrypted Secrets in ipni/infra (`vault_argocd_age_key`), the deploy key + for ipni/infra while it is private (`vault_argocd_repo_ssh_key`, optional), + and the Cloudflare DNS token for the monitoring certificate. +* **Firewall**: 8443 on lith-1 (Traefik for Grafana, Argo CD and metrics + ingest) from Cloudflare's edge and the other boxes, and 8000 on sing-1 + (`healthchecks_host`, the hub's dead-man's switch) from lith-1. + +**It runs from CI** (`.github/workflows/platform.yml`), the only playbook that +does; the others still run from an operator's machine: + +| When | What | +|---|---| +| Pull request touching it | `--check --diff` against the boxes (forks get no secrets and are skipped) | +| Merge to `main` | the real run, once a required reviewer approves it in the `production` environment | +| Actions → platform → Run workflow | `check` or `apply`, optionally `--limit`ed | + +It restarts nothing that serves traffic, so it runs every box in one batch: the +hub registers the others using what they produced earlier in the same run. + +CI has no vault files, so `platform.yml`'s vault variables come from +repository secrets (`scripts/ci-platform.sh` maps them), and it signs in with +its own SSH key, `keys/github-actions.pub` in `admin_pubkey_file`. It only +connects to the host keys pinned in `inventory/known_hosts`. + +| Secret | Becomes | +|---|---| +| `ANSIBLE_SSH_PRIVATE_KEY` | CI's SSH key as the admin user | +| `ARGOCD_AGE_KEY` | `vault_argocd_age_key`: Argo CD's age key (public half in ipni/infra's `.sops.yaml`) | +| `CLOUDFLARE_DNS_TOKEN` | `vault_cloudflare_dns_token`: the same token cert-manager uses for WSS | +| `ARGOCD_REPO_SSH_KEY` | `vault_argocd_repo_ssh_key`: optional, only while ipni/infra is private | + +Copies of all of them are in 1Password (IPFS account, Compass vault): GitHub +cannot show them again. To revoke CI's access to the boxes, remove its line +from `admin_pubkey_file` and re-run the hardening role. + +From an operator's machine it still works as usual, with the same variables +in the vault: `ansible-playbook platform.yml --check --diff`. + ## Secrets **This repository is public, so no credentials are committed, not even @@ -1434,7 +1489,7 @@ DNS. | File | Variables | |------|-----------| | `host_vars//vault.yml` | `vault_root_password`, `vault_console_password` (provider console), `vault_bootstrap_privkey`, `vault_bootstrap_api_token`, `vault_content_cluster_privkey`, `vault_content_kubo_privkey`; on chic-1 also `vault_bootstrap_b_privkey`, `vault_bootstrap_b_api_token` | -| `group_vars/ipfs_nodes/vault.yml` | `vault_route_origin_tls_key`, `vault_origin_pull_ca_key`, `vault_origin_pull_client_key`, `vault_cloudflare_dns_token`, `vault_content_cluster_secret`, `vault_content_pin_users` (remote pinning users, see "Remote pinning") | +| `group_vars/ipfs_nodes/vault.yml` | `vault_route_origin_tls_key`, `vault_origin_pull_ca_key`, `vault_origin_pull_client_key`, `vault_cloudflare_dns_token`, `vault_content_cluster_secret`, `vault_content_pin_users` (remote pinning users, see "Remote pinning"), `vault_argocd_age_key` and optionally `vault_argocd_repo_ssh_key` (see "Argo CD and monitoring") | The per-box files were generated from a plaintext `servers.txt` by `scripts/bootstrap-vault.sh`; that file was never committed and has since been @@ -1462,7 +1517,7 @@ same path a deploy takes), so what is checked is what would be applied: | Check | Catches | | --- | --- | -| `ansible-playbook --syntax-check` on all five playbooks | a role edit that breaks a playbook nobody ran | +| `ansible-playbook --syntax-check` on all six playbooks | a role edit that breaks a playbook nobody ran | | `ansible-lint`, `yamllint` | the rest, minus the style rules in `.ansible-lint` | | `kubectl kustomize` on every rendered `k8s/*/` | a patch or digest pin that no longer applies | | rendering at `needle_instances` 1, 2 and 4 | the Deployments and Envoy's endpoint list falling out of step | @@ -1472,6 +1527,9 @@ same path a deploy takes), so what is checked is what would be applied: | every `pinset.yml` section is in the role's `known_sections` | a section the content role silently ignores | | `shellcheck scripts/*.sh` | the usual shell traps | +`platform.yml` also has its own workflow, which does talk to the boxes (see +"Argo CD and monitoring"). + The Cloudflare range check is a separate weekly workflow rather than part of this one, because it reaches `api.cloudflare.com`: an outage there should not turn an unrelated pull request red. @@ -1567,7 +1625,8 @@ turn an unrelated pull request red. are pinned, and compare `Provide.Strategy` `roots` against `pinned`. - [ ] Record which HTTP gateways serve the website to browsers without an IPFS client, who operates them, and whether they continue after 2026-09-30. -- [ ] Set up monitoring/alerting, and scrape `/debug/metrics/prometheus`. Include +- [ ] Set up monitoring/alerting (in progress: `platform.yml` here, the stack in + ipni/infra), and scrape `/debug/metrics/prometheus`. Include `/data` usage: the bootstrappers' DHT record store grows with traffic, the content repos grow with every pin on every box, and local-path enforces neither the 20 GiB nor the 100 GiB claim. @@ -1587,6 +1646,9 @@ roles/needle/ needle firewall + deploy roles/route_origin/ Envoy origin: cert preflight, Cloudflare allowlist, TLS secret tasks/verify.yml post-deploy request checks (paths, Host, traversal; AOP when enforced) roles/kustomize_apply/ shared: ship, dry-run/apply, wait, prune stale ConfigMaps +roles/ufw_allowlist/ shared: open a port to an exact source list, removing rules for dropped sources +roles/argocd/ Argo CD's first install on argocd_hub, cluster registration, age key, 6443 to the hub +roles/monitoring/ firewall and Cloudflare token for the monitoring stack in ipni/infra k8s/needle/ needle kustomize manifests; the Deployments are .j2, rendered from needle_all_instances k8s/route-origin/ Envoy kustomize manifests and envoy.yaml.j2 (needle endpoints rendered) k8s/bootstrap/ kubo bootstrapper manifests (deployment, repo PVC) @@ -1628,6 +1690,11 @@ dns.txt Cloudflare-importable bootstrap DNS records (generated dns-libp2p.txt bootstrap.libp2p.io records for the legacy identities (generated) site.yml base preparation playbook k3s-upgrade.yml k3s upgrade, one minor version at a time, backup per step +platform.yml Argo CD (first install) and the monitoring stack's prerequisites, run from CI +.github/workflows/platform.yml platform.yml: dry run on PRs, approved run on merge +scripts/ci-platform.sh CI's wrapper for platform.yml: secrets to variables, SSH key, pinned host keys +inventory/known_hosts the boxes' SSH host keys, pinned for CI +keys/*.pub the SSH keys allowed in as the admin user: takgn, nymd, byo, break-glass, CI (admin_pubkey_file) routing.yml routing service playbook (needle + origin) bootstrap.yml bootstrap node playbook (with cert-manager) content.yml content cluster node playbook (deploy, load the pinset, wait for PINNED, summary) diff --git a/group_vars/ipfs_nodes/main.yml b/group_vars/ipfs_nodes/main.yml index 6d89729..60a556b 100644 --- a/group_vars/ipfs_nodes/main.yml +++ b/group_vars/ipfs_nodes/main.yml @@ -6,15 +6,24 @@ # the ONLY way in over the network once hardening completes; the provider # KVM/web console is the designated out-of-band fallback. # -# One path, or a list of them when more than one operator needs in. The role -# installs them exclusively, so a key that is not listed here does not survive -# the next run - add the second operator BEFORE the run that would lock them -# out, not after: -# admin_pubkey_file: -# - ~/.ssh/id_ed25519.pub -# - ~/.ssh/operators/molly.pub +# The role installs these keys exclusively: a key that is not listed here is +# REMOVED on the next run. So every key is a committed public key file in +# keys/, never "whoever runs the playbook" (~/.ssh/id_ed25519.pub), which would +# lock out every other operator. To add someone, commit their public key here +# BEFORE the run that would lock them out; to remove someone, delete their line +# and re-run the hardening role (site.yml --tags hardening, one box at a time). admin_user: ipni -admin_pubkey_file: ~/.ssh/id_ed25519.pub +admin_pubkey_file: + - "{{ playbook_dir }}/keys/takgn.pub" + - "{{ playbook_dir }}/keys/nymd.pub" + - "{{ playbook_dir }}/keys/byo.pub" + # Break-glass: only for when no operator key works. Its private half exists + # only in 1Password (IPFS account, Compass vault); see AGENTS.md. Never + # remove it without putting another emergency key in its place. + - "{{ playbook_dir }}/keys/break-glass.pub" + # CI (.github/workflows/platform.yml). Its private key is the + # ANSIBLE_SSH_PRIVATE_KEY repository secret. + - "{{ playbook_dir }}/keys/github-actions.pub" # --------------------------------------------------------------------------- # SSH hardening @@ -33,8 +42,9 @@ ssh_allow_users: "{{ admin_user }}" # # The k3s API server (6443) is deliberately absent from this list: it stays # closed to the internet and remote kubectl goes over an SSH tunnel (see -# scripts/kubectl-tunnel.sh). This list is the only switch - there is no -# separate "expose the API" flag. +# scripts/kubectl-tunnel.sh). Do not add it here - this list opens a port to +# every source. The one exception is narrower: roles/argocd opens 6443 on the +# other boxes to argocd_hub's address only, for Argo CD (see below). firewall_allowed_tcp: - "{{ ssh_port }}" firewall_allowed_udp: [] @@ -124,6 +134,7 @@ sysctl_tuning: # 5011 bootstrapper API 5012 bootstrap-b API # 5021 content kubo RPC API (loopback) # 6443-6444 k3s API 8190-8193 needle APIs (loopback, four instances) + # 8000 Healthchecks (healthchecks_host only) 8443 monitoring Traefik (argocd_hub only) # 8472/udp flannel VXLAN # 9094 content ipfs-cluster REST API (loopback) # 9095 content ipfs-cluster IPFS proxy (loopback) @@ -133,7 +144,7 @@ sysctl_tuning: # 9901 route-origin Envoy admin 9902 bootstrap-wss Envoy admin # 9903 bootstrap-b's bootstrap-wss Envoy admin # 10010 containerd 10248-10259 kubelet/kube-proxy/k3s - net.ipv4.ip_local_reserved_ports: "443,4001-4002,4004-4007,4011-4012,4101,4443,4453,5011-5012,5021,6443-6444,8190-8193,8472,9094-9097,9443,9901-9904,10010,10248-10259" + net.ipv4.ip_local_reserved_ports: "443,4001-4002,4004-4007,4011-4012,4101,4443,4453,5011-5012,5021,6443-6444,8000,8190-8193,8443,8472,9094-9097,9443,9901-9904,10010,10248-10259" net.ipv4.tcp_fin_timeout: 15 # Must equal what k3s's kube-proxy sets on every start: max(32768 * CPUs, # 131072), i.e. 1048576 on these 32-core boxes. A fixed lower value both @@ -170,3 +181,23 @@ bootstrap_instances: >- {'host': inventory_hostname ~ bootstrap_instance_table[i.name].suffix})) -%} {%- endfor -%} {{ out }} + +# --------------------------------------------------------------------------- +# Argo CD and monitoring (platform.yml) +# --------------------------------------------------------------------------- +# One Argo CD, on argocd_hub, deploys the monitoring stack in ipni/infra to +# every box. It reaches the other boxes' API servers directly, so roles/argocd +# opens 6443 on them to argocd_hub's address and nothing else. The trade-off: +# argocd_hub holds a cluster-admin token for every box, so a compromise of it +# reaches all of them. The monitoring hub (Prometheus, Loki, Alertmanager, +# Grafana) runs on the same box, behind Traefik on :8443. +argocd_hub: lith-1 + +# Runs Healthchecks, the dead-man's switch for the hub: Alertmanager on +# argocd_hub pings it every minute, and it alerts when the pings stop. Must not +# be argocd_hub, or it would go down with the thing it watches. +healthchecks_host: sing-1 + +# The name each box has in Argo CD, and so in the `cluster` label on every +# metric and log line. ipni/infra refers to the boxes by these names. +argocd_cluster_name: "{{ inventory_hostname | replace('-', '') }}" diff --git a/inventory/known_hosts b/inventory/known_hosts new file mode 100644 index 0000000..7b7a4ff --- /dev/null +++ b/inventory/known_hosts @@ -0,0 +1,8 @@ +# SSH host keys of the boxes, for CI (.github/workflows/platform.yml), which +# must refuse to connect to anything else. Public. Checked on 2026-10-12: the +# scanned key, the key file on each box and an operator's known_hosts agreed. +# Update when a box is reinstalled: ssh-keyscan -t ed25519 , then confirm +# against /etc/ssh/ssh_host_ed25519_key.pub on the box itself. +5.199.165.77 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK2+9yvOjU3tE3K7zEVHn5M30X3FHRA6mgkSvFjmH2uL +46.166.169.131 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBbz1MWGYfe2RzaH/SV+zBctwhUNWO0336K5N5x5ATOp +84.32.48.211 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGsaCZG1aU29AqxU84RhnxlMdYyvzz1f2hBhpaYu2gHR diff --git a/keys/break-glass.pub b/keys/break-glass.pub new file mode 100644 index 0000000..074ba85 --- /dev/null +++ b/keys/break-glass.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINnPc7QVK8i7IdZpmF3Mup9rsCoaGWjlR9bqyubu06ni break-glass (1Password: IPFS / Compass) diff --git a/keys/byo.pub b/keys/byo.pub new file mode 100644 index 0000000..b17eb1b --- /dev/null +++ b/keys/byo.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC3DF4HILnhYWbTZb0XdD7R8STH+bnTzxU9HartE4M2L byo (hardware key) diff --git a/keys/github-actions.pub b/keys/github-actions.pub new file mode 100644 index 0000000..a6e27d5 --- /dev/null +++ b/keys/github-actions.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKLL+jM3SN0N+MtLKcpzBQZ+2RDHyJoibiuvZsD5CWRm github-actions@ipni/workers (platform.yml) diff --git a/keys/nymd.pub b/keys/nymd.pub new file mode 100644 index 0000000..30b14a5 --- /dev/null +++ b/keys/nymd.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOXuNpM0vZTW1r1LcvunW5JURjaMOiku2AZZjgzF61s2 nymd diff --git a/keys/takgn.pub b/keys/takgn.pub new file mode 100644 index 0000000..02240f1 --- /dev/null +++ b/keys/takgn.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO6RgP8GfmKttUW2fTwEeBD6ObNGp5KX6Nmor1imMuyH takgn diff --git a/platform.yml b/platform.yml new file mode 100644 index 0000000..f8c67f0 --- /dev/null +++ b/platform.yml @@ -0,0 +1,34 @@ +--- +# Argo CD's first install on the hub box, the Secrets it needs that cannot live +# in Git, and what the monitoring stack it deploys needs from the boxes: +# firewall openings and the Cloudflare token for its certificate. After the +# first install Argo CD manages itself, and everything else it deploys, from +# ipni/infra; see docs/03-monitoring.md there. +# +# ansible-playbook platform.yml # all three boxes +# ansible-playbook platform.yml --check --diff # dry run +# +# Requires site.yml to have run first (admin user, k3s, ufw, reserved ports), +# and bootstrap.yml (cert-manager) for the monitoring certificate. +# +# Nothing here restarts a running service: it adds firewall rules and Secrets, +# and installs Argo CD on argocd_hub if it is not there yet. So unlike the service +# playbooks there is no production_rollout batching - and there must not be: +# the hub registers the other boxes using what they produced earlier in this +# same play, so every box has to be in one batch. With -l, the hub registers +# only the boxes in the run, and leaves the others as an earlier run left them. + +- name: Argo CD and the monitoring stack's prerequisites + hosts: ipfs_nodes + remote_user: "{{ admin_user }}" + become: true + gather_facts: false + # route_origin_cloudflare_ranges: the pinned list the 8443 allowlist reuses + # (role defaults are not visible outside the role's play). + vars_files: + - roles/route_origin/defaults/main.yml + roles: + - role: argocd + tags: [argocd] + - role: monitoring + tags: [monitoring] diff --git a/roles/argocd/defaults/main.yml b/roles/argocd/defaults/main.yml new file mode 100644 index 0000000..49b4854 --- /dev/null +++ b/roles/argocd/defaults/main.yml @@ -0,0 +1,26 @@ +--- +# Argo CD's first install. After it, Argo CD manages itself from ipni/infra +# (kubernetes/argocd/apps/argocd.yaml), which is where upgrades and settings +# change. This role never touches the release again once it exists, so keep +# the chart version here in step with that file only for new boxes. +argocd_chart_repo: https://argoproj.github.io/argo-helm +argocd_chart_version: 10.10.0 + +# Helm, only for that first install, kept out of PATH. +argocd_helm_version: v3.22.0 +argocd_helm_sha256: 1e4ab49e429626cf6c6958d914248b78c9730803c2751b87627e171dc800e7bb +argocd_helm_dir: "/usr/local/lib/ipfs-workers/helm-{{ argocd_helm_version }}" + +# Where Argo CD deploys from: the root Application created on first install +# points here, and ipni/infra takes it from there. +argocd_repo_url: git@github.com:ipni/infra.git +argocd_repo_revision: main +argocd_root_path: kubernetes/argocd/apps + +# While argocd_repo_url is private, Argo CD reads it with a deploy key: set +# vault_argocd_repo_ssh_key in group_vars/ipfs_nodes/vault.yml. Once the repo +# is public, leave it unset and Argo CD reads it anonymously. +# +# vault_argocd_age_key (same file) is Argo CD's age private key, which decrypts +# the SOPS-encrypted Secrets in ipni/infra. Its public half is in that repo's +# .sops.yaml. diff --git a/roles/argocd/tasks/hub.yml b/roles/argocd/tasks/hub.yml new file mode 100644 index 0000000..37f6964 --- /dev/null +++ b/roles/argocd/tasks/hub.yml @@ -0,0 +1,214 @@ +--- +# Argo CD's first install, and the Secrets it needs that cannot live in Git. +# +# Ansible installs Argo CD ONCE, with the same Helm chart and release name +# ipni/infra uses. From then on Argo CD manages its own release from that repo +# and this role leaves it alone: upgrades and settings go through ipni/infra. +# The Secrets below are refreshed on every run; Argo CD does not manage them. + +- name: Fail if there is no age key for Argo CD + ansible.builtin.assert: + that: vault_argocd_age_key is defined and vault_argocd_age_key is search('AGE-SECRET-KEY-') + fail_msg: >- + vault_argocd_age_key is not set (group_vars/ipfs_nodes/vault.yml). Argo CD + needs it to decrypt the SOPS-encrypted Secrets in ipni/infra; see the README. + quiet: true + +- name: Check whether Argo CD is installed + ansible.builtin.command: + argv: [k3s, kubectl, --namespace=argocd, get, deployment/argocd-server, --ignore-not-found, -o, name] + register: argocd_installed + changed_when: false + check_mode: false + +# Under --check there is nothing to dry-run against yet: no Helm on the box, and +# no Argo CD CRDs for the root Application. Say so instead. +# The argocd namespace only exists once the Helm release does, so the Secrets +# below cannot be dry-run either (the API server rejects them: namespace not +# found). Say so instead, like route_origin's TLS secret. +- name: Note that a first install is not dry-run + ansible.builtin.debug: + msg: >- + Argo CD is not installed; a real run would install chart {{ argocd_chart_version }}, + then its age key, the cluster and repository Secrets, and the root Application. + when: + - ansible_check_mode + - not argocd_installed.stdout + +- name: Install Argo CD for the first time + when: + - not ansible_check_mode + - not argocd_installed.stdout + block: + - name: Create the Helm directory + ansible.builtin.file: + path: "{{ argocd_helm_dir }}" + state: directory + owner: root + group: root + mode: "0755" + + - name: Download Helm, checked against its pinned checksum + ansible.builtin.get_url: + url: "https://get.helm.sh/helm-{{ argocd_helm_version }}-linux-amd64.tar.gz" + dest: "{{ argocd_helm_dir }}/helm.tar.gz" + checksum: "sha256:{{ argocd_helm_sha256 }}" + mode: "0644" + + - name: Unpack the Helm binary + ansible.builtin.unarchive: + src: "{{ argocd_helm_dir }}/helm.tar.gz" + remote_src: true + dest: "{{ argocd_helm_dir }}" + extra_opts: [--strip-components=1, linux-amd64/helm] + creates: "{{ argocd_helm_dir }}/helm" + + # Enough to start. The first sync of the argocd Application in ipni/infra + # replaces these with that repo's values (SOPS decryption, resources, ...). + - name: Install the Argo CD Helm release + kubernetes.core.helm: + binary_path: "{{ argocd_helm_dir }}/helm" + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: argocd + chart_repo_url: "{{ argocd_chart_repo }}" + chart_ref: argo-cd + chart_version: "{{ argocd_chart_version }}" + release_namespace: argocd + create_namespace: true + wait: true + wait_timeout: 10m + values: + global: + domain: argocd.ipni.io + configs: + params: + server.insecure: true + dex: + enabled: false + +# Read by repo-server to decrypt the SOPS-encrypted Secrets in ipni/infra; its +# values (kubernetes/argocd/values.yaml) mount it. Refreshed on every run, so +# rotating the key is: replace it in the vault, re-encrypt, re-run. +- name: Install Argo CD's age key + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: argocd-sops-age + namespace: argocd + data: + keys.txt: "{{ vault_argocd_age_key | b64encode }}" + when: not (ansible_check_mode and not argocd_installed.stdout) + no_log: true + +# One Secret per box, named after the box. ipni/infra targets boxes by these +# names, and its ApplicationSets pick up every box labelled ipni.io/managed. +# The hub talks to its own API server in-cluster. A box outside this run (-l) +# keeps the Secret an earlier run left; none is ever deleted here. +- name: List the boxes to register + ansible.builtin.set_fact: + argocd_clusters: >- + {%- set out = [{'name': argocd_cluster_name, + 'server': 'https://kubernetes.default.svc', + 'config': {'tlsClientConfig': {'insecure': false}}}] -%} + {%- for h in ansible_play_hosts if h != argocd_hub and hostvars[h].argocd_manager_token is defined -%} + {%- set _ = out.append({'name': hostvars[h].argocd_cluster_name, + 'server': 'https://' ~ hostvars[h].ansible_host ~ ':6443', + 'config': {'bearerToken': hostvars[h].argocd_manager_token, + 'tlsClientConfig': {'insecure': false, + 'caData': hostvars[h].argocd_manager_ca}}}) -%} + {%- endfor -%} + {{ out }} + no_log: true + +- name: Register the boxes with Argo CD + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: "cluster-{{ item.name }}" + namespace: argocd + labels: + argocd.argoproj.io/secret-type: cluster + ipni.io/managed: "true" + data: + name: "{{ item.name | b64encode }}" + server: "{{ item.server | b64encode }}" + config: "{{ item.config | to_json | b64encode }}" + loop: "{{ argocd_clusters }}" + loop_control: + label: "{{ item.name }} {{ item.server }}" + when: not (ansible_check_mode and not argocd_installed.stdout) + no_log: true + +- name: Give Argo CD read access to the infra repository + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: repo-infra + namespace: argocd + labels: + argocd.argoproj.io/secret-type: repository + data: + type: "{{ 'git' | b64encode }}" + url: "{{ argocd_repo_url | b64encode }}" + sshPrivateKey: "{{ vault_argocd_repo_ssh_key | b64encode }}" + when: + - vault_argocd_repo_ssh_key is defined + - not (ansible_check_mode and not argocd_installed.stdout) + no_log: true + +# The hand-off: the root Application points Argo CD at ipni/infra, whose apps +# include Argo CD's own release. Created once, like the release; after that it +# is ipni/infra's to change (kubernetes/argocd/root-app.yaml). +- name: Point Argo CD at ipni/infra + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: argoproj.io/v1alpha1 + kind: Application + metadata: + name: root + namespace: argocd + spec: + project: default + source: + repoURL: "{{ argocd_repo_url }}" + targetRevision: "{{ argocd_repo_revision }}" + path: "{{ argocd_root_path }}" + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true + when: + - not ansible_check_mode + - not argocd_installed.stdout + +- name: Report what Argo CD manages + ansible.builtin.debug: + msg: + - "{{ 'installed Argo CD ' ~ argocd_chart_version ~ ' and handed it to ' ~ argocd_repo_url + if not argocd_installed.stdout else + 'Argo CD already installed; it manages itself from ' ~ argocd_repo_url ~ ', so its release was left alone' }}" + - "registered: {{ argocd_clusters | map(attribute='name') | join(', ') }}" + - "repository access: {{ 'deploy key from the vault' if vault_argocd_repo_ssh_key is defined + else 'anonymous, so ' ~ argocd_repo_url ~ ' must be public' }}" + - "UI: https://argocd.ipni.io once ipni/infra's monitoring stack serves it. Before that, on the box: + sudo k3s kubectl -n argocd port-forward svc/argocd-server 8080:80, with ssh -L 8080:127.0.0.1:8080." diff --git a/roles/argocd/tasks/main.yml b/roles/argocd/tasks/main.yml new file mode 100644 index 0000000..e052e64 --- /dev/null +++ b/roles/argocd/tasks/main.yml @@ -0,0 +1,16 @@ +--- +# Argo CD on argocd_hub, managing every box in the play. Ansible installs it +# once; after that it manages itself from ipni/infra (see tasks/hub.yml). +# +# The other boxes go first: each opens 6443 to the hub and creates the account +# Argo CD signs in with. Only then does the hub install Argo CD if needed and +# register them, reading what they produced. That ordering relies on every box +# being in the same batch, which is why platform.yml sets no `serial`. + +- name: Prepare this box to be managed from the hub + ansible.builtin.include_tasks: managed.yml + when: inventory_hostname != argocd_hub + +- name: Install Argo CD on the hub and register the boxes + ansible.builtin.include_tasks: hub.yml + when: inventory_hostname == argocd_hub diff --git a/roles/argocd/tasks/managed.yml b/roles/argocd/tasks/managed.yml new file mode 100644 index 0000000..0c79fcf --- /dev/null +++ b/roles/argocd/tasks/managed.yml @@ -0,0 +1,69 @@ +--- +# A box Argo CD on argocd_hub manages from outside. + +# 6443 is otherwise closed to everything but the SSH tunnel; see the firewall +# section of group_vars/ipfs_nodes/main.yml. +- name: Allow the Argo CD hub, and only the hub, to reach the API server + ansible.builtin.include_role: + name: ufw_allowlist + vars: + ufw_allowlist_port: "6443" + ufw_allowlist_comment: argocd hub + ufw_allowlist_sources: ["{{ hostvars[argocd_hub].ansible_host }}"] + +# cluster-admin, because Argo CD creates namespaces, CRDs and cluster roles for +# the charts it deploys. The token never expires; delete the Secret to revoke +# it, and the next run issues a new one. +- name: Create the service account Argo CD signs in with + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: argocd-manager + namespace: kube-system + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: argocd-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: argocd-manager + namespace: kube-system + - apiVersion: v1 + kind: Secret + type: kubernetes.io/service-account-token + metadata: + name: argocd-manager-token + namespace: kube-system + annotations: + kubernetes.io/service-account.name: argocd-manager + +# The token controller fills the Secret in a moment after it is created. Under +# --check nothing was created, so there is nothing to read. +- name: Read the Argo CD token and the cluster CA + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + kind: Secret + namespace: kube-system + name: argocd-manager-token + register: argocd_manager_secret + retries: 15 + delay: 2 + until: (argocd_manager_secret.resources | first | default({})).data.token is defined + when: not ansible_check_mode + no_log: true + +# Read back by hub.yml as hostvars[].argocd_manager_*. +- name: Keep the token and CA for the hub + ansible.builtin.set_fact: + argocd_manager_token: "{{ (argocd_manager_secret.resources | first).data.token | b64decode }}" + argocd_manager_ca: "{{ (argocd_manager_secret.resources | first).data['ca.crt'] }}" + when: not ansible_check_mode + no_log: true diff --git a/roles/monitoring/defaults/main.yml b/roles/monitoring/defaults/main.yml new file mode 100644 index 0000000..714996b --- /dev/null +++ b/roles/monitoring/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# Addresses outside the inventory that push metrics and logs to the hub too. +# The boxes in the inventory are always allowed. sf and sf2 (ipni/infra) go +# here when they move off grafana.ipni.io: +# monitoring_ingest_extra_sources: [103.232.206.165, 103.232.206.162] +monitoring_ingest_extra_sources: [] diff --git a/roles/monitoring/tasks/healthchecks.yml b/roles/monitoring/tasks/healthchecks.yml new file mode 100644 index 0000000..60e644d --- /dev/null +++ b/roles/monitoring/tasks/healthchecks.yml @@ -0,0 +1,11 @@ +--- +# Healthchecks (ipni/infra, kubernetes/manifests/ap/sing1/healthchecks) listens +# on host port 8000. Only Alertmanager on the hub sends to it; people reach the +# UI over an SSH tunnel. +- name: Allow the monitoring hub, and only the hub, to ping Healthchecks + ansible.builtin.include_role: + name: ufw_allowlist + vars: + ufw_allowlist_port: "8000" + ufw_allowlist_comment: healthchecks from hub + ufw_allowlist_sources: ["{{ hostvars[argocd_hub].ansible_host }}"] diff --git a/roles/monitoring/tasks/hub.yml b/roles/monitoring/tasks/hub.yml new file mode 100644 index 0000000..9d2f838 --- /dev/null +++ b/roles/monitoring/tasks/hub.yml @@ -0,0 +1,71 @@ +--- +# Traefik in the monitoring namespace listens on host port 8443 (443 is the +# route-origin Envoy's). It serves three hostnames: +# monitoring.ipni.io, argocd.ipni.io people, through Cloudflare's proxy +# ingest.monitoring.ipni.io Alloy on the other boxes, DNS only + +- name: Fail if there is no Cloudflare token for the monitoring certificate + ansible.builtin.assert: + that: vault_cloudflare_dns_token is defined and vault_cloudflare_dns_token | length > 0 + fail_msg: >- + vault_cloudflare_dns_token is not set (group_vars/ipfs_nodes/vault.yml). + cert-manager needs it for the monitoring certificate, as for the WSS one. + quiet: true + +# The same pinned list as the route-origin allowlist on 443; platform.yml reads +# it from roles/route_origin/defaults. +- name: Allow Cloudflare's edge to reach the monitoring hub + ansible.builtin.include_role: + name: ufw_allowlist + vars: + ufw_allowlist_port: "8443" + ufw_allowlist_comment: cloudflare monitoring + ufw_allowlist_sources: "{{ route_origin_cloudflare_ranges }}" + +- name: Allow the other boxes to push metrics and logs to the hub + ansible.builtin.include_role: + name: ufw_allowlist + vars: + ufw_allowlist_port: "8443" + ufw_allowlist_comment: monitoring ingest + ufw_allowlist_sources: >- + {{ groups['ipfs_nodes'] | reject('equalto', argocd_hub) + | map('extract', hostvars, 'ansible_host') | list + + monitoring_ingest_extra_sources }} + +# The Issuer in ipni/infra (kubernetes/manifests/eu/lith1/monitoring) reads +# this Secret. Created here so the token goes from the vault straight to the +# API server and is never in either repo. Argo CD creates the namespace too; +# whichever runs first wins, and neither deletes it. +- name: Check whether the monitoring namespace exists yet + ansible.builtin.command: + argv: [k3s, kubectl, get, namespace, monitoring, --ignore-not-found, -o, name] + register: monitoring_ns + changed_when: false + check_mode: false + +- name: Ensure the monitoring namespace exists + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + api_version: v1 + kind: Namespace + name: monitoring + state: present + +- name: Install the Cloudflare DNS token for the monitoring certificate + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: cloudflare-api-token + namespace: monitoring + data: + api-token: "{{ vault_cloudflare_dns_token | b64encode }}" + # Under --check on a first run the namespace above is only simulated, and a + # dry-run create of the Secret fails with "namespace not found". + when: not (ansible_check_mode and not monitoring_ns.stdout) + no_log: true diff --git a/roles/monitoring/tasks/main.yml b/roles/monitoring/tasks/main.yml new file mode 100644 index 0000000..885c76f --- /dev/null +++ b/roles/monitoring/tasks/main.yml @@ -0,0 +1,12 @@ +--- +# What the monitoring stack in ipni/infra needs from the boxes and cannot +# create itself: firewall openings and the one credential that lives in this +# repo's vault. The stack itself is deployed by Argo CD (roles/argocd). + +- name: Prepare the monitoring hub + ansible.builtin.include_tasks: hub.yml + when: inventory_hostname == argocd_hub + +- name: Prepare the Healthchecks box + ansible.builtin.include_tasks: healthchecks.yml + when: inventory_hostname == healthchecks_host diff --git a/roles/ufw_allowlist/defaults/main.yml b/roles/ufw_allowlist/defaults/main.yml new file mode 100644 index 0000000..dca49f4 --- /dev/null +++ b/roles/ufw_allowlist/defaults/main.yml @@ -0,0 +1,8 @@ +--- +# Required, set by the including role: +# ufw_allowlist_port port to open, e.g. "6443" +# ufw_allowlist_comment ufw comment that marks the rules this call owns; +# must be unique across the box's rules +# ufw_allowlist_sources the addresses or ranges allowed to reach the port + +ufw_allowlist_proto: tcp diff --git a/roles/ufw_allowlist/tasks/main.yml b/roles/ufw_allowlist/tasks/main.yml new file mode 100644 index 0000000..73dda95 --- /dev/null +++ b/roles/ufw_allowlist/tasks/main.yml @@ -0,0 +1,40 @@ +--- +# Open one port to an exact list of sources, and keep ufw in step with that +# list in BOTH directions: a source dropped from the list loses its rule, so +# the port never stays open to an address nobody asked for. Only rules carrying +# ufw_allowlist_comment are considered, which is why it must be unique. +# +# The same approach as the Cloudflare allowlist in roles/route_origin, for the +# ports roles/argocd and roles/monitoring open to a handful of addresses. + +- name: "Allow {{ ufw_allowlist_port }}/{{ ufw_allowlist_proto }} from each source ({{ ufw_allowlist_comment }})" + community.general.ufw: + rule: allow + proto: "{{ ufw_allowlist_proto }}" + port: "{{ ufw_allowlist_port }}" + from_ip: "{{ item }}" + comment: "{{ ufw_allowlist_comment }}" + loop: "{{ ufw_allowlist_sources }}" + +# `ufw status` prints each rule as `/ ALLOW # `. +- name: "Read the {{ ufw_allowlist_comment }} rules currently in ufw" + ansible.builtin.shell: + cmd: >- + set -o pipefail; + ufw status | grep -F -- {{ ('# ' ~ ufw_allowlist_comment) | quote }} | + awk '{for (i = 1; i <= NF; i++) if ($i == "ALLOW") print $(i + 1)}' + executable: /bin/bash + register: ufw_allowlist_current + changed_when: false + check_mode: false + # grep exits 1 when there are no such rules yet, which is not an error. + failed_when: ufw_allowlist_current.rc not in [0, 1] + +- name: "Remove {{ ufw_allowlist_comment }} rules for sources no longer on the list" + community.general.ufw: + rule: allow + proto: "{{ ufw_allowlist_proto }}" + port: "{{ ufw_allowlist_port }}" + from_ip: "{{ item }}" + delete: true + loop: "{{ ufw_allowlist_current.stdout_lines | difference(ufw_allowlist_sources) }}" diff --git a/scripts/ci-platform.sh b/scripts/ci-platform.sh new file mode 100755 index 0000000..28cc8b6 --- /dev/null +++ b/scripts/ci-platform.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Run platform.yml from CI (.github/workflows/platform.yml). +# +# scripts/ci-platform.sh check [limit] dry run: --check --diff +# scripts/ci-platform.sh apply [limit] the real run +# +# The vault files are gitignored, so CI has none: platform.yml's three vault +# variables come from repository secrets instead, in the environment: +# ARGOCD_AGE_KEY -> vault_argocd_age_key +# CLOUDFLARE_DNS_TOKEN -> vault_cloudflare_dns_token +# ARGOCD_REPO_SSH_KEY -> vault_argocd_repo_ssh_key (optional: only while +# ipni/infra is private) +# and SSH as the admin user from ANSIBLE_SSH_PRIVATE_KEY, whose public half is +# keys/github-actions.pub. Copies of every secret are in 1Password (IPFS +# account, Compass vault). +set -euo pipefail + +mode=${1:?usage: $0 check|apply [limit]} +limit=${2:-} +[[ $mode == check || $mode == apply ]] || { echo "mode must be check or apply" >&2; exit 2; } +: "${ANSIBLE_SSH_PRIVATE_KEY:?not set}" "${ARGOCD_AGE_KEY:?not set}" "${CLOUDFLARE_DNS_TOKEN:?not set}" + +cd "$(dirname "$0")/.." +umask 077 + +# Connect only to the boxes whose host keys are pinned in the repo. +mkdir -p "$HOME/.ssh" +printf '%s\n' "$ANSIBLE_SSH_PRIVATE_KEY" > "$HOME/.ssh/id_ed25519" +grep -v '^#' inventory/known_hosts > "$HOME/.ssh/known_hosts" + +# ansible.cfg names .vault_pass; nothing CI reads is vault-encrypted. +echo ci > .vault_pass + +# The secrets as extra vars, from a file rather than the command line, so they +# never appear in a process list or the log. +secrets=$(mktemp) +trap 'rm -f "$secrets"' EXIT +python3 - "$secrets" <<'PY' +import json, os, sys +values = { + "vault_argocd_age_key": os.environ["ARGOCD_AGE_KEY"], + "vault_cloudflare_dns_token": os.environ["CLOUDFLARE_DNS_TOKEN"], +} +if os.environ.get("ARGOCD_REPO_SSH_KEY"): + values["vault_argocd_repo_ssh_key"] = os.environ["ARGOCD_REPO_SSH_KEY"] +with open(sys.argv[1], "w") as f: + json.dump(values, f) # JSON is YAML, which -e @file reads +PY + +args=(platform.yml -e "@$secrets") +[[ $mode == check ]] && args+=(--check --diff) +[[ -n $limit ]] && args+=(--limit "$limit") + +ansible-playbook "${args[@]}"