Install Argo CD on lith-1 and open the ports monitoring needs

.github/workflows/ci.yml

@@ -40,7 +40,7 @@ jobs:
       # others and nothing else would say so.
       - name: Syntax-check every playbook
         run: |
-          for playbook in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml; do
+          for playbook in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml platform.yml; do
             echo "== $playbook"
             ansible-playbook --syntax-check "$playbook"
           done

.github/workflows/platform.yml

@@ -0,0 +1,115 @@
+---
+# platform.yml from CI: Argo CD's first install, registering the boxes with
+# it, and the firewall rules the monitoring stack needs (README, "Argo CD and
+# monitoring"). The service playbooks still run from an operator's machine.
+#
+#   pull request   -> dry run (--check --diff) against the boxes
+#   merge to main  -> the real run, once approved in the `production`
+#                     environment (required reviewers)
+#   manual         -> either, optionally limited to some boxes
+#
+# Pull requests from forks get no secrets and are skipped.
+name: platform
+
+on:
+  pull_request:
+    paths:
+      - platform.yml
+      - roles/argocd/**
+      - roles/monitoring/**
+      - roles/ufw_allowlist/**
+      - roles/route_origin/defaults/main.yml
+      - group_vars/**
+      - inventory/**
+      - scripts/ci-platform.sh
+      - .github/workflows/platform.yml
+  push:
+    branches: [main]
+    paths:
+      - platform.yml
+      - roles/argocd/**
+      - roles/monitoring/**
+      - roles/ufw_allowlist/**
+      - roles/route_origin/defaults/main.yml
+      - group_vars/**
+      - inventory/**
+      - scripts/ci-platform.sh
+      - .github/workflows/platform.yml
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "check (dry run) or apply"
+        type: choice
+        options: [check, apply]
+        default: check
+      limit:
+        description: "Boxes to run on (ansible --limit), empty for all"
+        type: string
+        default: ""
+
+permissions:
+  contents: read
+
+# One run at a time against the boxes; never cancel one half-way.
+concurrency:
+  group: platform
+  cancel-in-progress: false
+
+env:
+  # ansible.cfg sets the yaml callback, which no installed collection provides
+  # here (see ci.yml).
+  ANSIBLE_STDOUT_CALLBACK: default
+  ANSIBLE_FORCE_COLOR: "true"
+
+jobs:
+  check:
+    if: >-
+      (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) ||
+      (github.event_name == 'workflow_dispatch' && inputs.mode == 'check')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install Ansible and the collections the roles use
+        run: |
+          pip install ansible-core
+          ansible-galaxy collection install community.general ansible.posix kubernetes.core
+      - name: Dry run platform.yml
+        env:
+          ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }}
+          ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }}
+          CLOUDFLARE_DNS_TOKEN: ${{ secrets.CLOUDFLARE_DNS_TOKEN }}
+          ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }}
+          LIMIT: ${{ inputs.limit }}
+        run: scripts/ci-platform.sh check "$LIMIT"
+
+  apply:
+    if: >-
+      (github.event_name == 'push') ||
+      (github.event_name == 'workflow_dispatch' && inputs.mode == 'apply')
+    runs-on: ubuntu-latest
+    # Required reviewers: nothing reaches the boxes until one approves.
+    environment: production
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install Ansible and the collections the roles use
+        run: |
+          pip install ansible-core
+          ansible-galaxy collection install community.general ansible.posix kubernetes.core
+      - name: Run platform.yml
+        env:
+          ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }}
+          ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }}
+          CLOUDFLARE_DNS_TOKEN: ${{ secrets.CLOUDFLARE_DNS_TOKEN }}
+          ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }}
+          LIMIT: ${{ inputs.limit }}
+        run: scripts/ci-platform.sh apply "$LIMIT"

AGENTS.md

@@ -12,8 +12,33 @@ three at once is a full outage of that service.
 
 ## Getting in
 
-SSH is key-only as the hardened admin user `ipni` (`~/.ssh/id_ed25519`); root
-login is disabled and the provider KVM console is the out-of-band fallback.
+SSH is key-only as the hardened admin user `ipni`; root login is disabled and
+the provider KVM console is the out-of-band fallback. The keys that may sign in
+are the files in `keys/`, listed in `admin_pubkey_file` (takgn, nymd, byo, the
+break-glass key and CI's own key); the hardening role removes any other.
+
+### Break-glass access
+
+For when no operator key works (lost laptop, someone away, a bad hardening
+run). The private key exists only in 1Password: IPFS account, Compass vault,
+item "IPNI boxes: break-glass SSH key (ipni user)", ID
+`4ghgjny7l4rawld4spedu7mxz4`. The reference below uses the ID: the title's `:`
+is not allowed in a 1Password reference, and the ID survives a rename. Use it
+without writing it to disk, through a throwaway agent:
+
+```bash
+eval "$(ssh-agent -s)"
+op read --account team-ipfs.1password.com \
+  "op://Compass/4ghgjny7l4rawld4spedu7mxz4/private key?ssh-format=openssh" | ssh-add -
+ssh -o IdentitiesOnly=yes -o IdentityFile=keys/break-glass.pub ipni@46.166.169.131
+ssh-agent -k
+```
+
+Then fix whatever locked people out, through the repo, not by hand. If the key
+has been used outside a real emergency or may have leaked, replace it: a new
+SSH Key item in Compass, its public half in `keys/break-glass.pub`, and a
+hardening run. Last resort if SSH itself is down: the provider's KVM console,
+with `vault_console_password`.
 
 The playbooks set `remote_user: "{{ admin_user }}"` themselves, so
 `ansible-playbook` needs no user flag. **Ad-hoc `ansible` commands do not**, and
@@ -33,7 +58,9 @@ controller; in CI it is `echo ci > .vault_pass`.
 ## kubectl
 
 The k3s API server (6443) is deliberately closed to the internet — ufw denies
-it and there is no flag to open it. Two ways in:
+it and there is no flag to open it. The one exception is Argo CD on lith-1
+(`argocd_hub`), which `platform.yml` lets reach the other boxes' 6443 from
+lith-1's address only. Two ways in for people:
 
 ```bash
 # 1. straight through ssh, no setup
@@ -162,7 +189,7 @@ deliberately disables `line-length` and `braces` (see `.ansible-lint`). A pass
 looks like `Passed: 0 failure(s), 0 warning(s) ... profile ... 'production'`.
 
 ```bash
-for pb in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml; do
+for pb in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml platform.yml; do
   ansible-playbook --syntax-check "$pb"
 done
 ANSIBLE_STDOUT_CALLBACK=default ansible-playbook scripts/render-manifests.yml -e dest=rendered

README.md

@@ -111,16 +111,18 @@ with the OS disk.
 > `/data` existed would otherwise make the in-use data disk look blank. Once a
 > volume labelled `ipfsdata` exists, the format step is skipped outright.
 
-**`hardening`** — creates the `ipni` admin user with your ed25519 key and
+**`hardening`** — creates the `ipni` admin user with the operators' keys and
 passwordless sudo, then locks SSH down to key-only with root login disabled,
 enables ufw with a default-deny inbound policy, and configures fail2ban.
 
 > **The key files are the single source of truth.** `authorized_key` runs with
 > `exclusive: true`, so any key added to the `ipni` account by hand is removed
-> on the next run. `admin_pubkey_file` in `group_vars` takes either one path or
-> a list of them; to give a second operator access, add their key file to that
-> list rather than editing the host. Do it **before** the run that would
-> otherwise strip their key.
+> on the next run. `admin_pubkey_file` in `group_vars` lists committed public
+> key files in `keys/` (`takgn.pub`, `nymd.pub`, `byo.pub`, the 1Password-only
+> `break-glass.pub`, and CI's `github-actions.pub`), so the result does not
+> depend on who runs it. To give someone access, commit
+> their public key there and add it to the list rather than editing the host.
+> Do it **before** the run that would otherwise strip their key.
 
 fail2ban is kept mainly to cut log noise: with password authentication
 disabled there are few credential guesses left for it to ban.
@@ -1421,6 +1423,59 @@ pending entries first, then entries with a CID the cluster does not hold, then
 the pinned entries with per-peer status and DNSLink drift, followed by the raw
 `ipfs-cluster-ctl pin ls` and `status`.
 
+## Argo CD and monitoring
+
+Monitoring (Prometheus, Loki, Alertmanager, Grafana, Alloy on every box) lives
+in [ipni/infra](https://github.com/ipni/infra) and is deployed by Argo CD,
+which runs on `argocd_hub` (lith-1). This repo does only what Argo CD cannot do
+for itself:
+
+* **First install of Argo CD**, with the same Helm chart and release name
+  ipni/infra uses, plus the root Application that hands it over. After that
+  Argo CD manages its own release from ipni/infra; `platform.yml` never touches
+  it again, so upgrades and settings go there.
+* **Registration**: on each other box, a `cluster-admin` service account for
+  Argo CD, and 6443 opened to lith-1's address only (the one exception to the
+  closed API server). lith-1 therefore holds an admin token for every box.
+* **Secrets that cannot be in Git**: Argo CD's age key, which decrypts the
+  SOPS-encrypted Secrets in ipni/infra (`vault_argocd_age_key`), the deploy key
+  for ipni/infra while it is private (`vault_argocd_repo_ssh_key`, optional),
+  and the Cloudflare DNS token for the monitoring certificate.
+* **Firewall**: 8443 on lith-1 (Traefik for Grafana, Argo CD and metrics
+  ingest) from Cloudflare's edge and the other boxes, and 8000 on sing-1
+  (`healthchecks_host`, the hub's dead-man's switch) from lith-1.
+
+**It runs from CI** (`.github/workflows/platform.yml`), the only playbook that
+does; the others still run from an operator's machine:
+
+| When | What |
+|---|---|
+| Pull request touching it | `--check --diff` against the boxes (forks get no secrets and are skipped) |
+| Merge to `main` | the real run, once a required reviewer approves it in the `production` environment |
+| Actions → platform → Run workflow | `check` or `apply`, optionally `--limit`ed |
+
+It restarts nothing that serves traffic, so it runs every box in one batch: the
+hub registers the others using what they produced earlier in the same run.
+
+CI has no vault files, so `platform.yml`'s vault variables come from
+repository secrets (`scripts/ci-platform.sh` maps them), and it signs in with
+its own SSH key, `keys/github-actions.pub` in `admin_pubkey_file`. It only
+connects to the host keys pinned in `inventory/known_hosts`.
+
+| Secret | Becomes |
+|---|---|
+| `ANSIBLE_SSH_PRIVATE_KEY` | CI's SSH key as the admin user |
+| `ARGOCD_AGE_KEY` | `vault_argocd_age_key`: Argo CD's age key (public half in ipni/infra's `.sops.yaml`) |
+| `CLOUDFLARE_DNS_TOKEN` | `vault_cloudflare_dns_token`: the same token cert-manager uses for WSS |
+| `ARGOCD_REPO_SSH_KEY` | `vault_argocd_repo_ssh_key`: optional, only while ipni/infra is private |
+
+Copies of all of them are in 1Password (IPFS account, Compass vault): GitHub
+cannot show them again. To revoke CI's access to the boxes, remove its line
+from `admin_pubkey_file` and re-run the hardening role.
+
+From an operator's machine it still works as usual, with the same variables
+in the vault: `ansible-playbook platform.yml --check --diff`.
+
 ## Secrets
 
 **This repository is public, so no credentials are committed, not even
@@ -1434,7 +1489,7 @@ DNS.
 | File | Variables |
 |------|-----------|
 | `host_vars/<box>/vault.yml` | `vault_root_password`, `vault_console_password` (provider console), `vault_bootstrap_privkey`, `vault_bootstrap_api_token`, `vault_content_cluster_privkey`, `vault_content_kubo_privkey`; on chic-1 also `vault_bootstrap_b_privkey`, `vault_bootstrap_b_api_token` |
-| `group_vars/ipfs_nodes/vault.yml` | `vault_route_origin_tls_key`, `vault_origin_pull_ca_key`, `vault_origin_pull_client_key`, `vault_cloudflare_dns_token`, `vault_content_cluster_secret`, `vault_content_pin_users` (remote pinning users, see "Remote pinning") |
+| `group_vars/ipfs_nodes/vault.yml` | `vault_route_origin_tls_key`, `vault_origin_pull_ca_key`, `vault_origin_pull_client_key`, `vault_cloudflare_dns_token`, `vault_content_cluster_secret`, `vault_content_pin_users` (remote pinning users, see "Remote pinning"), `vault_argocd_age_key` and optionally `vault_argocd_repo_ssh_key` (see "Argo CD and monitoring") |
 
 The per-box files were generated from a plaintext `servers.txt` by
 `scripts/bootstrap-vault.sh`; that file was never committed and has since been
@@ -1462,7 +1517,7 @@ same path a deploy takes), so what is checked is what would be applied:
 
 | Check | Catches |
 | --- | --- |
-| `ansible-playbook --syntax-check` on all five playbooks | a role edit that breaks a playbook nobody ran |
+| `ansible-playbook --syntax-check` on all six playbooks | a role edit that breaks a playbook nobody ran |
 | `ansible-lint`, `yamllint` | the rest, minus the style rules in `.ansible-lint` |
 | `kubectl kustomize` on every rendered `k8s/*/` | a patch or digest pin that no longer applies |
 | rendering at `needle_instances` 1, 2 and 4 | the Deployments and Envoy's endpoint list falling out of step |
@@ -1472,6 +1527,9 @@ same path a deploy takes), so what is checked is what would be applied:
 | every `pinset.yml` section is in the role's `known_sections` | a section the content role silently ignores |
 | `shellcheck scripts/*.sh` | the usual shell traps |
 
+`platform.yml` also has its own workflow, which does talk to the boxes (see
+"Argo CD and monitoring").
+
 The Cloudflare range check is a separate weekly workflow rather than part of
 this one, because it reaches `api.cloudflare.com`: an outage there should not
 turn an unrelated pull request red.
@@ -1567,7 +1625,8 @@ turn an unrelated pull request red.
       are pinned, and compare `Provide.Strategy` `roots` against `pinned`.
 - [ ] Record which HTTP gateways serve the website to browsers without an IPFS
       client, who operates them, and whether they continue after 2026-09-30.
-- [ ] Set up monitoring/alerting, and scrape `/debug/metrics/prometheus`. Include
+- [ ] Set up monitoring/alerting (in progress: `platform.yml` here, the stack in
+      ipni/infra), and scrape `/debug/metrics/prometheus`. Include
       `/data` usage: the bootstrappers' DHT record store grows with traffic, the
       content repos grow with every pin on every box, and local-path enforces
       neither the 20 GiB nor the 100 GiB claim.
@@ -1587,6 +1646,9 @@ roles/needle/            needle firewall + deploy
 roles/route_origin/       Envoy origin: cert preflight, Cloudflare allowlist, TLS secret
   tasks/verify.yml        post-deploy request checks (paths, Host, traversal; AOP when enforced)
 roles/kustomize_apply/    shared: ship, dry-run/apply, wait, prune stale ConfigMaps
+roles/ufw_allowlist/      shared: open a port to an exact source list, removing rules for dropped sources
+roles/argocd/             Argo CD's first install on argocd_hub, cluster registration, age key, 6443 to the hub
+roles/monitoring/         firewall and Cloudflare token for the monitoring stack in ipni/infra
 k8s/needle/              needle kustomize manifests; the Deployments are .j2, rendered from needle_all_instances
 k8s/route-origin/         Envoy kustomize manifests and envoy.yaml.j2 (needle endpoints rendered)
 k8s/bootstrap/            kubo bootstrapper manifests (deployment, repo PVC)
@@ -1628,6 +1690,11 @@ dns.txt                   Cloudflare-importable bootstrap DNS records (generated
 dns-libp2p.txt            bootstrap.libp2p.io records for the legacy identities (generated)
 site.yml                  base preparation playbook
 k3s-upgrade.yml           k3s upgrade, one minor version at a time, backup per step
+platform.yml              Argo CD (first install) and the monitoring stack's prerequisites, run from CI
+.github/workflows/platform.yml  platform.yml: dry run on PRs, approved run on merge
+scripts/ci-platform.sh    CI's wrapper for platform.yml: secrets to variables, SSH key, pinned host keys
+inventory/known_hosts     the boxes' SSH host keys, pinned for CI
+keys/*.pub                the SSH keys allowed in as the admin user: takgn, nymd, byo, break-glass, CI (admin_pubkey_file)
 routing.yml               routing service playbook (needle + origin)
 bootstrap.yml             bootstrap node playbook (with cert-manager)
 content.yml               content cluster node playbook (deploy, load the pinset, wait for PINNED, summary)