From a2958fa66ad00d82f4db2b96b5fdfff85b40d862 Mon Sep 17 00:00:00 2001 From: Takwa Date: Fri, 9 Oct 2026 09:39:45 +0200 Subject: [PATCH 1/7] Install Argo CD on lith-1 and open the ports monitoring needs platform.yml prepares the boxes for the monitoring stack in ipni/infra: - roles/argocd installs Argo CD on argocd_hub once, with the argo-cd Helm chart and the release name ipni/infra uses, and creates the root Application. From then on Argo CD manages its own release from ipni/infra. Every run it registers the boxes, and installs Argo CD's age key (for the SOPS-encrypted Secrets in ipni/infra) and, while that repo is private, its deploy key, both from the vault. - On the other boxes it opens 6443 to lith-1's address only, the one exception to the closed API server, and creates the account Argo CD uses. - roles/monitoring opens 8443 on lith-1 to Cloudflare and the other boxes, and 8000 on sing-1 to lith-1 (Healthchecks), and installs the Cloudflare token for the monitoring certificate. - roles/ufw_allowlist opens a port to an exact source list and removes rules for sources that are dropped from it. Ports 8000 and 8443 are reserved, and CI syntax-checks platform.yml. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- AGENTS.md | 6 +- README.md | 52 +++++- group_vars/ipfs_nodes/main.yml | 28 +++- platform.yml | 34 ++++ roles/argocd/defaults/main.yml | 26 +++ roles/argocd/tasks/hub.yml | 205 ++++++++++++++++++++++++ roles/argocd/tasks/main.yml | 16 ++ roles/argocd/tasks/managed.yml | 69 ++++++++ roles/monitoring/defaults/main.yml | 6 + roles/monitoring/tasks/healthchecks.yml | 11 ++ roles/monitoring/tasks/hub.yml | 61 +++++++ roles/monitoring/tasks/main.yml | 12 ++ roles/ufw_allowlist/defaults/main.yml | 8 + roles/ufw_allowlist/tasks/main.yml | 40 +++++ 15 files changed, 567 insertions(+), 9 deletions(-) create mode 100644 platform.yml create mode 100644 roles/argocd/defaults/main.yml create mode 100644 roles/argocd/tasks/hub.yml create mode 100644 roles/argocd/tasks/main.yml create mode 100644 roles/argocd/tasks/managed.yml create mode 100644 roles/monitoring/defaults/main.yml create mode 100644 roles/monitoring/tasks/healthchecks.yml create mode 100644 roles/monitoring/tasks/hub.yml create mode 100644 roles/monitoring/tasks/main.yml create mode 100644 roles/ufw_allowlist/defaults/main.yml create mode 100644 roles/ufw_allowlist/tasks/main.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0366fd..911fcc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: # others and nothing else would say so. - name: Syntax-check every playbook run: | - for playbook in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml; do + for playbook in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml platform.yml; do echo "== $playbook" ansible-playbook --syntax-check "$playbook" done diff --git a/AGENTS.md b/AGENTS.md index 473e379..96d9a3e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,7 +33,9 @@ controller; in CI it is `echo ci > .vault_pass`. ## kubectl The k3s API server (6443) is deliberately closed to the internet — ufw denies -it and there is no flag to open it. Two ways in: +it and there is no flag to open it. The one exception is Argo CD on lith-1 +(`argocd_hub`), which `platform.yml` lets reach the other boxes' 6443 from +lith-1's address only. Two ways in for people: ```bash # 1. straight through ssh, no setup @@ -162,7 +164,7 @@ deliberately disables `line-length` and `braces` (see `.ansible-lint`). A pass looks like `Passed: 0 failure(s), 0 warning(s) ... profile ... 'production'`. ```bash -for pb in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml; do +for pb in site.yml routing.yml bootstrap.yml content.yml k3s-upgrade.yml platform.yml; do ansible-playbook --syntax-check "$pb" done ANSIBLE_STDOUT_CALLBACK=default ansible-playbook scripts/render-manifests.yml -e dest=rendered diff --git a/README.md b/README.md index b38b1aa..108dedb 100644 --- a/README.md +++ b/README.md @@ -1421,6 +1421,47 @@ pending entries first, then entries with a CID the cluster does not hold, then the pinned entries with per-peer status and DNSLink drift, followed by the raw `ipfs-cluster-ctl pin ls` and `status`. +## Argo CD and monitoring + +Monitoring (Prometheus, Loki, Alertmanager, Grafana, Alloy on every box) lives +in [ipni/infra](https://github.com/ipni/infra) and is deployed by Argo CD, +which runs on `argocd_hub` (lith-1). This repo does only what Argo CD cannot do +for itself: + +* **First install of Argo CD**, with the same Helm chart and release name + ipni/infra uses, plus the root Application that hands it over. After that + Argo CD manages its own release from ipni/infra; `platform.yml` never touches + it again, so upgrades and settings go there. +* **Registration**: on each other box, a `cluster-admin` service account for + Argo CD, and 6443 opened to lith-1's address only (the one exception to the + closed API server). lith-1 therefore holds an admin token for every box. +* **Secrets that cannot be in Git**: Argo CD's age key, which decrypts the + SOPS-encrypted Secrets in ipni/infra (`vault_argocd_age_key`), the deploy key + for ipni/infra while it is private (`vault_argocd_repo_ssh_key`, optional), + and the Cloudflare DNS token for the monitoring certificate. +* **Firewall**: 8443 on lith-1 (Traefik for Grafana, Argo CD and metrics + ingest) from Cloudflare's edge and the other boxes, and 8000 on sing-1 + (`healthchecks_host`, the hub's dead-man's switch) from lith-1. + +```bash +ansible-playbook platform.yml --check --diff +ansible-playbook platform.yml +``` + +It restarts nothing that serves traffic, so it runs every box in one batch: the +hub registers the others using what they produced earlier in the same run. + +Argo CD's age key is generated once with `age-keygen`; its public half goes in +ipni/infra's `.sops.yaml`, the private half into the vault: + +```bash +age-keygen -o argocd.agekey # prints the public key +ansible-vault edit group_vars/ipfs_nodes/vault.yml +# vault_argocd_age_key: | +# +rm argocd.agekey +``` + ## Secrets **This repository is public, so no credentials are committed, not even @@ -1434,7 +1475,7 @@ DNS. | File | Variables | |------|-----------| | `host_vars//vault.yml` | `vault_root_password`, `vault_console_password` (provider console), `vault_bootstrap_privkey`, `vault_bootstrap_api_token`, `vault_content_cluster_privkey`, `vault_content_kubo_privkey`; on chic-1 also `vault_bootstrap_b_privkey`, `vault_bootstrap_b_api_token` | -| `group_vars/ipfs_nodes/vault.yml` | `vault_route_origin_tls_key`, `vault_origin_pull_ca_key`, `vault_origin_pull_client_key`, `vault_cloudflare_dns_token`, `vault_content_cluster_secret`, `vault_content_pin_users` (remote pinning users, see "Remote pinning") | +| `group_vars/ipfs_nodes/vault.yml` | `vault_route_origin_tls_key`, `vault_origin_pull_ca_key`, `vault_origin_pull_client_key`, `vault_cloudflare_dns_token`, `vault_content_cluster_secret`, `vault_content_pin_users` (remote pinning users, see "Remote pinning"), `vault_argocd_age_key` and optionally `vault_argocd_repo_ssh_key` (see "Argo CD and monitoring") | The per-box files were generated from a plaintext `servers.txt` by `scripts/bootstrap-vault.sh`; that file was never committed and has since been @@ -1462,7 +1503,7 @@ same path a deploy takes), so what is checked is what would be applied: | Check | Catches | | --- | --- | -| `ansible-playbook --syntax-check` on all five playbooks | a role edit that breaks a playbook nobody ran | +| `ansible-playbook --syntax-check` on all six playbooks | a role edit that breaks a playbook nobody ran | | `ansible-lint`, `yamllint` | the rest, minus the style rules in `.ansible-lint` | | `kubectl kustomize` on every rendered `k8s/*/` | a patch or digest pin that no longer applies | | rendering at `needle_instances` 1, 2 and 4 | the Deployments and Envoy's endpoint list falling out of step | @@ -1567,7 +1608,8 @@ turn an unrelated pull request red. are pinned, and compare `Provide.Strategy` `roots` against `pinned`. - [ ] Record which HTTP gateways serve the website to browsers without an IPFS client, who operates them, and whether they continue after 2026-09-30. -- [ ] Set up monitoring/alerting, and scrape `/debug/metrics/prometheus`. Include +- [ ] Set up monitoring/alerting (in progress: `platform.yml` here, the stack in + ipni/infra), and scrape `/debug/metrics/prometheus`. Include `/data` usage: the bootstrappers' DHT record store grows with traffic, the content repos grow with every pin on every box, and local-path enforces neither the 20 GiB nor the 100 GiB claim. @@ -1587,6 +1629,9 @@ roles/needle/ needle firewall + deploy roles/route_origin/ Envoy origin: cert preflight, Cloudflare allowlist, TLS secret tasks/verify.yml post-deploy request checks (paths, Host, traversal; AOP when enforced) roles/kustomize_apply/ shared: ship, dry-run/apply, wait, prune stale ConfigMaps +roles/ufw_allowlist/ shared: open a port to an exact source list, removing rules for dropped sources +roles/argocd/ Argo CD's first install on argocd_hub, cluster registration, age key, 6443 to the hub +roles/monitoring/ firewall and Cloudflare token for the monitoring stack in ipni/infra k8s/needle/ needle kustomize manifests; the Deployments are .j2, rendered from needle_all_instances k8s/route-origin/ Envoy kustomize manifests and envoy.yaml.j2 (needle endpoints rendered) k8s/bootstrap/ kubo bootstrapper manifests (deployment, repo PVC) @@ -1628,6 +1673,7 @@ dns.txt Cloudflare-importable bootstrap DNS records (generated dns-libp2p.txt bootstrap.libp2p.io records for the legacy identities (generated) site.yml base preparation playbook k3s-upgrade.yml k3s upgrade, one minor version at a time, backup per step +platform.yml Argo CD (first install) and the monitoring stack's prerequisites routing.yml routing service playbook (needle + origin) bootstrap.yml bootstrap node playbook (with cert-manager) content.yml content cluster node playbook (deploy, load the pinset, wait for PINNED, summary) diff --git a/group_vars/ipfs_nodes/main.yml b/group_vars/ipfs_nodes/main.yml index 6d89729..54229b5 100644 --- a/group_vars/ipfs_nodes/main.yml +++ b/group_vars/ipfs_nodes/main.yml @@ -33,8 +33,9 @@ ssh_allow_users: "{{ admin_user }}" # # The k3s API server (6443) is deliberately absent from this list: it stays # closed to the internet and remote kubectl goes over an SSH tunnel (see -# scripts/kubectl-tunnel.sh). This list is the only switch - there is no -# separate "expose the API" flag. +# scripts/kubectl-tunnel.sh). Do not add it here - this list opens a port to +# every source. The one exception is narrower: roles/argocd opens 6443 on the +# other boxes to argocd_hub's address only, for Argo CD (see below). firewall_allowed_tcp: - "{{ ssh_port }}" firewall_allowed_udp: [] @@ -124,6 +125,7 @@ sysctl_tuning: # 5011 bootstrapper API 5012 bootstrap-b API # 5021 content kubo RPC API (loopback) # 6443-6444 k3s API 8190-8193 needle APIs (loopback, four instances) + # 8000 Healthchecks (healthchecks_host only) 8443 monitoring Traefik (argocd_hub only) # 8472/udp flannel VXLAN # 9094 content ipfs-cluster REST API (loopback) # 9095 content ipfs-cluster IPFS proxy (loopback) @@ -133,7 +135,7 @@ sysctl_tuning: # 9901 route-origin Envoy admin 9902 bootstrap-wss Envoy admin # 9903 bootstrap-b's bootstrap-wss Envoy admin # 10010 containerd 10248-10259 kubelet/kube-proxy/k3s - net.ipv4.ip_local_reserved_ports: "443,4001-4002,4004-4007,4011-4012,4101,4443,4453,5011-5012,5021,6443-6444,8190-8193,8472,9094-9097,9443,9901-9904,10010,10248-10259" + net.ipv4.ip_local_reserved_ports: "443,4001-4002,4004-4007,4011-4012,4101,4443,4453,5011-5012,5021,6443-6444,8000,8190-8193,8443,8472,9094-9097,9443,9901-9904,10010,10248-10259" net.ipv4.tcp_fin_timeout: 15 # Must equal what k3s's kube-proxy sets on every start: max(32768 * CPUs, # 131072), i.e. 1048576 on these 32-core boxes. A fixed lower value both @@ -170,3 +172,23 @@ bootstrap_instances: >- {'host': inventory_hostname ~ bootstrap_instance_table[i.name].suffix})) -%} {%- endfor -%} {{ out }} + +# --------------------------------------------------------------------------- +# Argo CD and monitoring (platform.yml) +# --------------------------------------------------------------------------- +# One Argo CD, on argocd_hub, deploys the monitoring stack in ipni/infra to +# every box. It reaches the other boxes' API servers directly, so roles/argocd +# opens 6443 on them to argocd_hub's address and nothing else. The trade-off: +# argocd_hub holds a cluster-admin token for every box, so a compromise of it +# reaches all of them. The monitoring hub (Prometheus, Loki, Alertmanager, +# Grafana) runs on the same box, behind Traefik on :8443. +argocd_hub: lith-1 + +# Runs Healthchecks, the dead-man's switch for the hub: Alertmanager on +# argocd_hub pings it every minute, and it alerts when the pings stop. Must not +# be argocd_hub, or it would go down with the thing it watches. +healthchecks_host: sing-1 + +# The name each box has in Argo CD, and so in the `cluster` label on every +# metric and log line. ipni/infra refers to the boxes by these names. +argocd_cluster_name: "{{ inventory_hostname | replace('-', '') }}" diff --git a/platform.yml b/platform.yml new file mode 100644 index 0000000..f8c67f0 --- /dev/null +++ b/platform.yml @@ -0,0 +1,34 @@ +--- +# Argo CD's first install on the hub box, the Secrets it needs that cannot live +# in Git, and what the monitoring stack it deploys needs from the boxes: +# firewall openings and the Cloudflare token for its certificate. After the +# first install Argo CD manages itself, and everything else it deploys, from +# ipni/infra; see docs/03-monitoring.md there. +# +# ansible-playbook platform.yml # all three boxes +# ansible-playbook platform.yml --check --diff # dry run +# +# Requires site.yml to have run first (admin user, k3s, ufw, reserved ports), +# and bootstrap.yml (cert-manager) for the monitoring certificate. +# +# Nothing here restarts a running service: it adds firewall rules and Secrets, +# and installs Argo CD on argocd_hub if it is not there yet. So unlike the service +# playbooks there is no production_rollout batching - and there must not be: +# the hub registers the other boxes using what they produced earlier in this +# same play, so every box has to be in one batch. With -l, the hub registers +# only the boxes in the run, and leaves the others as an earlier run left them. + +- name: Argo CD and the monitoring stack's prerequisites + hosts: ipfs_nodes + remote_user: "{{ admin_user }}" + become: true + gather_facts: false + # route_origin_cloudflare_ranges: the pinned list the 8443 allowlist reuses + # (role defaults are not visible outside the role's play). + vars_files: + - roles/route_origin/defaults/main.yml + roles: + - role: argocd + tags: [argocd] + - role: monitoring + tags: [monitoring] diff --git a/roles/argocd/defaults/main.yml b/roles/argocd/defaults/main.yml new file mode 100644 index 0000000..49b4854 --- /dev/null +++ b/roles/argocd/defaults/main.yml @@ -0,0 +1,26 @@ +--- +# Argo CD's first install. After it, Argo CD manages itself from ipni/infra +# (kubernetes/argocd/apps/argocd.yaml), which is where upgrades and settings +# change. This role never touches the release again once it exists, so keep +# the chart version here in step with that file only for new boxes. +argocd_chart_repo: https://argoproj.github.io/argo-helm +argocd_chart_version: 10.10.0 + +# Helm, only for that first install, kept out of PATH. +argocd_helm_version: v3.22.0 +argocd_helm_sha256: 1e4ab49e429626cf6c6958d914248b78c9730803c2751b87627e171dc800e7bb +argocd_helm_dir: "/usr/local/lib/ipfs-workers/helm-{{ argocd_helm_version }}" + +# Where Argo CD deploys from: the root Application created on first install +# points here, and ipni/infra takes it from there. +argocd_repo_url: git@github.com:ipni/infra.git +argocd_repo_revision: main +argocd_root_path: kubernetes/argocd/apps + +# While argocd_repo_url is private, Argo CD reads it with a deploy key: set +# vault_argocd_repo_ssh_key in group_vars/ipfs_nodes/vault.yml. Once the repo +# is public, leave it unset and Argo CD reads it anonymously. +# +# vault_argocd_age_key (same file) is Argo CD's age private key, which decrypts +# the SOPS-encrypted Secrets in ipni/infra. Its public half is in that repo's +# .sops.yaml. diff --git a/roles/argocd/tasks/hub.yml b/roles/argocd/tasks/hub.yml new file mode 100644 index 0000000..6ef1f90 --- /dev/null +++ b/roles/argocd/tasks/hub.yml @@ -0,0 +1,205 @@ +--- +# Argo CD's first install, and the Secrets it needs that cannot live in Git. +# +# Ansible installs Argo CD ONCE, with the same Helm chart and release name +# ipni/infra uses. From then on Argo CD manages its own release from that repo +# and this role leaves it alone: upgrades and settings go through ipni/infra. +# The Secrets below are refreshed on every run; Argo CD does not manage them. + +- name: Fail if there is no age key for Argo CD + ansible.builtin.assert: + that: vault_argocd_age_key is defined and vault_argocd_age_key is search('AGE-SECRET-KEY-') + fail_msg: >- + vault_argocd_age_key is not set (group_vars/ipfs_nodes/vault.yml). Argo CD + needs it to decrypt the SOPS-encrypted Secrets in ipni/infra; see the README. + quiet: true + +- name: Check whether Argo CD is installed + ansible.builtin.command: + argv: [k3s, kubectl, --namespace=argocd, get, deployment/argocd-server, --ignore-not-found, -o, name] + register: argocd_installed + changed_when: false + check_mode: false + +# Under --check there is nothing to dry-run against yet: no Helm on the box, and +# no Argo CD CRDs for the root Application. Say so instead. +- name: Note that a first install is not dry-run + ansible.builtin.debug: + msg: "Argo CD is not installed; a real run would install chart {{ argocd_chart_version }} and create the root Application." + when: + - ansible_check_mode + - not argocd_installed.stdout + +- name: Install Argo CD for the first time + when: + - not ansible_check_mode + - not argocd_installed.stdout + block: + - name: Create the Helm directory + ansible.builtin.file: + path: "{{ argocd_helm_dir }}" + state: directory + owner: root + group: root + mode: "0755" + + - name: Download Helm, checked against its pinned checksum + ansible.builtin.get_url: + url: "https://get.helm.sh/helm-{{ argocd_helm_version }}-linux-amd64.tar.gz" + dest: "{{ argocd_helm_dir }}/helm.tar.gz" + checksum: "sha256:{{ argocd_helm_sha256 }}" + mode: "0644" + + - name: Unpack the Helm binary + ansible.builtin.unarchive: + src: "{{ argocd_helm_dir }}/helm.tar.gz" + remote_src: true + dest: "{{ argocd_helm_dir }}" + extra_opts: [--strip-components=1, linux-amd64/helm] + creates: "{{ argocd_helm_dir }}/helm" + + # Enough to start. The first sync of the argocd Application in ipni/infra + # replaces these with that repo's values (SOPS decryption, resources, ...). + - name: Install the Argo CD Helm release + kubernetes.core.helm: + binary_path: "{{ argocd_helm_dir }}/helm" + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: argocd + chart_repo_url: "{{ argocd_chart_repo }}" + chart_ref: argo-cd + chart_version: "{{ argocd_chart_version }}" + release_namespace: argocd + create_namespace: true + wait: true + wait_timeout: 10m + values: + global: + domain: argocd.ipni.io + configs: + params: + server.insecure: true + dex: + enabled: false + +# Read by repo-server to decrypt the SOPS-encrypted Secrets in ipni/infra; its +# values (kubernetes/argocd/values.yaml) mount it. Refreshed on every run, so +# rotating the key is: replace it in the vault, re-encrypt, re-run. +- name: Install Argo CD's age key + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: argocd-sops-age + namespace: argocd + data: + keys.txt: "{{ vault_argocd_age_key | b64encode }}" + no_log: true + +# One Secret per box, named after the box. ipni/infra targets boxes by these +# names, and its ApplicationSets pick up every box labelled ipni.io/managed. +# The hub talks to its own API server in-cluster. A box outside this run (-l) +# keeps the Secret an earlier run left; none is ever deleted here. +- name: List the boxes to register + ansible.builtin.set_fact: + argocd_clusters: >- + {%- set out = [{'name': argocd_cluster_name, + 'server': 'https://kubernetes.default.svc', + 'config': {'tlsClientConfig': {'insecure': false}}}] -%} + {%- for h in ansible_play_hosts if h != argocd_hub and hostvars[h].argocd_manager_token is defined -%} + {%- set _ = out.append({'name': hostvars[h].argocd_cluster_name, + 'server': 'https://' ~ hostvars[h].ansible_host ~ ':6443', + 'config': {'bearerToken': hostvars[h].argocd_manager_token, + 'tlsClientConfig': {'insecure': false, + 'caData': hostvars[h].argocd_manager_ca}}}) -%} + {%- endfor -%} + {{ out }} + no_log: true + +- name: Register the boxes with Argo CD + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: "cluster-{{ item.name }}" + namespace: argocd + labels: + argocd.argoproj.io/secret-type: cluster + ipni.io/managed: "true" + data: + name: "{{ item.name | b64encode }}" + server: "{{ item.server | b64encode }}" + config: "{{ item.config | to_json | b64encode }}" + loop: "{{ argocd_clusters }}" + loop_control: + label: "{{ item.name }} {{ item.server }}" + no_log: true + +- name: Give Argo CD read access to the infra repository + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: repo-infra + namespace: argocd + labels: + argocd.argoproj.io/secret-type: repository + data: + type: "{{ 'git' | b64encode }}" + url: "{{ argocd_repo_url | b64encode }}" + sshPrivateKey: "{{ vault_argocd_repo_ssh_key | b64encode }}" + when: vault_argocd_repo_ssh_key is defined + no_log: true + +# The hand-off: the root Application points Argo CD at ipni/infra, whose apps +# include Argo CD's own release. Created once, like the release; after that it +# is ipni/infra's to change (kubernetes/argocd/root-app.yaml). +- name: Point Argo CD at ipni/infra + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: argoproj.io/v1alpha1 + kind: Application + metadata: + name: root + namespace: argocd + spec: + project: default + source: + repoURL: "{{ argocd_repo_url }}" + targetRevision: "{{ argocd_repo_revision }}" + path: "{{ argocd_root_path }}" + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true + when: + - not ansible_check_mode + - not argocd_installed.stdout + +- name: Report what Argo CD manages + ansible.builtin.debug: + msg: + - "{{ 'installed Argo CD ' ~ argocd_chart_version ~ ' and handed it to ' ~ argocd_repo_url + if not argocd_installed.stdout else + 'Argo CD already installed; it manages itself from ' ~ argocd_repo_url ~ ', so its release was left alone' }}" + - "registered: {{ argocd_clusters | map(attribute='name') | join(', ') }}" + - "repository access: {{ 'deploy key from the vault' if vault_argocd_repo_ssh_key is defined + else 'anonymous, so ' ~ argocd_repo_url ~ ' must be public' }}" + - "UI: https://argocd.ipni.io once ipni/infra's monitoring stack serves it. Before that, on the box: + sudo k3s kubectl -n argocd port-forward svc/argocd-server 8080:80, with ssh -L 8080:127.0.0.1:8080." diff --git a/roles/argocd/tasks/main.yml b/roles/argocd/tasks/main.yml new file mode 100644 index 0000000..e052e64 --- /dev/null +++ b/roles/argocd/tasks/main.yml @@ -0,0 +1,16 @@ +--- +# Argo CD on argocd_hub, managing every box in the play. Ansible installs it +# once; after that it manages itself from ipni/infra (see tasks/hub.yml). +# +# The other boxes go first: each opens 6443 to the hub and creates the account +# Argo CD signs in with. Only then does the hub install Argo CD if needed and +# register them, reading what they produced. That ordering relies on every box +# being in the same batch, which is why platform.yml sets no `serial`. + +- name: Prepare this box to be managed from the hub + ansible.builtin.include_tasks: managed.yml + when: inventory_hostname != argocd_hub + +- name: Install Argo CD on the hub and register the boxes + ansible.builtin.include_tasks: hub.yml + when: inventory_hostname == argocd_hub diff --git a/roles/argocd/tasks/managed.yml b/roles/argocd/tasks/managed.yml new file mode 100644 index 0000000..0c79fcf --- /dev/null +++ b/roles/argocd/tasks/managed.yml @@ -0,0 +1,69 @@ +--- +# A box Argo CD on argocd_hub manages from outside. + +# 6443 is otherwise closed to everything but the SSH tunnel; see the firewall +# section of group_vars/ipfs_nodes/main.yml. +- name: Allow the Argo CD hub, and only the hub, to reach the API server + ansible.builtin.include_role: + name: ufw_allowlist + vars: + ufw_allowlist_port: "6443" + ufw_allowlist_comment: argocd hub + ufw_allowlist_sources: ["{{ hostvars[argocd_hub].ansible_host }}"] + +# cluster-admin, because Argo CD creates namespaces, CRDs and cluster roles for +# the charts it deploys. The token never expires; delete the Secret to revoke +# it, and the next run issues a new one. +- name: Create the service account Argo CD signs in with + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: argocd-manager + namespace: kube-system + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: argocd-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: argocd-manager + namespace: kube-system + - apiVersion: v1 + kind: Secret + type: kubernetes.io/service-account-token + metadata: + name: argocd-manager-token + namespace: kube-system + annotations: + kubernetes.io/service-account.name: argocd-manager + +# The token controller fills the Secret in a moment after it is created. Under +# --check nothing was created, so there is nothing to read. +- name: Read the Argo CD token and the cluster CA + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + kind: Secret + namespace: kube-system + name: argocd-manager-token + register: argocd_manager_secret + retries: 15 + delay: 2 + until: (argocd_manager_secret.resources | first | default({})).data.token is defined + when: not ansible_check_mode + no_log: true + +# Read back by hub.yml as hostvars[].argocd_manager_*. +- name: Keep the token and CA for the hub + ansible.builtin.set_fact: + argocd_manager_token: "{{ (argocd_manager_secret.resources | first).data.token | b64decode }}" + argocd_manager_ca: "{{ (argocd_manager_secret.resources | first).data['ca.crt'] }}" + when: not ansible_check_mode + no_log: true diff --git a/roles/monitoring/defaults/main.yml b/roles/monitoring/defaults/main.yml new file mode 100644 index 0000000..714996b --- /dev/null +++ b/roles/monitoring/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# Addresses outside the inventory that push metrics and logs to the hub too. +# The boxes in the inventory are always allowed. sf and sf2 (ipni/infra) go +# here when they move off grafana.ipni.io: +# monitoring_ingest_extra_sources: [103.232.206.165, 103.232.206.162] +monitoring_ingest_extra_sources: [] diff --git a/roles/monitoring/tasks/healthchecks.yml b/roles/monitoring/tasks/healthchecks.yml new file mode 100644 index 0000000..60e644d --- /dev/null +++ b/roles/monitoring/tasks/healthchecks.yml @@ -0,0 +1,11 @@ +--- +# Healthchecks (ipni/infra, kubernetes/manifests/ap/sing1/healthchecks) listens +# on host port 8000. Only Alertmanager on the hub sends to it; people reach the +# UI over an SSH tunnel. +- name: Allow the monitoring hub, and only the hub, to ping Healthchecks + ansible.builtin.include_role: + name: ufw_allowlist + vars: + ufw_allowlist_port: "8000" + ufw_allowlist_comment: healthchecks from hub + ufw_allowlist_sources: ["{{ hostvars[argocd_hub].ansible_host }}"] diff --git a/roles/monitoring/tasks/hub.yml b/roles/monitoring/tasks/hub.yml new file mode 100644 index 0000000..141f82a --- /dev/null +++ b/roles/monitoring/tasks/hub.yml @@ -0,0 +1,61 @@ +--- +# Traefik in the monitoring namespace listens on host port 8443 (443 is the +# route-origin Envoy's). It serves three hostnames: +# monitoring.ipni.io, argocd.ipni.io people, through Cloudflare's proxy +# ingest.monitoring.ipni.io Alloy on the other boxes, DNS only + +- name: Fail if there is no Cloudflare token for the monitoring certificate + ansible.builtin.assert: + that: vault_cloudflare_dns_token is defined and vault_cloudflare_dns_token | length > 0 + fail_msg: >- + vault_cloudflare_dns_token is not set (group_vars/ipfs_nodes/vault.yml). + cert-manager needs it for the monitoring certificate, as for the WSS one. + quiet: true + +# The same pinned list as the route-origin allowlist on 443; platform.yml reads +# it from roles/route_origin/defaults. +- name: Allow Cloudflare's edge to reach the monitoring hub + ansible.builtin.include_role: + name: ufw_allowlist + vars: + ufw_allowlist_port: "8443" + ufw_allowlist_comment: cloudflare monitoring + ufw_allowlist_sources: "{{ route_origin_cloudflare_ranges }}" + +- name: Allow the other boxes to push metrics and logs to the hub + ansible.builtin.include_role: + name: ufw_allowlist + vars: + ufw_allowlist_port: "8443" + ufw_allowlist_comment: monitoring ingest + ufw_allowlist_sources: >- + {{ groups['ipfs_nodes'] | reject('equalto', argocd_hub) + | map('extract', hostvars, 'ansible_host') | list + + monitoring_ingest_extra_sources }} + +# The Issuer in ipni/infra (kubernetes/manifests/eu/lith1/monitoring) reads +# this Secret. Created here so the token goes from the vault straight to the +# API server and is never in either repo. Argo CD creates the namespace too; +# whichever runs first wins, and neither deletes it. +- name: Ensure the monitoring namespace exists + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + api_version: v1 + kind: Namespace + name: monitoring + state: present + +- name: Install the Cloudflare DNS token for the monitoring certificate + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: cloudflare-api-token + namespace: monitoring + data: + api-token: "{{ vault_cloudflare_dns_token | b64encode }}" + no_log: true diff --git a/roles/monitoring/tasks/main.yml b/roles/monitoring/tasks/main.yml new file mode 100644 index 0000000..885c76f --- /dev/null +++ b/roles/monitoring/tasks/main.yml @@ -0,0 +1,12 @@ +--- +# What the monitoring stack in ipni/infra needs from the boxes and cannot +# create itself: firewall openings and the one credential that lives in this +# repo's vault. The stack itself is deployed by Argo CD (roles/argocd). + +- name: Prepare the monitoring hub + ansible.builtin.include_tasks: hub.yml + when: inventory_hostname == argocd_hub + +- name: Prepare the Healthchecks box + ansible.builtin.include_tasks: healthchecks.yml + when: inventory_hostname == healthchecks_host diff --git a/roles/ufw_allowlist/defaults/main.yml b/roles/ufw_allowlist/defaults/main.yml new file mode 100644 index 0000000..dca49f4 --- /dev/null +++ b/roles/ufw_allowlist/defaults/main.yml @@ -0,0 +1,8 @@ +--- +# Required, set by the including role: +# ufw_allowlist_port port to open, e.g. "6443" +# ufw_allowlist_comment ufw comment that marks the rules this call owns; +# must be unique across the box's rules +# ufw_allowlist_sources the addresses or ranges allowed to reach the port + +ufw_allowlist_proto: tcp diff --git a/roles/ufw_allowlist/tasks/main.yml b/roles/ufw_allowlist/tasks/main.yml new file mode 100644 index 0000000..73dda95 --- /dev/null +++ b/roles/ufw_allowlist/tasks/main.yml @@ -0,0 +1,40 @@ +--- +# Open one port to an exact list of sources, and keep ufw in step with that +# list in BOTH directions: a source dropped from the list loses its rule, so +# the port never stays open to an address nobody asked for. Only rules carrying +# ufw_allowlist_comment are considered, which is why it must be unique. +# +# The same approach as the Cloudflare allowlist in roles/route_origin, for the +# ports roles/argocd and roles/monitoring open to a handful of addresses. + +- name: "Allow {{ ufw_allowlist_port }}/{{ ufw_allowlist_proto }} from each source ({{ ufw_allowlist_comment }})" + community.general.ufw: + rule: allow + proto: "{{ ufw_allowlist_proto }}" + port: "{{ ufw_allowlist_port }}" + from_ip: "{{ item }}" + comment: "{{ ufw_allowlist_comment }}" + loop: "{{ ufw_allowlist_sources }}" + +# `ufw status` prints each rule as `/ ALLOW # `. +- name: "Read the {{ ufw_allowlist_comment }} rules currently in ufw" + ansible.builtin.shell: + cmd: >- + set -o pipefail; + ufw status | grep -F -- {{ ('# ' ~ ufw_allowlist_comment) | quote }} | + awk '{for (i = 1; i <= NF; i++) if ($i == "ALLOW") print $(i + 1)}' + executable: /bin/bash + register: ufw_allowlist_current + changed_when: false + check_mode: false + # grep exits 1 when there are no such rules yet, which is not an error. + failed_when: ufw_allowlist_current.rc not in [0, 1] + +- name: "Remove {{ ufw_allowlist_comment }} rules for sources no longer on the list" + community.general.ufw: + rule: allow + proto: "{{ ufw_allowlist_proto }}" + port: "{{ ufw_allowlist_port }}" + from_ip: "{{ item }}" + delete: true + loop: "{{ ufw_allowlist_current.stdout_lines | difference(ufw_allowlist_sources) }}" From df359364d76d97c30ba1934f662a9e2b58b0ae8b Mon Sep 17 00:00:00 2001 From: Takwa Date: Mon, 12 Oct 2026 19:12:31 +0200 Subject: [PATCH 2/7] Run platform.yml from CI .github/workflows/platform.yml dry-runs platform.yml (--check --diff) on pull requests and runs it on merge to main, once a required reviewer approves the `production` environment; it can also be started by hand, check or apply, with a --limit. It is the only playbook that runs from CI. CI has no vault files, so scripts/ci-platform.sh turns repository secrets into platform.yml's three vault variables (passed from a file, never on the command line), signs in with CI's own SSH key (keys/github-actions.pub, now in admin_pubkey_file so the hardening role keeps it) and only connects to the host keys pinned in inventory/known_hosts. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/platform.yml | 115 +++++++++++++++++++++++++++++++++ README.md | 47 ++++++++++---- group_vars/ipfs_nodes/main.yml | 7 +- inventory/known_hosts | 8 +++ keys/github-actions.pub | 1 + scripts/ci-platform.sh | 54 ++++++++++++++++ 6 files changed, 217 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/platform.yml create mode 100644 inventory/known_hosts create mode 100644 keys/github-actions.pub create mode 100755 scripts/ci-platform.sh diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml new file mode 100644 index 0000000..96ff9e7 --- /dev/null +++ b/.github/workflows/platform.yml @@ -0,0 +1,115 @@ +--- +# platform.yml from CI: Argo CD's first install, registering the boxes with +# it, and the firewall rules the monitoring stack needs (README, "Argo CD and +# monitoring"). The service playbooks still run from an operator's machine. +# +# pull request -> dry run (--check --diff) against the boxes +# merge to main -> the real run, once approved in the `production` +# environment (required reviewers) +# manual -> either, optionally limited to some boxes +# +# Pull requests from forks get no secrets and are skipped. +name: platform + +on: + pull_request: + paths: + - platform.yml + - roles/argocd/** + - roles/monitoring/** + - roles/ufw_allowlist/** + - roles/route_origin/defaults/main.yml + - group_vars/** + - inventory/** + - scripts/ci-platform.sh + - .github/workflows/platform.yml + push: + branches: [main] + paths: + - platform.yml + - roles/argocd/** + - roles/monitoring/** + - roles/ufw_allowlist/** + - roles/route_origin/defaults/main.yml + - group_vars/** + - inventory/** + - scripts/ci-platform.sh + - .github/workflows/platform.yml + workflow_dispatch: + inputs: + mode: + description: "check (dry run) or apply" + type: choice + options: [check, apply] + default: check + limit: + description: "Boxes to run on (ansible --limit), empty for all" + type: string + default: "" + +permissions: + contents: read + +# One run at a time against the boxes; never cancel one half-way. +concurrency: + group: platform + cancel-in-progress: false + +env: + # ansible.cfg sets the yaml callback, which no installed collection provides + # here (see ci.yml). + ANSIBLE_STDOUT_CALLBACK: default + ANSIBLE_FORCE_COLOR: "true" + +jobs: + check: + if: >- + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'workflow_dispatch' && inputs.mode == 'check') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Ansible and the collections the roles use + run: | + pip install ansible-core + ansible-galaxy collection install community.general ansible.posix kubernetes.core + - name: Dry run platform.yml + env: + ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} + ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }} + CLOUDFLARE_DNS_TOKEN: ${{ secrets.CLOUDFLARE_DNS_TOKEN }} + ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }} + LIMIT: ${{ inputs.limit }} + run: scripts/ci-platform.sh check "$LIMIT" + + apply: + if: >- + (github.event_name == 'push') || + (github.event_name == 'workflow_dispatch' && inputs.mode == 'apply') + runs-on: ubuntu-latest + # Required reviewers: nothing reaches the boxes until one approves. + environment: production + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install Ansible and the collections the roles use + run: | + pip install ansible-core + ansible-galaxy collection install community.general ansible.posix kubernetes.core + - name: Run platform.yml + env: + ANSIBLE_SSH_PRIVATE_KEY: ${{ secrets.ANSIBLE_SSH_PRIVATE_KEY }} + ARGOCD_AGE_KEY: ${{ secrets.ARGOCD_AGE_KEY }} + CLOUDFLARE_DNS_TOKEN: ${{ secrets.CLOUDFLARE_DNS_TOKEN }} + ARGOCD_REPO_SSH_KEY: ${{ secrets.ARGOCD_REPO_SSH_KEY }} + LIMIT: ${{ inputs.limit }} + run: scripts/ci-platform.sh apply "$LIMIT" diff --git a/README.md b/README.md index 108dedb..0de950e 100644 --- a/README.md +++ b/README.md @@ -1443,24 +1443,36 @@ for itself: ingest) from Cloudflare's edge and the other boxes, and 8000 on sing-1 (`healthchecks_host`, the hub's dead-man's switch) from lith-1. -```bash -ansible-playbook platform.yml --check --diff -ansible-playbook platform.yml -``` +**It runs from CI** (`.github/workflows/platform.yml`), the only playbook that +does; the others still run from an operator's machine: + +| When | What | +|---|---| +| Pull request touching it | `--check --diff` against the boxes (forks get no secrets and are skipped) | +| Merge to `main` | the real run, once a required reviewer approves it in the `production` environment | +| Actions → platform → Run workflow | `check` or `apply`, optionally `--limit`ed | It restarts nothing that serves traffic, so it runs every box in one batch: the hub registers the others using what they produced earlier in the same run. -Argo CD's age key is generated once with `age-keygen`; its public half goes in -ipni/infra's `.sops.yaml`, the private half into the vault: +CI has no vault files, so `platform.yml`'s vault variables come from +repository secrets (`scripts/ci-platform.sh` maps them), and it signs in with +its own SSH key, `keys/github-actions.pub` in `admin_pubkey_file`. It only +connects to the host keys pinned in `inventory/known_hosts`. -```bash -age-keygen -o argocd.agekey # prints the public key -ansible-vault edit group_vars/ipfs_nodes/vault.yml -# vault_argocd_age_key: | -# -rm argocd.agekey -``` +| Secret | Becomes | +|---|---| +| `ANSIBLE_SSH_PRIVATE_KEY` | CI's SSH key as the admin user | +| `ARGOCD_AGE_KEY` | `vault_argocd_age_key`: Argo CD's age key (public half in ipni/infra's `.sops.yaml`) | +| `CLOUDFLARE_DNS_TOKEN` | `vault_cloudflare_dns_token`: the same token cert-manager uses for WSS | +| `ARGOCD_REPO_SSH_KEY` | `vault_argocd_repo_ssh_key`: optional, only while ipni/infra is private | + +Copies of all of them are in 1Password (IPFS account, Compass vault): GitHub +cannot show them again. To revoke CI's access to the boxes, remove its line +from `admin_pubkey_file` and re-run the hardening role. + +From an operator's machine it still works as usual, with the same variables +in the vault: `ansible-playbook platform.yml --check --diff`. ## Secrets @@ -1513,6 +1525,9 @@ same path a deploy takes), so what is checked is what would be applied: | every `pinset.yml` section is in the role's `known_sections` | a section the content role silently ignores | | `shellcheck scripts/*.sh` | the usual shell traps | +`platform.yml` also has its own workflow, which does talk to the boxes (see +"Argo CD and monitoring"). + The Cloudflare range check is a separate weekly workflow rather than part of this one, because it reaches `api.cloudflare.com`: an outage there should not turn an unrelated pull request red. @@ -1673,7 +1688,11 @@ dns.txt Cloudflare-importable bootstrap DNS records (generated dns-libp2p.txt bootstrap.libp2p.io records for the legacy identities (generated) site.yml base preparation playbook k3s-upgrade.yml k3s upgrade, one minor version at a time, backup per step -platform.yml Argo CD (first install) and the monitoring stack's prerequisites +platform.yml Argo CD (first install) and the monitoring stack's prerequisites, run from CI +.github/workflows/platform.yml platform.yml: dry run on PRs, approved run on merge +scripts/ci-platform.sh CI's wrapper for platform.yml: secrets to variables, SSH key, pinned host keys +inventory/known_hosts the boxes' SSH host keys, pinned for CI +keys/github-actions.pub CI's SSH key (in admin_pubkey_file) routing.yml routing service playbook (needle + origin) bootstrap.yml bootstrap node playbook (with cert-manager) content.yml content cluster node playbook (deploy, load the pinset, wait for PINNED, summary) diff --git a/group_vars/ipfs_nodes/main.yml b/group_vars/ipfs_nodes/main.yml index 54229b5..2fc911f 100644 --- a/group_vars/ipfs_nodes/main.yml +++ b/group_vars/ipfs_nodes/main.yml @@ -14,7 +14,12 @@ # - ~/.ssh/id_ed25519.pub # - ~/.ssh/operators/molly.pub admin_user: ipni -admin_pubkey_file: ~/.ssh/id_ed25519.pub +admin_pubkey_file: + - ~/.ssh/id_ed25519.pub + # CI (.github/workflows/platform.yml). Its private key is the + # ANSIBLE_SSH_PRIVATE_KEY repository secret; to revoke CI's access, remove + # this line and re-run the hardening role. + - "{{ playbook_dir }}/keys/github-actions.pub" # --------------------------------------------------------------------------- # SSH hardening diff --git a/inventory/known_hosts b/inventory/known_hosts new file mode 100644 index 0000000..7b7a4ff --- /dev/null +++ b/inventory/known_hosts @@ -0,0 +1,8 @@ +# SSH host keys of the boxes, for CI (.github/workflows/platform.yml), which +# must refuse to connect to anything else. Public. Checked on 2026-10-12: the +# scanned key, the key file on each box and an operator's known_hosts agreed. +# Update when a box is reinstalled: ssh-keyscan -t ed25519 , then confirm +# against /etc/ssh/ssh_host_ed25519_key.pub on the box itself. +5.199.165.77 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK2+9yvOjU3tE3K7zEVHn5M30X3FHRA6mgkSvFjmH2uL +46.166.169.131 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBbz1MWGYfe2RzaH/SV+zBctwhUNWO0336K5N5x5ATOp +84.32.48.211 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGsaCZG1aU29AqxU84RhnxlMdYyvzz1f2hBhpaYu2gHR diff --git a/keys/github-actions.pub b/keys/github-actions.pub new file mode 100644 index 0000000..a6e27d5 --- /dev/null +++ b/keys/github-actions.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKLL+jM3SN0N+MtLKcpzBQZ+2RDHyJoibiuvZsD5CWRm github-actions@ipni/workers (platform.yml) diff --git a/scripts/ci-platform.sh b/scripts/ci-platform.sh new file mode 100755 index 0000000..28cc8b6 --- /dev/null +++ b/scripts/ci-platform.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Run platform.yml from CI (.github/workflows/platform.yml). +# +# scripts/ci-platform.sh check [limit] dry run: --check --diff +# scripts/ci-platform.sh apply [limit] the real run +# +# The vault files are gitignored, so CI has none: platform.yml's three vault +# variables come from repository secrets instead, in the environment: +# ARGOCD_AGE_KEY -> vault_argocd_age_key +# CLOUDFLARE_DNS_TOKEN -> vault_cloudflare_dns_token +# ARGOCD_REPO_SSH_KEY -> vault_argocd_repo_ssh_key (optional: only while +# ipni/infra is private) +# and SSH as the admin user from ANSIBLE_SSH_PRIVATE_KEY, whose public half is +# keys/github-actions.pub. Copies of every secret are in 1Password (IPFS +# account, Compass vault). +set -euo pipefail + +mode=${1:?usage: $0 check|apply [limit]} +limit=${2:-} +[[ $mode == check || $mode == apply ]] || { echo "mode must be check or apply" >&2; exit 2; } +: "${ANSIBLE_SSH_PRIVATE_KEY:?not set}" "${ARGOCD_AGE_KEY:?not set}" "${CLOUDFLARE_DNS_TOKEN:?not set}" + +cd "$(dirname "$0")/.." +umask 077 + +# Connect only to the boxes whose host keys are pinned in the repo. +mkdir -p "$HOME/.ssh" +printf '%s\n' "$ANSIBLE_SSH_PRIVATE_KEY" > "$HOME/.ssh/id_ed25519" +grep -v '^#' inventory/known_hosts > "$HOME/.ssh/known_hosts" + +# ansible.cfg names .vault_pass; nothing CI reads is vault-encrypted. +echo ci > .vault_pass + +# The secrets as extra vars, from a file rather than the command line, so they +# never appear in a process list or the log. +secrets=$(mktemp) +trap 'rm -f "$secrets"' EXIT +python3 - "$secrets" <<'PY' +import json, os, sys +values = { + "vault_argocd_age_key": os.environ["ARGOCD_AGE_KEY"], + "vault_cloudflare_dns_token": os.environ["CLOUDFLARE_DNS_TOKEN"], +} +if os.environ.get("ARGOCD_REPO_SSH_KEY"): + values["vault_argocd_repo_ssh_key"] = os.environ["ARGOCD_REPO_SSH_KEY"] +with open(sys.argv[1], "w") as f: + json.dump(values, f) # JSON is YAML, which -e @file reads +PY + +args=(platform.yml -e "@$secrets") +[[ $mode == check ]] && args+=(--check --diff) +[[ -n $limit ]] && args+=(--limit "$limit") + +ansible-playbook "${args[@]}" From 909a29f1bb0e7d381894d0aa38d60c4a3d7f4341 Mon Sep 17 00:00:00 2001 From: Takwa Date: Mon, 12 Oct 2026 19:22:14 +0200 Subject: [PATCH 3/7] List the operators' SSH keys explicitly, so hardening keeps them admin_pubkey_file was ~/.ssh/id_ed25519.pub: the key of whoever runs the playbook. The hardening role installs it exclusively, so the next run would have removed every other operator's key. The keys now on the boxes (takgn, nymd) are committed as keys/*.pub and listed with CI's, so the result is the same whoever runs it. Rendered with the role's own template: it keeps both existing keys, adds CI's, and removes nothing else. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 ++++-- README.md | 13 +++++++------ group_vars/ipfs_nodes/main.yml | 19 +++++++++---------- keys/nymd.pub | 1 + keys/takgn.pub | 1 + 5 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 keys/nymd.pub create mode 100644 keys/takgn.pub diff --git a/AGENTS.md b/AGENTS.md index 96d9a3e..310c8b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,8 +12,10 @@ three at once is a full outage of that service. ## Getting in -SSH is key-only as the hardened admin user `ipni` (`~/.ssh/id_ed25519`); root -login is disabled and the provider KVM console is the out-of-band fallback. +SSH is key-only as the hardened admin user `ipni`; root login is disabled and +the provider KVM console is the out-of-band fallback. The keys that may sign in +are the files in `keys/`, listed in `admin_pubkey_file` (takgn, nymd, and CI's +own key); the hardening role removes any other. The playbooks set `remote_user: "{{ admin_user }}"` themselves, so `ansible-playbook` needs no user flag. **Ad-hoc `ansible` commands do not**, and diff --git a/README.md b/README.md index 0de950e..aa267f7 100644 --- a/README.md +++ b/README.md @@ -111,16 +111,17 @@ with the OS disk. > `/data` existed would otherwise make the in-use data disk look blank. Once a > volume labelled `ipfsdata` exists, the format step is skipped outright. -**`hardening`** — creates the `ipni` admin user with your ed25519 key and +**`hardening`** — creates the `ipni` admin user with the operators' keys and passwordless sudo, then locks SSH down to key-only with root login disabled, enables ufw with a default-deny inbound policy, and configures fail2ban. > **The key files are the single source of truth.** `authorized_key` runs with > `exclusive: true`, so any key added to the `ipni` account by hand is removed -> on the next run. `admin_pubkey_file` in `group_vars` takes either one path or -> a list of them; to give a second operator access, add their key file to that -> list rather than editing the host. Do it **before** the run that would -> otherwise strip their key. +> on the next run. `admin_pubkey_file` in `group_vars` lists committed public +> key files in `keys/` (`takgn.pub`, `nymd.pub`, and CI's `github-actions.pub`), +> so the result does not depend on who runs it. To give someone access, commit +> their public key there and add it to the list rather than editing the host. +> Do it **before** the run that would otherwise strip their key. fail2ban is kept mainly to cut log noise: with password authentication disabled there are few credential guesses left for it to ban. @@ -1692,7 +1693,7 @@ platform.yml Argo CD (first install) and the monitoring stack's pre .github/workflows/platform.yml platform.yml: dry run on PRs, approved run on merge scripts/ci-platform.sh CI's wrapper for platform.yml: secrets to variables, SSH key, pinned host keys inventory/known_hosts the boxes' SSH host keys, pinned for CI -keys/github-actions.pub CI's SSH key (in admin_pubkey_file) +keys/*.pub the SSH keys allowed in as the admin user: takgn, nymd, CI (admin_pubkey_file) routing.yml routing service playbook (needle + origin) bootstrap.yml bootstrap node playbook (with cert-manager) content.yml content cluster node playbook (deploy, load the pinset, wait for PINNED, summary) diff --git a/group_vars/ipfs_nodes/main.yml b/group_vars/ipfs_nodes/main.yml index 2fc911f..d8f905f 100644 --- a/group_vars/ipfs_nodes/main.yml +++ b/group_vars/ipfs_nodes/main.yml @@ -6,19 +6,18 @@ # the ONLY way in over the network once hardening completes; the provider # KVM/web console is the designated out-of-band fallback. # -# One path, or a list of them when more than one operator needs in. The role -# installs them exclusively, so a key that is not listed here does not survive -# the next run - add the second operator BEFORE the run that would lock them -# out, not after: -# admin_pubkey_file: -# - ~/.ssh/id_ed25519.pub -# - ~/.ssh/operators/molly.pub +# The role installs these keys exclusively: a key that is not listed here is +# REMOVED on the next run. So every key is a committed public key file in +# keys/, never "whoever runs the playbook" (~/.ssh/id_ed25519.pub), which would +# lock out every other operator. To add someone, commit their public key here +# BEFORE the run that would lock them out; to remove someone, delete their line +# and re-run the hardening role (site.yml --tags hardening, one box at a time). admin_user: ipni admin_pubkey_file: - - ~/.ssh/id_ed25519.pub + - "{{ playbook_dir }}/keys/takgn.pub" + - "{{ playbook_dir }}/keys/nymd.pub" # CI (.github/workflows/platform.yml). Its private key is the - # ANSIBLE_SSH_PRIVATE_KEY repository secret; to revoke CI's access, remove - # this line and re-run the hardening role. + # ANSIBLE_SSH_PRIVATE_KEY repository secret. - "{{ playbook_dir }}/keys/github-actions.pub" # --------------------------------------------------------------------------- diff --git a/keys/nymd.pub b/keys/nymd.pub new file mode 100644 index 0000000..30b14a5 --- /dev/null +++ b/keys/nymd.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOXuNpM0vZTW1r1LcvunW5JURjaMOiku2AZZjgzF61s2 nymd diff --git a/keys/takgn.pub b/keys/takgn.pub new file mode 100644 index 0000000..02240f1 --- /dev/null +++ b/keys/takgn.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO6RgP8GfmKttUW2fTwEeBD6ObNGp5KX6Nmor1imMuyH takgn From b174f8138f110ef935aecbc22ca3062674299870 Mon Sep 17 00:00:00 2001 From: Takwa Date: Mon, 12 Oct 2026 19:30:49 +0200 Subject: [PATCH 4/7] Add a break-glass SSH key that lives only in 1Password So the boxes stay reachable when no operator key works. 1Password generated the key in the IPFS account's Compass vault; only its public half is here (keys/break-glass.pub, in admin_pubkey_file). Installed on all three boxes and tested: it logs in as ipni with sudo, with only that key offered and the host keys pinned. A hardening run now installs exactly the keys on the boxes. AGENTS.md explains when and how to use it. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 25 +++++++++++++++++++++++-- README.md | 7 ++++--- group_vars/ipfs_nodes/main.yml | 4 ++++ keys/break-glass.pub | 1 + 4 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 keys/break-glass.pub diff --git a/AGENTS.md b/AGENTS.md index 310c8b2..5aefa45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,8 +14,29 @@ three at once is a full outage of that service. SSH is key-only as the hardened admin user `ipni`; root login is disabled and the provider KVM console is the out-of-band fallback. The keys that may sign in -are the files in `keys/`, listed in `admin_pubkey_file` (takgn, nymd, and CI's -own key); the hardening role removes any other. +are the files in `keys/`, listed in `admin_pubkey_file` (takgn, nymd, the +break-glass key and CI's own key); the hardening role removes any other. + +### Break-glass access + +For when no operator key works (lost laptop, someone away, a bad hardening +run). The private key exists only in 1Password: IPFS account, Compass vault, +"IPNI boxes: break-glass SSH key (ipni user)". Use it without writing it to +disk, through a throwaway agent: + +```bash +eval "$(ssh-agent -s)" +op read --account team-ipfs.1password.com \ + "op://Compass/IPNI boxes: break-glass SSH key (ipni user)/private key?ssh-format=openssh" | ssh-add - +ssh -o IdentitiesOnly=yes -o IdentityFile=keys/break-glass.pub ipni@46.166.169.131 +ssh-agent -k +``` + +Then fix whatever locked people out, through the repo, not by hand. If the key +has been used outside a real emergency or may have leaked, replace it: a new +SSH Key item in Compass, its public half in `keys/break-glass.pub`, and a +hardening run. Last resort if SSH itself is down: the provider's KVM console, +with `vault_console_password`. The playbooks set `remote_user: "{{ admin_user }}"` themselves, so `ansible-playbook` needs no user flag. **Ad-hoc `ansible` commands do not**, and diff --git a/README.md b/README.md index aa267f7..2beb610 100644 --- a/README.md +++ b/README.md @@ -118,8 +118,9 @@ enables ufw with a default-deny inbound policy, and configures fail2ban. > **The key files are the single source of truth.** `authorized_key` runs with > `exclusive: true`, so any key added to the `ipni` account by hand is removed > on the next run. `admin_pubkey_file` in `group_vars` lists committed public -> key files in `keys/` (`takgn.pub`, `nymd.pub`, and CI's `github-actions.pub`), -> so the result does not depend on who runs it. To give someone access, commit +> key files in `keys/` (`takgn.pub`, `nymd.pub`, the 1Password-only +> `break-glass.pub`, and CI's `github-actions.pub`), so the result does not +> depend on who runs it. To give someone access, commit > their public key there and add it to the list rather than editing the host. > Do it **before** the run that would otherwise strip their key. @@ -1693,7 +1694,7 @@ platform.yml Argo CD (first install) and the monitoring stack's pre .github/workflows/platform.yml platform.yml: dry run on PRs, approved run on merge scripts/ci-platform.sh CI's wrapper for platform.yml: secrets to variables, SSH key, pinned host keys inventory/known_hosts the boxes' SSH host keys, pinned for CI -keys/*.pub the SSH keys allowed in as the admin user: takgn, nymd, CI (admin_pubkey_file) +keys/*.pub the SSH keys allowed in as the admin user: takgn, nymd, break-glass, CI (admin_pubkey_file) routing.yml routing service playbook (needle + origin) bootstrap.yml bootstrap node playbook (with cert-manager) content.yml content cluster node playbook (deploy, load the pinset, wait for PINNED, summary) diff --git a/group_vars/ipfs_nodes/main.yml b/group_vars/ipfs_nodes/main.yml index d8f905f..10621c9 100644 --- a/group_vars/ipfs_nodes/main.yml +++ b/group_vars/ipfs_nodes/main.yml @@ -16,6 +16,10 @@ admin_user: ipni admin_pubkey_file: - "{{ playbook_dir }}/keys/takgn.pub" - "{{ playbook_dir }}/keys/nymd.pub" + # Break-glass: only for when no operator key works. Its private half exists + # only in 1Password (IPFS account, Compass vault); see AGENTS.md. Never + # remove it without putting another emergency key in its place. + - "{{ playbook_dir }}/keys/break-glass.pub" # CI (.github/workflows/platform.yml). Its private key is the # ANSIBLE_SSH_PRIVATE_KEY repository secret. - "{{ playbook_dir }}/keys/github-actions.pub" diff --git a/keys/break-glass.pub b/keys/break-glass.pub new file mode 100644 index 0000000..074ba85 --- /dev/null +++ b/keys/break-glass.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINnPc7QVK8i7IdZpmF3Mup9rsCoaGWjlR9bqyubu06ni break-glass (1Password: IPFS / Compass) From f21291db6ab539cdf99f100044dbf3a47e52288b Mon Sep 17 00:00:00 2001 From: Takwa Date: Mon, 12 Oct 2026 19:31:35 +0200 Subject: [PATCH 5/7] Reference the break-glass key by its 1Password item ID op rejects the ':' in the item's title in a secret reference ("invalid character in secret reference"), and the CLI cannot rename SSH Key items. The ID works and survives a rename; tested. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5aefa45..8333b25 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,13 +21,15 @@ break-glass key and CI's own key); the hardening role removes any other. For when no operator key works (lost laptop, someone away, a bad hardening run). The private key exists only in 1Password: IPFS account, Compass vault, -"IPNI boxes: break-glass SSH key (ipni user)". Use it without writing it to -disk, through a throwaway agent: +item "IPNI boxes: break-glass SSH key (ipni user)", ID +`4ghgjny7l4rawld4spedu7mxz4`. The reference below uses the ID: the title's `:` +is not allowed in a 1Password reference, and the ID survives a rename. Use it +without writing it to disk, through a throwaway agent: ```bash eval "$(ssh-agent -s)" op read --account team-ipfs.1password.com \ - "op://Compass/IPNI boxes: break-glass SSH key (ipni user)/private key?ssh-format=openssh" | ssh-add - + "op://Compass/4ghgjny7l4rawld4spedu7mxz4/private key?ssh-format=openssh" | ssh-add - ssh -o IdentitiesOnly=yes -o IdentityFile=keys/break-glass.pub ipni@46.166.169.131 ssh-agent -k ``` From b9390b8ff1f71e1689c21f123d630573f5d7a908 Mon Sep 17 00:00:00 2001 From: Takwa Date: Tue, 13 Oct 2026 10:11:08 +0200 Subject: [PATCH 6/7] Give byo SSH access to the boxes byo's key (a hardware key, one of those on github.com/byo.keys) is now in keys/byo.pub and admin_pubkey_file, and installed on all three boxes. A hardening run installs exactly the five keys now on them. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- README.md | 4 ++-- group_vars/ipfs_nodes/main.yml | 1 + keys/byo.pub | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 keys/byo.pub diff --git a/AGENTS.md b/AGENTS.md index 8333b25..caa43a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ three at once is a full outage of that service. SSH is key-only as the hardened admin user `ipni`; root login is disabled and the provider KVM console is the out-of-band fallback. The keys that may sign in -are the files in `keys/`, listed in `admin_pubkey_file` (takgn, nymd, the +are the files in `keys/`, listed in `admin_pubkey_file` (takgn, nymd, byo, the break-glass key and CI's own key); the hardening role removes any other. ### Break-glass access diff --git a/README.md b/README.md index 2beb610..4cb292c 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ enables ufw with a default-deny inbound policy, and configures fail2ban. > **The key files are the single source of truth.** `authorized_key` runs with > `exclusive: true`, so any key added to the `ipni` account by hand is removed > on the next run. `admin_pubkey_file` in `group_vars` lists committed public -> key files in `keys/` (`takgn.pub`, `nymd.pub`, the 1Password-only +> key files in `keys/` (`takgn.pub`, `nymd.pub`, `byo.pub`, the 1Password-only > `break-glass.pub`, and CI's `github-actions.pub`), so the result does not > depend on who runs it. To give someone access, commit > their public key there and add it to the list rather than editing the host. @@ -1694,7 +1694,7 @@ platform.yml Argo CD (first install) and the monitoring stack's pre .github/workflows/platform.yml platform.yml: dry run on PRs, approved run on merge scripts/ci-platform.sh CI's wrapper for platform.yml: secrets to variables, SSH key, pinned host keys inventory/known_hosts the boxes' SSH host keys, pinned for CI -keys/*.pub the SSH keys allowed in as the admin user: takgn, nymd, break-glass, CI (admin_pubkey_file) +keys/*.pub the SSH keys allowed in as the admin user: takgn, nymd, byo, break-glass, CI (admin_pubkey_file) routing.yml routing service playbook (needle + origin) bootstrap.yml bootstrap node playbook (with cert-manager) content.yml content cluster node playbook (deploy, load the pinset, wait for PINNED, summary) diff --git a/group_vars/ipfs_nodes/main.yml b/group_vars/ipfs_nodes/main.yml index 10621c9..60a556b 100644 --- a/group_vars/ipfs_nodes/main.yml +++ b/group_vars/ipfs_nodes/main.yml @@ -16,6 +16,7 @@ admin_user: ipni admin_pubkey_file: - "{{ playbook_dir }}/keys/takgn.pub" - "{{ playbook_dir }}/keys/nymd.pub" + - "{{ playbook_dir }}/keys/byo.pub" # Break-glass: only for when no operator key works. Its private half exists # only in 1Password (IPFS account, Compass vault); see AGENTS.md. Never # remove it without putting another emergency key in its place. diff --git a/keys/byo.pub b/keys/byo.pub new file mode 100644 index 0000000..b17eb1b --- /dev/null +++ b/keys/byo.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC3DF4HILnhYWbTZb0XdD7R8STH+bnTzxU9HartE4M2L byo (hardware key) From f201bfebf5044b8090d1cfbcdfd7ddf1af72d519 Mon Sep 17 00:00:00 2001 From: Takwa Date: Tue, 13 Oct 2026 13:06:52 +0200 Subject: [PATCH 7/7] Let platform.yml dry-run before Argo CD is installed The first --check run in CI failed on lith-1: with Argo CD not installed, the argocd namespace does not exist yet, so the API server rejected the dry-run create of Argo CD's age key ("namespaces \"argocd\" not found"). The monitoring role's Cloudflare token Secret would have hit the same thing. Under --check, both now skip their Secrets while the namespace is missing and say so, as route_origin does for its TLS secret. A real run is unchanged. Dry-run against the three boxes from an operator's machine: failed=0. Co-Authored-By: Claude Opus 5.5 --- roles/argocd/tasks/hub.yml | 13 +++++++++++-- roles/monitoring/tasks/hub.yml | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/roles/argocd/tasks/hub.yml b/roles/argocd/tasks/hub.yml index 6ef1f90..37f6964 100644 --- a/roles/argocd/tasks/hub.yml +++ b/roles/argocd/tasks/hub.yml @@ -23,9 +23,14 @@ # Under --check there is nothing to dry-run against yet: no Helm on the box, and # no Argo CD CRDs for the root Application. Say so instead. +# The argocd namespace only exists once the Helm release does, so the Secrets +# below cannot be dry-run either (the API server rejects them: namespace not +# found). Say so instead, like route_origin's TLS secret. - name: Note that a first install is not dry-run ansible.builtin.debug: - msg: "Argo CD is not installed; a real run would install chart {{ argocd_chart_version }} and create the root Application." + msg: >- + Argo CD is not installed; a real run would install chart {{ argocd_chart_version }}, + then its age key, the cluster and repository Secrets, and the root Application. when: - ansible_check_mode - not argocd_installed.stdout @@ -97,6 +102,7 @@ namespace: argocd data: keys.txt: "{{ vault_argocd_age_key | b64encode }}" + when: not (ansible_check_mode and not argocd_installed.stdout) no_log: true # One Secret per box, named after the box. ipni/infra targets boxes by these @@ -140,6 +146,7 @@ loop: "{{ argocd_clusters }}" loop_control: label: "{{ item.name }} {{ item.server }}" + when: not (ansible_check_mode and not argocd_installed.stdout) no_log: true - name: Give Argo CD read access to the infra repository @@ -159,7 +166,9 @@ type: "{{ 'git' | b64encode }}" url: "{{ argocd_repo_url | b64encode }}" sshPrivateKey: "{{ vault_argocd_repo_ssh_key | b64encode }}" - when: vault_argocd_repo_ssh_key is defined + when: + - vault_argocd_repo_ssh_key is defined + - not (ansible_check_mode and not argocd_installed.stdout) no_log: true # The hand-off: the root Application points Argo CD at ipni/infra, whose apps diff --git a/roles/monitoring/tasks/hub.yml b/roles/monitoring/tasks/hub.yml index 141f82a..9d2f838 100644 --- a/roles/monitoring/tasks/hub.yml +++ b/roles/monitoring/tasks/hub.yml @@ -37,6 +37,13 @@ # this Secret. Created here so the token goes from the vault straight to the # API server and is never in either repo. Argo CD creates the namespace too; # whichever runs first wins, and neither deletes it. +- name: Check whether the monitoring namespace exists yet + ansible.builtin.command: + argv: [k3s, kubectl, get, namespace, monitoring, --ignore-not-found, -o, name] + register: monitoring_ns + changed_when: false + check_mode: false + - name: Ensure the monitoring namespace exists kubernetes.core.k8s: kubeconfig: /etc/rancher/k3s/k3s.yaml @@ -58,4 +65,7 @@ namespace: monitoring data: api-token: "{{ vault_cloudflare_dns_token | b64encode }}" + # Under --check on a first run the namespace above is only simulated, and a + # dry-run create of the Secret fails with "namespace not found". + when: not (ansible_check_mode and not monitoring_ns.stdout) no_log: true