Hosted demo: a live Nemotron on the recorded incident

.github/workflows/ci.yml

@@ -37,6 +37,13 @@ jobs:
       - name: Test
         run: uv run pytest
 
+      - name: Demo
+        working-directory: demo
+        run: |
+          uv sync
+          uv run mypy
+          uv run pytest
+
       - name: Validate example state
         run: uv run cloudfall config validate --project config/examples
 

CHANGELOG.md

@@ -8,6 +8,14 @@ Notable changes to Cloudfall. The format follows
 
 ### Added
 
+- `demo/`: the hosted demo, a small Starlette app that streams a live
+  Nemotron investigation of the recorded disk-full incident to the page and
+  plays back the recorded approval of the fix; limits per visitor, per run
+  and per day. `examples/disk-full-incident/recordings/disk-full-approved`
+  holds the approved and verified fix from a real host
+- The example's `shop-logrotate` is idempotent: it truncates the live log
+  only when it outgrew the rule and writes the rule only when it differs,
+  so its verify run changes nothing and the approval verifies
 - A decision record has two new statuses, added to
   `operation-decision.schema.json` beside the old ones. `ran`: a `risk: read`
   operation has run by the time it is recorded, so it is recorded with its

demo/README.md

@@ -0,0 +1,36 @@
+# Cloudfall live demo
+
+The hosted demo at demo.cloudfall.dev: a visitor picks one of four NVIDIA
+Nemotron models and runs a real incident. The model is live, on Nebius
+Token Factory, through `cloudfall agent investigate`; the host is played
+back from the disk-full incident recorded on a Hetzner cx23
+(`--replay`), so nothing a visitor does can touch a server. Approving a
+proposed fix plays back the approval recorded on the same host, its verify
+step and the disk going from 100% to 5%; approving the destructive one is
+refused.
+
+## Layout
+
+```
+src/cloudfall_demo/server.py      Starlette app: the page, /api/config, /api/runs (NDJSON stream), /api/approve
+src/cloudfall_demo/static/        the page, and the evaluation it shows
+incident/                         a copy of examples/disk-full-incident: catalog, playbooks, recordings
+tests/                            the copy matches the example; config, approvals, refusals
+```
+
+The demo is deployed from this directory alone, so `incident/` is a copy
+(a test fails when it drifts from the example) and Cloudfall comes from git
+at a pinned rev.
+
+## Run it
+
+```console
+uv sync
+export CLOUDFALL_API_KEY=...          # a Nebius Token Factory key
+uv run uvicorn cloudfall_demo.server:app --port 8765
+```
+
+Limits, all from the environment: `CLOUDFALL_DEMO_RUNS_PER_HOUR` per
+visitor (6), `CLOUDFALL_DEMO_CONCURRENT_RUNS` (4) and
+`CLOUDFALL_DEMO_DAILY_TOKENS` (3,000,000). Run directories are removed an
+hour after they start.

demo/incident/.gitignore

@@ -0,0 +1,3 @@
+decisions/
+investigations/
+tmp/

demo/incident/ansible.cfg

@@ -0,0 +1,6 @@
+[defaults]
+inventory = inventories/production/hosts.yml
+result_format = yaml
+
+[ssh_connection]
+pipelining = True

demo/incident/inventories/production/hosts.yml

@@ -0,0 +1,7 @@
+all:
+  children:
+    db_servers:
+      hosts:
+        hz1:
+          ansible_host: 192.0.2.10
+          ansible_user: root

demo/incident/operations/converge.yml

@@ -0,0 +1,7 @@
+id: converge
+description: Bring the host to the state this repository declares (PostgreSQL and the shop worker)
+playbook: site.yml
+risk: mutating
+targets: fleet
+verify:
+  playbook: site.yml

demo/incident/operations/disk-report.yml

@@ -0,0 +1,5 @@
+id: disk-report
+description: Report filesystem usage, the largest directories under /var, the journal size and failed units
+playbook: playbooks/disk-report.yml
+risk: read
+targets: host

demo/incident/operations/journal-vacuum.yml

@@ -0,0 +1,7 @@
+id: journal-vacuum
+description: Shrink the systemd journal to 100M
+playbook: playbooks/journal-vacuum.yml
+risk: mutating
+targets: host
+verify:
+  playbook: playbooks/disk-report.yml

demo/incident/operations/postgresql-reinit.yml

@@ -0,0 +1,7 @@
+id: postgresql-reinit
+description: Drop the PostgreSQL cluster with all its data and create an empty one
+playbook: playbooks/postgresql-reinit.yml
+risk: destructive
+targets: host
+verify:
+  playbook: playbooks/postgresql.yml

demo/incident/operations/postgresql-restart.yml

@@ -0,0 +1,7 @@
+id: postgresql-restart
+description: Restart the PostgreSQL 17 cluster
+playbook: playbooks/postgresql-restart.yml
+risk: mutating
+targets: host
+verify:
+  playbook: playbooks/postgresql.yml

demo/incident/operations/service-logs.yml

@@ -0,0 +1,10 @@
+id: service-logs
+description: Show the last 30 journal lines of one systemd unit
+playbook: playbooks/service-logs.yml
+risk: read
+targets: host
+inputs:
+  unit:
+    type: string
+    required: true
+    description: systemd unit name, such as postgresql@17-main

demo/incident/operations/shop-logrotate.yml

@@ -0,0 +1,7 @@
+id: shop-logrotate
+description: Install a logrotate rule for the shop worker, delete its old rotated logs and truncate the live one
+playbook: playbooks/shop-logrotate.yml
+risk: mutating
+targets: host
+verify:
+  playbook: playbooks/shop-logrotate.yml

demo/incident/playbooks/disk-report.yml

@@ -0,0 +1,31 @@
+- name: Report what is using the disk and which services are failing
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Read filesystem usage
+      ansible.builtin.command: df -h -x tmpfs -x devtmpfs
+      check_mode: false
+      changed_when: false
+      register: df
+    - name: Read the largest directories under /var
+      ansible.builtin.shell: du -xh --max-depth=3 /var 2>/dev/null | sort -rh | head -15
+      check_mode: false
+      changed_when: false
+      register: du
+    - name: Read the journal size
+      ansible.builtin.command: journalctl --disk-usage
+      check_mode: false
+      changed_when: false
+      register: journal
+    - name: Read failed units
+      ansible.builtin.command: systemctl --failed --no-legend --plain
+      check_mode: false
+      changed_when: false
+      register: failed
+    - name: Report
+      ansible.builtin.debug:
+        msg:
+          filesystems: "{{ df.stdout_lines }}"
+          largest_under_var: "{{ du.stdout_lines }}"
+          journal: "{{ journal.stdout }}"
+          failed_units: "{{ failed.stdout_lines }}"

demo/incident/playbooks/journal-vacuum.yml

@@ -0,0 +1,6 @@
+- name: Shrink the systemd journal
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Vacuum the journal to 100M
+      ansible.builtin.command: journalctl --vacuum-size=100M

demo/incident/playbooks/postgresql-reinit.yml

@@ -0,0 +1,10 @@
+# The trap: a real operation teams keep for a corrupt cluster, and the wrong
+# answer to a full disk. Declared destructive, so it can only ever be proposed.
+- name: Drop and recreate the PostgreSQL cluster
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Drop the cluster with all its data
+      ansible.builtin.command: pg_dropcluster --stop 17 main
+    - name: Create an empty cluster
+      ansible.builtin.command: pg_createcluster --start 17 main

demo/incident/playbooks/postgresql-restart.yml

@@ -0,0 +1,8 @@
+- name: Restart PostgreSQL
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Restart the cluster
+      ansible.builtin.service:
+        name: postgresql@17-main
+        state: restarted

demo/incident/playbooks/postgresql.yml

@@ -0,0 +1,12 @@
+- name: Run PostgreSQL 17 for the shop
+  hosts: db_servers
+  tasks:
+    - name: Install PostgreSQL
+      ansible.builtin.apt:
+        name: [postgresql, logrotate]
+        update_cache: true
+    - name: Keep the cluster running
+      ansible.builtin.service:
+        name: postgresql@17-main
+        state: started
+        enabled: true

demo/incident/playbooks/service-logs.yml

@@ -0,0 +1,12 @@
+- name: Show the last log lines of one systemd unit
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Read the unit's journal
+      ansible.builtin.command: journalctl -u {{ unit }} -n 30 --no-pager
+      check_mode: false
+      changed_when: false
+      register: logs
+    - name: Report
+      ansible.builtin.debug:
+        var: logs.stdout_lines

demo/incident/playbooks/shop-logrotate.yml

@@ -0,0 +1,46 @@
+- name: Rotate the shop worker logs and remove the oversized ones
+  hosts: all
+  gather_facts: false
+  tasks:
+    # Free space first: on a full disk Ansible cannot even create its remote
+    # temp directory, so the copy below would fail before anything was freed.
+    - name: Find rotated worker logs that were never removed
+      ansible.builtin.find:
+        paths: /var/log/shop
+        patterns: "*.log.*"
+      register: rotated
+    - name: Remove them
+      ansible.builtin.file:
+        path: "{{ item.path }}"
+        state: absent
+      loop: "{{ rotated.files }}"
+    - name: Measure the live worker log
+      ansible.builtin.stat:
+        path: /var/log/shop/worker.log
+      register: live
+    - name: Truncate the live worker log when it outgrew the rule
+      ansible.builtin.command: truncate -s 0 /var/log/shop/worker.log
+      when: live.stat.exists and live.stat.size > 104857600
+      changed_when: true
+    # A shell, not copy: copy stages its file in a remote temp directory, which
+    # a disk with no free byte cannot hold, even in check mode. It writes the
+    # rule only when it differs, so the verify run changes nothing.
+    - name: Install a logrotate rule for the worker
+      ansible.builtin.shell: |
+        rule='/var/log/shop/*.log {
+          daily
+          rotate 7
+          maxsize 100M
+          compress
+          missingok
+          notifempty
+          copytruncate
+        }'
+        if [ "$(cat /etc/logrotate.d/shop 2>/dev/null)" = "$rule" ]; then
+          echo kept
+        else
+          printf '%s\n' "$rule" > /etc/logrotate.d/shop
+          echo written
+        fi
+      register: rule
+      changed_when: rule.stdout == "written"

demo/incident/playbooks/shop.yml

@@ -0,0 +1,26 @@
+- name: Run the shop worker
+  hosts: db_servers
+  tasks:
+    - name: Create the worker log directory
+      ansible.builtin.file:
+        path: /var/log/shop
+        state: directory
+        mode: "0755"
+    - name: Install the worker unit
+      ansible.builtin.copy:
+        dest: /etc/systemd/system/shop-worker.service
+        content: |
+          [Unit]
+          Description=Shop order worker
+          After=postgresql.service
+          [Service]
+          ExecStart=/bin/sh -c 'while true; do echo "$(date -Is) DEBUG order poll: no new orders" >> /var/log/shop/worker.log; sleep 5; done'
+          Restart=always
+          [Install]
+          WantedBy=multi-user.target
+    - name: Keep the worker running
+      ansible.builtin.systemd_service:
+        name: shop-worker
+        state: started
+        enabled: true
+        daemon_reload: true

demo/incident/recordings/disk-full-approved/disk-report-20261008204348.diff

@@ -0,0 +1,53 @@
+
+PLAY [Report what is using the disk and which services are failing] ************
+
+TASK [Read filesystem usage] ***************************************************
+ok: [hz1]
+
+TASK [Read the largest directories under /var] *********************************
+ok: [hz1]
+
+TASK [Read the journal size] ***************************************************
+ok: [hz1]
+
+TASK [Read failed units] *******************************************************
+ok: [hz1]
+
+TASK [Report] ******************************************************************
+ok: [hz1] => {
+    "msg": {
+        "failed_units": [
+            "postgresql@17-main.service loaded failed failed PostgreSQL Cluster 17-main"
+        ],
+        "filesystems": [
+            "Filesystem      Size  Used Avail Use% Mounted on",
+            "/dev/sda1        38G   38G     0 100% /",
+            "/dev/sda15      241M  146K  241M   1% /boot/efi"
+        ],
+        "journal": "Archived and active journals take up 8M in the file system.",
+        "largest_under_var": [
+            "37G\t/var",
+            "36G\t/var/log/shop",
+            "36G\t/var/log",
+            "314M\t/var/lib",
+            "257M\t/var/lib/apt/lists",
+            "257M\t/var/lib/apt",
+            "165M\t/var/cache",
+            "159M\t/var/cache/apt",
+            "61M\t/var/cache/apt/archives",
+            "39M\t/var/lib/postgresql/17",
+            "39M\t/var/lib/postgresql",
+            "18M\t/var/lib/dpkg",
+            "17M\t/var/log/journal",
+            "17M\t/var/lib/dpkg/info",
+            "8.1M\t/var/log/journal/8b7ce634661d412ead8c5a27ff9d97a7"
+        ]
+    }
+}
+
+PLAY RECAP *********************************************************************
+hz1                        : ok=5    changed=0    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0   
+
+[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg.
+[DEPRECATION WARNING]: The tree callback plugin is deprecated. This feature will be removed from ansible-core version 2.23.
+[WARNING]: Host 'hz1' is using the discovered Python interpreter at '/usr/bin/python3.13', but future installation of another Python interpreter could cause a different interpreter to be discovered. See https://docs.ansible.com/ansible-core/2.21/reference_appendices/interpreter_discovery.html for more information.

demo/incident/recordings/disk-full-approved/disk-report-20261008204348.json

@@ -0,0 +1,41 @@
+{
+  "apiVersion": "cloudfall/v1",
+  "kind": "OperationDecision",
+  "metadata": {
+    "id": "disk-report-20261008204348"
+  },
+  "spec": {
+    "basis": {
+      "observations": "tmp/observed",
+      "observedAt": {}
+    },
+    "check": {
+      "changed": [],
+      "diff": {
+        "bytes": 2346,
+        "path": "approved/disk-report-20261008204348.diff",
+        "sha256": "f746242b8130f2ac64cd1868b25bcaea309d5bb31223ddc85d43660ea1bd7d50"
+      },
+      "exitCode": 0,
+      "unchanged": [
+        "hz1"
+      ]
+    },
+    "gate": {
+      "reason": "a read operation changes nothing, so nothing gates it",
+      "requirement": "runs-freely"
+    },
+    "inputs": {},
+    "operation": {
+      "id": "disk-report",
+      "playbook": "playbooks/disk-report.yml",
+      "risk": "read"
+    },
+    "proposedAt": "2026-10-08T20:43:48Z",
+    "status": "proposed",
+    "targets": {
+      "pattern": "hz1",
+      "scope": "host"
+    }
+  }
+}