diff --git a/.gitignore b/.gitignore index d646a9cd..b1fcae02 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ reports/ traces/ users/ saved_traces/ +result +result-* diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..4cfbe867 --- /dev/null +++ b/flake.lock @@ -0,0 +1,86 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1790898470, + "narHash": "sha256-zTwuwezD0w3hpga6a6P8emidzAZFyWqlNeRBlpWOOl8=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "024633cd702b10285db5cb19b40ad48d2399ba60", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "microvm": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "spectrum": "spectrum" + }, + "locked": { + "lastModified": 1791551030, + "narHash": "sha256-jTx3O3MKqPjv+cDfBFQ5zlKECBEwzF91scC+mSumZyE=", + "owner": "astro", + "repo": "microvm.nix", + "rev": "f13928fe9900e91709d7ce581eefa47eaafa79e1", + "type": "github" + }, + "original": { + "owner": "astro", + "repo": "microvm.nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1792502884, + "narHash": "sha256-IoNisKcGbUjdd45qCqywGMCYmZ5EnkxezvLII0FWwNA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "02c087f3fe673c9e317f206a627cb6b910336dbe", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "microvm": "microvm", + "nixpkgs": "nixpkgs" + } + }, + "spectrum": { + "flake": false, + "locked": { + "lastModified": 1786032750, + "narHash": "sha256-+Ba/kIn5BjvV7PN8bWOo7RoqgTY/Yv2M7qeMf3wS2Ag=", + "ref": "refs/heads/main", + "rev": "77d2389874bb092b8dbf05ecc5b1030372e80e98", + "revCount": 1417, + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + }, + "original": { + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..820d78b5 --- /dev/null +++ b/flake.nix @@ -0,0 +1,95 @@ +# +# flake.nix for Homa (HomaModule) — a modular Nix layer over the out-of-tree kernel module. +# +# Design & rationale: nix/docs/DESIGN.md +# Status + findings: nix/STATUS.md +# Quick start: nix/README.md +# +# ── Usage ────────────────────────────────────────────────────────────────────────────── +# nix develop # dev shell (toolchain + analysis tools) +# +# nix build .#homa-module # build homa.ko (release) against the pinned kernel +# nix build .#homa-module-debug # build homa.ko (debug: -g -O1) +# nix build .#homa-module-6_12 # per kernel: 6_1 6_6 6_12 6_18 7_3 (nixpkgs), +# # 7_0 7_1 7_2 (kernel.org source, compile only) +# nix build .#homa-build-matrix # every kernel above -> summary.md (never fails) +# nix build .#homa-module-riscv64 # cross-compiled homa.ko: riscv64, aarch64 (+ -strip) +# nix build .#homa-cross-matrix # both arches: homa.ko + util/ report -> summary.md +# nix build .#linux-kdir-riscv64-7_3 # the prepared kernel tree a cross build uses (+ aarch64) +# HOMA_LINUX_SRC=~/Downloads/linux nix build --impure .#homa-module-local +# nix build .#homa-utils # user-space tools from util/ +# nix build .#homa-utils-debug # user-space tools with AddressSanitizer +# nix build .#homa-tests # build + run the unit tests (ASan) +# +# nix run .#homa-microvm-test # boot a microVM, insmod homa.ko, loopback echo RPCs +# nix run .#homa-microvm-test-6_12 # ... per VM kernel: 6_12 7_3 (kernel/config.nix vmSeries) +# nix run .#homa-microvm-test-debug # ... under a KASAN/lockdep/UBSAN/KMEMLEAK kernel +# nix run .#homa-microvm-test-kcsan # ... under KCSAN (data races in Homa) +# nix run .#homa-pair-lifecycle-debug # the pair lifecycle on those kernels (also -kcsan) +# nix run .#homa-lifecycle[-6_12] # timed phases driven over ttyS0 + a virtio console +# nix run .#homa-pair-test # two VMs: echo + cp_node (Homa, TCP) over a link +# nix run .#homa-pair-test-6_12-7_3 # ... server kernel, client kernel (every pairing) +# nix run .#homa-pair-lifecycle[-6_12] # the pair over its consoles: netem loss, Homa qdisc +# nix run .#homa-pair-exp # latency sweep (sysctls x workloads, metric deltas) +# nix run .#homa-vm-matrix # every VM test + pair in parallel, summary table +# +# nix build .#homa-image # OCI userspace image (docker load < result) +# nix build .#homa-image-debug +# +# nix build .#analysis-quick # python + shell + clang-format (no kernel needed) +# nix build .#analysis-standard # + clang-tidy + cppcheck +# nix build .#analysis-deep # + sparse (endian) + smatch +# nix build .#analysis-netdev # netdev's checks on the code as submitted +# # (checkpatch, kernel-doc, W=1 gcc/clang/i386) +# nix build .#homa-tests-coverage # unit tests under gcov: line/branch coverage +# +# nix build .#homa-evidence # every check above in one report (evidence.html) +# nix run .#homa-evidence-full # ... plus the VM matrix (and sanitizer VMs) +# +# nix build .#fuzz # build all libFuzzer harnesses +# FUZZ_TIME=300 ./result/bin/run-fuzzers # ... then run them for 5 min +# nix build .#fuzz-run # bounded run -> summary.txt / count.txt / crashes +# nix run .#fuzz-deep -- [outdir] # fork mode + dictionary (FUZZ_DEEP_TIME) +# nix build .#fuzz-cov # coverage of the wire (de)serialization functions +# nix build .#fuzz-selftest # canary gate (fails if the harness is vacuous) +# +# nix flake show # list every output +# nix flake check # evaluate/build the checks +# +# If flakes are not enabled globally: +# nix --extra-experimental-features 'nix-command flakes' develop . +# ─────────────────────────────────────────────────────────────────────────────────────── +{ + description = "Homa: modular Nix flake (module + userspace derivations, microVM, static analysis, fuzzing)"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-parts = { + url = "github:hercules-ci/flake-parts"; + inputs.nixpkgs-lib.follows = "nixpkgs"; + }; + microvm = { + url = "github:astro/microvm.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + # flake-parts merges the modules in nix/parts/, one per area, each adding its packages + # and checks for every system (nix/parts/README.md). nix/scope.nix builds the + # derivations themselves. + outputs = inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + # Homa is a Linux kernel module; restrict to Linux systems (nixpkgs unstable also no + # longer supports x86_64-darwin). + systems = [ "x86_64-linux" "aarch64-linux" ]; + imports = [ + ./nix/parts/scope.nix + ./nix/parts/build.nix + ./nix/parts/analysis.nix + ./nix/parts/fuzz.nix + ./nix/parts/vms.nix + ./nix/parts/evidence.nix + ./nix/parts/dev.nix + ]; + }; +} diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 00000000..211cad17 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,288 @@ +# Homa — Nix flake + +This directory holds a modular [Nix](https://nixos.org) setup for the Homa kernel module. One +command gives you a working toolchain. Others build `homa.ko` against many kernels, boot each +kernel in a microVM and run Homa traffic in it, run static analysis, and fuzz the wire +protocol. + +## Goals + +- **Easy onboarding:** build and test Homa on any Linux distro without hunting for kernel + headers or matching compiler versions. +- **Reproducibility:** `flake.lock` pins every input (nixpkgs, kernels, tools), so a build or + test result means the same thing on every machine. No more "it worked on my machine". +- **Test many kernels:** Homa tracks a moving kernel API. The flake builds the module against + every kernel series below and boots real kernels to load and exercise it. +- **No effect on the host:** nothing is installed system-wide. Tools exist only inside + `nix develop` or a `nix build`/`nix run`. + +> ⚠️ **Linux only.** The microVM targets also need KVM (`/dev/kvm` readable and writable by +> your user). +> +> The flake lives on the `nix-flake` branch of +> [randomizedcoder/HomaModule](https://github.com/randomizedcoder/HomaModule/tree/nix-flake), +> rebased onto upstream `main`. It is not part of upstream. + +- **Design and rationale:** [`docs/DESIGN.md`](docs/DESIGN.md) +- **Status, findings and follow-ups:** [`STATUS.md`](STATUS.md) +- **Next: Homa traffic between pairs of VMs:** [`microvms/PLAN-pairs.md`](microvms/PLAN-pairs.md) + +--- + +## Quick start + +### 1. Install Nix + +Pick a **multi-user** (daemon) or **single-user** install: + +- [Multi-user](https://nix.dev/manual/nix/2.24/installation/#multi-user), recommended on most + distros: + ```sh + bash <(curl -L https://nixos.org/nix/install) --daemon + ``` +- [Single-user](https://nix.dev/manual/nix/2.24/installation/#single-user), no daemon, with the + store owned by your user: + ```sh + bash <(curl -L https://nixos.org/nix/install) --no-daemon + ``` + +#### Video walkthroughs + +| Platform | Video | +|---|---| +| Ubuntu | [Installing Nix on Ubuntu](https://youtu.be/cb7BBZLhuUY) | +| Fedora | [Installing Nix on Fedora](https://youtu.be/RvaTxMa4IiY) | + +### 2. Enable flakes (once) + +```sh +test -d /etc/nix || sudo mkdir /etc/nix +echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf +``` + +Or per command: `nix --extra-experimental-features 'nix-command flakes' develop`. +See also the [Nix flakes wiki](https://nixos.wiki/wiki/flakes). + +### 3. Get the code and enter the dev shell + +```sh +git clone -b nix-flake https://github.com/randomizedcoder/HomaModule.git +cd HomaModule +nix develop +``` + +The shell has the kernel build tree for the default kernel (`$KDIR`), the compilers, the +analysis tools, and helper functions: `homa-build`, `homa-test`, `homa-clean`, +`homa-checkpatch`, `homa-lint`. + +The first run downloads and builds the dependencies, which can take a few minutes. After that +they come from `/nix/store` and start almost instantly. + +### 4. Try it: boot a kernel and run Homa + +```sh +nix run .#homa-microvm-test-7_3 +``` + +This boots a 7.3 kernel in a microVM, loads `homa.ko`, runs echo RPCs over loopback and then +between two network namespaces with GRO and RPS on, unloads the module, and checks dmesg: + +``` +HOMA_SMOKE_START kernel=7.3.0 +HOMA_SMOKE_INSMOD_OK +HOMA_SMOKE_LSMOD_OK +HOMA_SMOKE_ECHO responses=6 (want 6) +HOMA_SMOKE_GRO responses=6 (want 6) homa_gro_receive calls=1724 homa_set_softirq_cpu calls=464 +HOMA_SMOKE_RMMOD_OK +HOMA_SMOKE_DMESG_OK +HOMA_SMOKE_PASS +VERDICT: PASS +``` + +The whole run takes about 15 seconds once the kernel is in the store. + +### 5. Homa between two VMs + +```sh +nix run .#homa-pair-test +``` + +This boots two VMs, `node0` and `node1`, joined by a private Ethernet link. The link is +QEMU UDP between two localhost ports, so it needs no root, tap or bridge. Homa then runs +over that link between two separate kernels: + +``` +--- node0 (server) +HOMA_PAIR_SERVER_START host=node0 kernel=7.3.0 +HOMA_PAIR_SERVER_INSMOD_OK +HOMA_PAIR_SERVER_SERVERS_OK +HOMA_PAIR_SERVER_READY +HOMA_PAIR_SERVER_CLIENT_REPORT PASS +HOMA_PAIR_SERVER_RMMOD_OK +HOMA_PAIR_SERVER_DMESG_OK +HOMA_PAIR_SERVER_PASS +--- node1 (client) +HOMA_PAIR_START host=node1 kernel=7.3.0 +HOMA_PAIR_INSMOD_OK +HOMA_PAIR_PING_OK +HOMA_PAIR_PEER_READY_OK +HOMA_PAIR_ECHO responses=6 (want 6) data+grant packets=1335 +HOMA_PAIR_ECHO_OK +HOMA_PAIR_CPNODE_homa ... homa_w4 clients: 0.40 Kops/sec, 0.19 Gbps out, 0.19 Gbps in, RTT (us) P50 522.07 P99 33163.40 ... +HOMA_PAIR_CPNODE_HOMA_OK +HOMA_PAIR_CPNODE_tcp ... tcp_w4 clients: 0.40 Kops/sec, 0.15 Gbps out, 0.15 Gbps in, RTT (us) P50 372.96 P99 22185.27 ... +HOMA_PAIR_CPNODE_TCP_OK +HOMA_PAIR_RMMOD_OK +HOMA_PAIR_DMESG_OK +HOMA_PAIR_PASS +VERDICT: PASS +``` + +The client does the following, in order: +1. pings `node0`; +2. runs echo RPCs from 100 B to 1 MB across the link; +3. drives a few seconds of Homa's `cp_node` workload `w4`; +4. drives the same workload over TCP for comparison. + +Then both VMs unload the module and check dmesg. A run takes about 35 seconds once +cached. The link is emulated and the VMs share host CPUs, so the rates show that traffic +flows, not how fast Homa is. + +--- + +## Testing against different kernels + +Every kernel series has its own targets, so you can build or boot exactly the one you want: + +| Series | Source | `homa-module-` | `homa-microvm[-test]-` | Status on `main` | +|---|---|---|---|---| +| `7_3` (default) | nixpkgs `linuxPackages_latest` | ✅ builds | ✅ boots, smoke and pair tests pass | supported | +| `7_0`, `7_1`, `7_2` | kernel.org tarballs (`kernel/src.nix`) | ✅ compile check | — (no bootable kernel yet) | supported | +| `6_12` | nixpkgs | ✅ builds | ✅ boots, smoke and pair tests pass | supported | +| `6_18` | nixpkgs (6.18.57) | ❌ | — | stable backported the 7.x socket API changes; see [`STATUS.md`](STATUS.md) F1 | +| `6_1`, `6_6` | nixpkgs | ❌ | — | below Homa's supported range | +| `local` | your checkout, `HOMA_LINUX_SRC=...` with `--impure` | ✅ compile check | — | e.g. net-next | + +```sh +nix build .#homa-module-6_12 # homa.ko for one kernel +nix build .#homa-build-matrix # try every kernel; result/summary.md is the table +nix run .#homa-microvm-test-6_12 # boot that kernel and run the smoke test +nix run .#homa-microvm-6_12 # the same guest, interactive (root, no password) +nix run .#homa-pair-test-6_12-7_3 # Homa between a 6.12 server VM and a 7.3 client VM +``` + +VM and pair targets exist only for kernels where the module builds (`vmSeries` in +`kernel/config.nix`). Pair targets cover every server/client combination: `6_12-6_12`, `6_12-7_3`, +`7_3-6_12` and `7_3-7_3`. The mixed pairs check that Homa built for one kernel talks to Homa +built for another. Each pair has its own link port, so they can all run at once: + +```sh +for p in 6_12-6_12 6_12-7_3 7_3-6_12 7_3-7_3; do nix run .#homa-pair-test-$p & done; wait +``` + +Each VM boots the same `kernelPackages` the module was built against, so the module's version +check (vermagic) always matches. The host `/nix/store` is shared into the guest, so nothing +is copied into an image. + +--- + +## All commands + +```sh +nix develop # dev shell (toolchain + analysis tools) + +nix build .#homa-module # homa.ko (release) against the default kernel (7.3) +nix build .#homa-module-debug # homa.ko (debug: -g -O1) +nix build .#homa-module- # one kernel (table above) +nix build .#homa-build-matrix # every kernel -> result/summary.md (never fails) +nix build .#homa-module-riscv64 # cross-compiled homa.ko (also -aarch64, and -strip) +nix build .#homa-cross-matrix # riscv64 + aarch64: homa.ko and util/ -> summary.md +HOMA_LINUX_SRC=~/src/linux nix build --impure .#homa-module-local # your kernel checkout +nix build .#homa-utils[-debug] # user-space tools (util/); -debug adds ASan +nix build .#homa-tests # build + run the unit tests (ASan) +nix build .#homa-tests-coverage # the unit tests under gcov: line/branch coverage per file + +nix run .#homa-microvm-test # default kernel: insmod, echo, GRO/RPS, rmmod +nix run .#homa-microvm-test- # ... on one kernel +nix run .#homa-microvm-test-debug # ... under KASAN, lockdep, UBSAN, KMEMLEAK (slow) +nix run .#homa-microvm-test-kcsan # ... under KCSAN: data races in Homa's functions (slow) +nix run .#homa-pair-lifecycle-debug # the pair lifecycle on that kernel (also -kcsan) +nix run .#homa-lifecycle[-] # host-driven, timed phases over ttyS0 + virtio console +nix run .#homa-microvm[-] # interactive guest +nix run .#homa-pair-test # two VMs: echo + cp_node (Homa, TCP) across a link +nix run .#homa-pair-test-- # ... server kernel , client kernel +nix run .#homa-pair-lifecycle[-] # the pair over both consoles, timed: netem loss/delay, Homa qdisc +nix run .#homa-pair-exp # latency sweep: cp_node per sysctl setting + metric deltas +nix run .#homa-vm-matrix # every VM test and pair in parallel -> summary table +nix run .#homa-vm-compare -- a b # two matrix runs' logs ($HOMA_VM_LOGDIR), phase by phase +nix run .#homa-tcl-test [-- -file x.test] # the Tcl drivers' tests, or some; homa-tcl-lint: their lint +nix run .#homa-pair-exp-socket # ... over QEMU's TCP socket link (STATUS.md F2) + +nix build .#homa-image[-debug] # OCI user-space image (docker load < result) + +nix build .#analysis-quick # python + shell + clang-format (no kernel needed) +nix build .#analysis-standard # + cppcheck + clang-tidy +nix build .#analysis-deep # + sparse (endianness) + smatch +nix build .#analysis-netdev # netdev's checks on the code as submitted (make net-next): + # checkpatch --strict, kernel-doc, W=1 gcc+sparse/clang/i386 + +nix build .#fuzz # standalone wire-decoder + selftest harnesses +FUZZ_TIME=300 ./result/bin/run-fuzzers # ... run them for 5 minutes +nix build .#fuzz-run # bounded run -> summary.txt / count.txt / crashes +nix run .#fuzz-deep -- [outdir] # fork mode + dictionary (FUZZ_DEEP_TIME) +nix build .#fuzz-cov # coverage of the wire decoder +nix build .#fuzz-selftest # canary gate (fails if the harness is vacuous) +nix build .#fuzz-mock # real Homa parsers against a kernel mock + +nix build .#homa-evidence # every check that needs no VM, in one report + # (result/evidence.html) +nix run .#homa-evidence-full -- out # ... plus the VM matrix; HOMA_EVIDENCE_SANITIZERS=1 adds + # the KASAN and KCSAN pair lifecycles + +nix flake show # list every output +nix flake check # run the checks +``` + +`HOMA_VM_LOGDIR=` keeps the microVM console logs in a directory of your choice. By +default each run makes a fresh `mktemp -d` and prints its path. + +## How to read this tree + +New to Nix? Four ideas cover most of what is here. + +- **A derivation is a build recipe.** It lists its inputs (source, compilers, other + derivations) and a script that writes its output to `/nix/store/-`. The hash + covers every input, so the same inputs give the same output on any machine, and a build + whose inputs have not changed is not run again. +- **A `.nix` file is usually a function.** `{ pkgs, lib, homaLib }: ...` takes an attribute + set (Nix's name for a record or dict) and returns derivations. The braces list what the + file depends on. +- **The scope wires the files together.** [`scope.nix`](scope.nix) calls each file with + `callPackage`, which passes it the arguments its braces name, from the scope or from + nixpkgs. No file imports another file's dependencies by hand. +- **The flake exposes the results.** [`../flake.nix`](../flake.nix) names its inputs + (nixpkgs, microvm.nix, flake-parts, all pinned in `flake.lock`) and lists the files in + [`parts/`](parts/). Each part picks targets from the scope, and those are the names that + `nix build .#` and `nix run .#` accept. + +So, to find where `nix build .#X` comes from, look in `parts/` for the area, then in the +scope for the file. Each folder's README covers what it holds, what to run, and the Nix +idioms it uses. + +## Folder map + +| Folder | What it holds | +|---|---| +| [`parts/`](parts/) | the flake's outputs, one file per area | +| [`lib/`](lib/) | `homaLib`: the source subsets with `patches/` applied, `mkKbuild`, `perKernel` | +| [`kernel/`](kernel/) | the kernels: the default and the matrix, trees prepared from source, the unit-test tree | +| [`build/`](build/) | `homa.ko` per kernel and cross-compiled, `util/`, the unit tests, the OCI image | +| [`dev/`](dev/) | `nix develop`: tool lists, environment, helper functions | +| [`analysis/`](analysis/) | static analysis in three tiers, and netdev's submission checks | +| [`fuzz/`](fuzz/) | libFuzzer harnesses for the wire format | +| [`microvms/`](microvms/) | the QEMU tests: [`guest/`](microvms/guest/) runs in the VMs, [`host/`](microvms/host/) and [`tcl/`](microvms/tcl/) drive them | +| [`evidence/`](evidence/) | `homa-evidence`: every check's result in one report | +| [`patches/`](patches/) | upstream-pending Homa fixes the flake applies (currently none) | +| [`docs/`](docs/) | [`DESIGN.md`](docs/DESIGN.md): the reasons behind the layout | + +Reports from the analysis and fuzzing rounds are on the fork's `nix-flake` branch, in [`analysis-reports/`](https://github.com/randomizedcoder/HomaModule/tree/nix-flake/analysis-reports). diff --git a/nix/STATUS.md b/nix/STATUS.md new file mode 100644 index 00000000..ff95c6f1 --- /dev/null +++ b/nix/STATUS.md @@ -0,0 +1,545 @@ +# Homa Nix flake: status, findings and follow-ups + +The single status doc for the flake: what builds and passes on each kernel, what the builds, +tests, analysis and microVM runs have shown about Homa, and what to look into next. + +Related docs: +- **Commands and quick start:** [`README.md`](README.md) +- **Architecture and rationale:** [`docs/DESIGN.md`](docs/DESIGN.md) +- **Upstream PRs and their outcomes:** [`analysis-reports/pr-tracker.md`](https://github.com/randomizedcoder/HomaModule/blob/nix-flake/analysis-reports/pr-tracker.md) +- **VM pair test plan:** [`microvms/PLAN-pairs.md`](microvms/PLAN-pairs.md) + +| | | +|---|---| +| **Last update** | 2026-10-31 | +| **Branch** | `nix-flake` on randomizedcoder/HomaModule, rebased on upstream `main` | +| **nixpkgs** | `nixos-unstable` | +| **Default kernel** | `linuxPackages_latest` (7.3); unit harness and fuzz-mock use 6.18.57 headers (F5) | +| **Upstream base** | `main` = `4bd0275` ("Get 'make `__STRIP__`=1' working again") | +| **Kernels** | nixpkgs 6.1.190, 6.6.159, 6.12.113, 6.18.57, 7.3.0; kernel.org 7.0.14, 7.1.13, 7.2.12 | + +--- + +## Results at a glance + +| Area | Command | Result on `main` | +|---|---|---| +| Module build | `nix build .#homa-build-matrix` | ✅ 6.12, 7.0, 7.1, 7.2, 7.3. ❌ 6.18.57 (F1, not a target). ❌ 6.1/6.6 (no `net/rps.h`, below Homa's range) | +| Stripped build | `make __STRIP__=1` (7.3) | ✅ clean | +| netdev checks | `nix build .#analysis-netdev` | ❌ the `make net-next` files do not build in-tree (F12); 8 kernel-doc warnings, 31 checkpatch findings | +| Unit-test coverage | `nix build .#homa-tests-coverage` | 86.7 % of lines, 81.6 % of branches, 88.5 % of functions | +| Evidence report | `nix build .#homa-evidence` | every check above that needs no VM, in `evidence.html` | +| Cross compile | `nix build .#homa-cross-matrix` | ✅ `homa.ko` for riscv64 and aarch64 (7.3, normal and stripped). ❌ `util/` is x86-only (F9) | +| Unit tests | `nix build .#homa-tests` | ✅ 884/884, in IPv6 and `--ipv4` modes | +| Fuzzing, real parsers | `nix build .#fuzz-mock` | ✅ all 10 harnesses run clean; the known-findings list is empty | +| Fuzzing, wire decoder | `fuzz-run`, `fuzz-selftest` | ✅ selftest canary PASS | +| Single VM | `homa-microvm-test-{6_12,7_3}` | ✅ insmod, loopback echo, netns/veth with GRO+RPS (about 1.7k `homa_gro_receive` and 460 `homa_set_softirq_cpu` calls), rmmod, clean dmesg | +| Two VMs | `homa-pair-test--` | ✅ all four 6.12/7.3 pairings: ping, echo 100 B–1 MB, cp_node w4 over Homa and over TCP, clean dmesg on both. 8 VMs in parallel finish in 50 s | +| Latency sweep | `homa-pair-exp`, `homa-pair-exp-socket` | ✅ F2 explained: QEMU's TCP `socket` link stalls packets; over the UDP `dgram` link (now the default) Homa's P50 matches TCP's | +| Lifecycle | `homa-lifecycle-{6_12,7_3}` | ❌ on `main`: phase 3 (smoke) crashes in `homa_route_free` (F8), and phases 7–8 report it. ✅ all 9 phases with the fix, about 22 s | +| Pair lifecycle | `homa-pair-lifecycle-{6_12,7_3}` | ✅ with the F8 and F11 fixes: 39 timed phases, including five `tc netem` phases (delay, 0.1–5 % loss) and Homa's qdisc with its pacer. Echo, cp_node Homa and TCP complete under every one, and Homa recovers by RESEND. Delay alone triggers many resends (F10). On `main`, traffic stalls under Homa's qdisc (F11), and both nodes oops at rmmod (F8) | +| VM matrix | `homa-vm-matrix` | ❌ on `main` (F8, F11). ✅ 10/10 with the fixes (forks #22, #23): smoke, lifecycle and pair lifecycle per kernel, plus four pairs, in 183 s | +| Sanitizer VMs | `homa-microvm-test-{debug,kcsan}`, `homa-pair-lifecycle-{debug,kcsan}` | ❌ lockdep fails the KASAN smoke test (F13; KASAN itself is clean with the F8 fix and names F8 without it). ❌ KCSAN finds one race in the Homa qdisc (F14); KCSAN smoke passes, watching 594 Homa functions | +| Static analysis | `analysis-deep` | Four rounds triaged; see the bugs below | + +### cp_node over the VM link (workload w4, 0.2 Gb/s target, 6 s) + +UDP `dgram` link (the default since F2). All five pair targets ran at once (10 VMs), so the +medians are higher than the single-pair sweep in F2. + +| Server ↔ client | Homa Kops/s | Homa RTT P50 / P99 (µs) | TCP Kops/s | TCP RTT P50 / P99 (µs) | +|---|---|---|---|---| +| 6.12 ↔ 6.12 | 0.42 | 294 / 31213 | 0.41 | 569 / 22088 | +| 6.12 ↔ 7.3 | 0.35 | 452 / 34976 | 0.41 | 378 / 23300 | +| 7.3 ↔ 6.12 | 0.37 | 490 / 31825 | 0.42 | 318 / 22345 | +| 7.3 ↔ 7.3 | 0.38 | 452 / 34574 | 0.42 | 383 / 20973 | + +On the earlier TCP `socket` link, Homa's P50 was 2927–5024 µs and TCP's P99 about 42 ms (F2). + +These are functional numbers: an emulated link between VMs that share host CPUs. They show +that traffic flows, not how fast Homa is. + +### Lifecycle test (`nix run .#homa-lifecycle`) + +This follows the pcp and xdp2 lifecycle tests. The guest has two consoles on host TCP +ports: + +- **ttyS0** is the only kernel console. QEMU logs it to `serial.log` from the first byte, + so an oops is caught even after the shell has died, during shutdown for instance. +- **hvc0** is a virtio console running a root shell. The host drives each phase through it, + and it is fast: reading 612 lines of `dmesg` takes 188 ms, where 115200 baud would take + seconds. Kernel messages never mix into command output. + +Results on 7.3 with the F8 fix: + +| Phase | | ms | +|---|---|---| +| 1 start | qemu running, ports listening | 108 | +| 2 serial | first kernel line on ttyS0 | 2482 | +| 2b virtio | root shell answering on hvc0 | 12698 | +| 3 smoke | `homa-smoke` over hvc0 | 2722 | +| 4 cycles | 5 × insmod, netns add/del, rmmod | 1025 | +| 5 dmesg | no BUG/Oops/WARNING/KASAN | 188 | +| 6 shutdown | `poweroff` over hvc0 | 2 | +| 7 exit | qemu exits | 3050 | +| 8 serial-log | no oops anywhere in `serial.log` | 8 | + +On `main`, phase 3 crashes. Phases 4–6 are then SKIPped (there is no shell), phase 7 reports +"qemu exited before poweroff", and phase 8 shows the oops and its `RIP:` line. + +### Pair lifecycle with netem (`nix run .#homa-pair-lifecycle`) + +The pair (node0 server, node1 client) driven over both guests' hvc0 consoles, with the same +timed phases as the lifecycle test, plus traffic phases. Each traffic phase runs echo +(`demo_client1`, six RPCs from 100 B to 1 MB), then cp_node w4 over Homa and over TCP for 6 s, +from node1 to node0. It records `homa-node-stats` deltas on both nodes, in +`traffic//deltas`. + +The netem phases put `tc netem` on eth1 of both nodes, so both directions are impaired. Every +netem qdisc has `limit 100000` (packets), enforced by an `assert` in `pair-lifecycle.nix`, so +the queue never overflows and the only drops are the configured random loss, not tail drops. +A phase fails unless: + +- echo returns 6/6 and both cp_node runs complete RPCs (they report a P99); +- each node's netem drop rate matches the configured loss (exactly 0 drops at 0 %); and +- with loss, Homa sent RESENDs. + +Results on 7.3 with the F8 fix (rates and RTTs from cp_node; drops and resends summed over +both nodes): + +| Phase | netem on eth1 (both nodes) | Homa Kops/s, P50 / P99 (µs) | TCP Kops/s, P99 (µs) | netem drops | Homa RESENDs, resent packets | TCP retrans | +|---|---|---|---|---|---|---| +| 7 baseline | none | 0.43, 176 / 27402 | 0.44, 21271 | — | 0, 0 | 0 | +| 8.1 | `delay 1ms` | 0.23, 2217 / 30993 | 0.25, 26340 | 0 % | 225, 15870 | 0 | +| 8.2 | `loss 0.1%` | 0.35, 189 / 34547 | 0.35, 26240 | 0.06–0.07 % | 303, 219 | 244 | +| 8.3 | `loss 1%` | 0.30, 176 / 37735 | 0.12, 204557 | 0.76 % | 2080, 1776 | 1199 | +| 8.4 | `delay 1ms loss 1%` | 0.14, 2223 / 48779 | 0.10, 209001 | 0.74–0.78 % | 1832, 9940 | 508 | +| 8.5 | `loss 5%` | 0.18, 185 / 46154 | 0.01, 553225 | 4.8 % | 6141, 5648 | 463 | +| 9 recovered | removed | 0.45, 186 / 32381 | 0.39, 29206 | — | 0, 0 | 0 | + +- **Homa rides out loss better than TCP here.** At 5 % loss, Homa keeps 40 % of its baseline + rate with P50 unchanged; TCP drops to 2 % with a P99 above half a second. There are no + `rpc_timeouts` in any phase. 6.12 gives the same picture. +- **netem's measured loss is a little under the configured rate** (0.76 % for 1 %). tc counts + sent packets per GSO segment but drops per skb, so a dropped GSO skb hides several + segments. Tail drops would push the measured rate up, which the check bounds at twice the + configured loss plus 0.5 points. +- **Delay alone makes Homa resend (F10).** + +**Homa's qdisc (phases 10–12).** `homa-qdisc on` (`qdisc.nix`) attaches Homa's qdisc to eth1 +on both nodes: mq with `homa` on each child, as `cloudlab/bin/config` does, or `homa` as the +root with one tx queue (the case here). Phase 10 runs traffic through it, and TCP must pass +through the qdisc (`qdisc_tcp_packets`). Phase 11 sets `net.homa.link_mbps=1000`, so a 1 MB +echo is several ms of link time and the pacer must defer and send Homa packets +(`pacer_homa_packets`). Phase 12 restores `link_mbps` and the default qdisc. + +| Phase | Qdisc | `main` + F8 fix | + F11 fix (fork #23), 7.3 | +|---|---|---|---| +| 10 | `homa`, `link_mbps` 25000 | ❌ echo, Homa and TCP stall: 0 Kops, 1170 RESENDs | ✅ Homa 0.39 Kops, TCP 0.42; TCP through the qdisc 107892 packets | +| 11 | `homa`, `link_mbps` 1000 | ❌ the same; the pacer sends nothing | ✅ Homa 0.36 Kops, TCP 0.39; the pacer sent 145549 Homa packets | + +6.12 matches 7.3. Rates are the same as without the qdisc, because the emulated link is far +slower than `link_mbps`. + +Times: boot to both shells about 18 s; each traffic phase about 13.6 s; the whole run about +3 min. + +### Build matrix (`nix build .#homa-build-matrix`) + +| Kernel | Source | Result | +|---|---|---| +| 6.1.190 | nixpkgs | ❌ no `net/rps.h` (below Homa's supported range) | +| 6.6.159 | nixpkgs | ❌ no `net/rps.h` (below Homa's supported range) | +| 6.12.113 | nixpkgs | ✅ | +| 6.18.57 | nixpkgs | ❌ 6.18-stable backports (F1; not a target, LTS will be 7.3+) | +| 7.0.14 | kernel.org | ✅ compile | +| 7.1.13 | kernel.org | ✅ compile | +| 7.2.12 | kernel.org | ✅ compile | +| 7.3 | nixpkgs | ✅ | +| 7.3-rc7 (`af32da41b032`) | local, `HOMA_LINUX_SRC` | ✅ compile | + +### Per target (verified with `nix build` / `nix run`) + +| Target | Status | +|---|---| +| `nix flake show`, `nix flake check` | ✅ evaluates, checks pass | +| `homa-module` (7.3), `homa-module-` | ✅ (see the matrix above) | +| `homa-utils`, `homa-utils-debug` | ✅ (`-debug` links ASan through `NIX_CFLAGS_LINK`; it did not link before 2026-10-30) | +| `homa-module-{riscv64,aarch64}[-strip]` | ✅ compile; `readelf` confirms RISC-V / AArch64 | +| `homa-utils-{riscv64,aarch64}` | ❌ x86-only code in `util/` (F9) | +| `homa-cross-matrix` | ✅ summary of both arches (fails only if a `homa.ko` does) | +| `homa-tests` | ✅ 884/884 (hermetic 6.18 unit KDIR) | +| `homa-image[-debug]` | ✅ | +| `analysis-quick` | ✅ | +| `analysis-sparse` / `-smatch` / `-compile-db` (7.3) | ✅ sparse 1575 raw (980 on 6.18), smatch 38; both fail if kbuild never runs the checker | +| `fuzz`, `fuzz-run`, `fuzz-cov` | ✅ standalone decoder harnesses | +| `fuzz-selftest` | ✅ PASS (the canary catches an injected overflow) | +| `fuzz-mock` | ✅ 10 real-code harnesses through `homa_softirq`; gated on `nix/fuzz/known-findings.txt` (empty) | +| `homa-microvm-test` (7.3.0), `homa-microvm-test-6_12` | ✅ PASS | +| `homa-microvm-test-debug` (KASAN), `-kcsan` | ❌ F13 (lockdep) / ✅ PASS | +| `homa-pair-lifecycle-debug` (KASAN), `-kcsan` | ❌ F13 (lockdep; KASAN clean, see below) / ❌ F14 (KCSAN) | +| `homa-pair-test`, `homa-pair-test--` | ✅ PASS for all four 6.12/7.3 pairings | +| `homa-lifecycle`, `homa-lifecycle-` | ✅ PASS with the F8 fix (see above) | +| `homa-pair-lifecycle`, `homa-pair-lifecycle-` | ✅ PASS with the F8 fix (see above) | +| `tcl-tests` (check) | ✅ the Tcl drivers: 349/349 tcltest cases on Tcl 8.6 with Expect, the 227 pure ones on Tcl 9.0 (style checks included), nagelfar and tclint clean, oops regex in step with the guests' | + +--- + +## Findings + +### F1: 6.18-stable backports break the build on 6.18.40 and later (won't fix) + +6.18 stable backported two 7.x socket API changes: the recvmsg `addr_len` removal in 6.18.40, +and `sockaddr_unsized` for bind in 6.18.57. Homa's gates (`< KERNEL_VERSION(7, 1, 0)` and +`(7, 0, 0)`) therefore pick the old signatures on 6.18.57. That is a hard error with gcc 15+, +and a `-Wincompatible-pointer-types` warning in the unit build. + +- **Status:** raised with John on + [#120](https://github.com/PlatformLab/HomaModule/pull/120#issuecomment-6486061071). + [His answer](https://github.com/PlatformLab/HomaModule/pull/120#issuecomment-6500983772) + (2026-10-29): the first LTS `main` targets will be 7.3 or later, so Homa doesn't need to + build on 6.18. No gate fix. +- **Follow-up:** none upstream. 6.18 stays in `homa-build-matrix` as a known failure, so a + change in the gates shows up there. + +### F2: Homa's 5 ms median over the VM link was QEMU's `socket` backend (resolved) + +The first pair runs showed Homa's cp_node P50 at about 3–5 ms against about 0.3 ms for TCP, +in every pairing. The sweep target (`nix run .#homa-pair-exp`) runs cp_node once per +experiment, sets `net.homa.*` sysctls on both nodes, and prints the per-node deltas of the Homa +metrics, eth1/virtio/qdisc/softnet drops and TCP retransmits. It found the cause in four runs. + +**1. The median is Homa's RESEND timer.** Homa sends a RESEND after `resend_ticks` (default 5) +ticks of 1 ms with nothing received. The P50 tracks that setting, and `link_mbps` does +nothing: + +| Run (7.3 ↔ 7.3, `socket` link) | Homa P50 / P99 (µs) | Client RESENDs sent | Server packets resent | +|---|---|---|---| +| w4, defaults | 4717 / 15611 | 2146 | 19467 | +| w4, `link_mbps=1000` | 4837 / 16026 | 2116 | 19403 | +| w4, `link_mbps=2000` | 4873 / 16217 | 2132 | 20252 | +| w4, `resend_ticks=2` | 2028 / 15064 | 2073 | 12512 | +| w4, `resend_ticks=10` | 9577 / 24441 (2580 / 30620 in 1 of 3 runs) | 2379 | 18745 | +| w1 (about 190 B), defaults | 113 / **5067** | 1672 | 2357 | +| TCP w4 / w1 | 225 / 42151, 117 / 41038 | 95 / 6 TCP retransmits | | + +**2. Nothing is lost; packets arrive late.** The client's eth1 rx count equals the server's +tx count to within one packet, and the eth1, virtio, qdisc and softnet drop counters all +stay at zero. On w1, the client's `unknown_rpcs` (2326) roughly equals the server's +`resent_packets` (2335). So the original response arrives before its retransmission, and +the late copy finds the RPC already finished. TCP retransmits too, on a link that drops +nothing, and its P99 is about 42 ms. + +**3. The UDP `dgram` backend removes the stalls** (`homa-pair-exp`; the old link is +`homa-pair-exp-socket`): + +| Run (7.3 ↔ 7.3) | `socket` P50 / P99 (µs) | `dgram` P50 / P99 (µs) | `dgram` Kops/s | `dgram` RESENDs | +|---|---|---|---|---| +| Homa w4, defaults | 4717 / 15611 | 177 / 33386 | 0.45 | 0 | +| Homa w4, `link_mbps=1000` | 4837 / 16026 | 156 / 25195 | 0.36 | 0 | +| TCP w4 | 225 / 42151 | 167 / 24519 | 0.44 | (0 retransmits) | +| Homa w1, defaults | 113 / 5067 | 99 / 166 | 9.78 | 0 | +| TCP w1 | 117 / 41038 | 106 / 225 | 9.02 | (0 retransmits) | + +- **Conclusion:** QEMU's `-netdev socket` (a TCP stream between the two QEMU processes) holds + some packets for milliseconds. Homa recovers through its 5 ms RESEND timer and TCP through + retransmission, which explains both the 5 ms Homa P50 and the 42 ms TCP P99. Homa behaves + as designed. It is not a Homa bug, and there is nothing to report upstream. Where in QEMU + the packets wait has not been investigated. +- **Change:** the pair tests now use `-netdev dgram` (UDP between two host ports) by default. + It also needs no listen/connect ordering. +- **Still open, minor:** over `dgram`, Homa's w4 P99 (25–33 ms) is close to TCP's (about + 25 ms). Both probably come from the w4 size mix on a slow emulated link, but this hasn't + been checked. +- **Not the cause:** `link_mbps` (no effect, above), and round-4 candidate 5 + (`SPEED_UNKNOWN`), because the Homa qdisc isn't attached in these tests. With it attached, + candidate 5 stalls traffic outright rather than slowing it (F11). + +### F3: Homa's built-in packet-drop knobs do nothing + +`homa_drop_packet()` (`homa_devel.c:946`) drops bursts of packets as configured by the +`accept_bits` and `drop_bits` sysctls, to exercise retry code. Its only call site is commented +out (`homa_offload.c:259`), so setting the sysctls has no effect. + +- **Follow-up:** loss tests now use `tc netem` (`homa-pair-lifecycle`). Ask John whether the + call should come back (it is a development knob) or the sysctls should go. + +### F4: Upstream bugs found and fixed + +| Round | Source | Sent | Landed | Open | +|---|---|---|---|---| +| 1 | static analysis (cppcheck, clang-tidy, ruff, shellcheck) | 11 | 10 | — (#94 rejected) | +| 2 | fuzz-mock and the 7.x port | 2 | 2 | — | +| 3 | sparse and gcc W=1 | 4 | 4 | — | +| 4 | reading the code (copy-paste asymmetries, error paths) | 4 | — | #137–#140 | + +Notable fixes: +- **Fuzzing:** a zero-length incoming message indexed `small_msg_bytes[-1]` (#121). +- **IPv6:** + - Non-Homa IPv6 TCP was sent to the IPv4 TCP GRO handler (#138). + - The qdisc bypass hash skipped one word of the IPv6 address, so same-flow TCP packets + could be reordered (#137). +- **Use-after-free:** the pacer's round-robin pointer was left on a destroyed qdisc (#140). +- **Wrong bit tested:** HOMAIOCINFO's `RPC_PRIVATE` check tested `APP_NEEDS_LOCK` (#139). + +Pending candidates: [`round4/candidates.md`](https://github.com/randomizedcoder/HomaModule/blob/nix-flake/analysis-reports/round4/candidates.md) items 5–8: +- unknown link speed; +- destroy after a failed init; +- `softirq_cores` bounds; +- the priority map applied twice, which needs re-verifying. + +### F5: The test harness lags the default kernel + +The unit harness and fuzz-mock build against 6.18.57 headers (`unitKernelPackages`), because +`test/mock.c` is maintained for John's development kernel, not for 7.3. That is fine for +logic tests, but kernel-API drift on 7.x is only caught by the module builds and the VMs. + +- **Follow-up:** `main`'s LTS will be 7.3 or later (F1), so 6.18 headers will soon be the + odd one out. Try `test/mock.c` against 7.3 headers now, and move the unit KDIR to 7.3 when + `main` switches. + +### F6: Small items noted while reading the code + +- **`homa_softirq` verbose `pr_warn`:** it reads `h->type` before `h` is set for the current + skb. It only runs with the verbose sysctl, so at most a one-line fix. +- **`homa_err_handler_v4` and `ICMP_FRAG_NEEDED`:** it treats FRAG_NEEDED as unreachable and + aborts RPCs, while the v6 handler ignores PKT_TOOBIG. Its `pr_notice` for other ICMP types is + not rate-limited. This is a policy question for John rather than a PR. + +### F7: Coverage gaps in our own testing + +- **Sanitizer VMs:** now run (KASAN, KFENCE, KMEMLEAK, lockdep, UBSAN; and KCSAN); see F13 + and F14. A sanitizer kernel is a full kernel build, cached afterwards, and its guests are + about 4× slower (`sanitizerSlowdown`). +- **Not exercised over a real link yet:** IPv6 between VMs, and more than two nodes. (Loss, + delay, and the Homa qdisc and pacer are now covered by `homa-pair-lifecycle`.) +- **7.0–7.2:** compile-checked only; there is no bootable kernel for them in the flake yet. + +### F8: Use-after-free in `homa_route_free` on netns exit and rmmod (fix: fork #22) + +`homa_peer_free_net()` (the pernet `.exit` path) releases each of the namespace's routes +with `call_rcu(homa_route_free)`. The callback dereferences `route->key.hnet->homa->peertab`. +But `hnet` is pernet data (`.size = sizeof(struct homa_net)`), which the netns core frees +straight after `.exit` and before its own `rcu_barrier()`. It came in with upstream +`8db9f5e` (routes, 2026-10-14). + +- **Found by:** the first `homa-vm-matrix` run. One 7.3 smoke VM hit a general protection + fault in `homa_route_free` (from `rcu_do_batch`) after `ip netns del`. Unloaded VMs rarely + show it (3 of 8 under load, 0 of 16 alone), because the freed `hnet` usually still holds + valid pointers. +- **Made deterministic:** the test guests now boot with `init_on_free=1`. Unpatched, every + smoke run oopses on 6.12 and 7.3 (8/8), and so does every pair client on `rmmod` + (`unregister_pernet_subsys` frees the init_net `hnet` the same way). +- **Fix:** `rcu_barrier()` at the end of `homa_peer_free_net()`. Unit test + `homa_peer_free_net__wait_for_rcu_callbacks` is red then green, with 885/885 unit tests + passing in IPv6 and `--ipv4` modes. The 7.3 build is clean normal and stripped, checkpatch + is clean, and the VMs go from 8/8 oops to 8/8 pass and the matrix from 0/6 to 6/6. + Draft: + [randomizedcoder/HomaModule#22](https://github.com/randomizedcoder/HomaModule/pull/22). +- **Until it lands,** `homa-vm-matrix` and the VM tests fail on `main`, as they should. + +### F9: `util/` builds only on x86; `homa.ko` cross-compiles cleanly + +`homa-cross-matrix` cross-compiles for riscv64 (`pkgsCross.riscv64`) and aarch64 +(`pkgsCross.aarch64-multiplatform`) on an x86_64 host. Toolchains come from the binary cache, +and the 7.3 trees are prepared from source with each arch's nixpkgs kernel config (about +2 minutes each). + +- **`homa.ko`:** builds for both arches, in normal and `__STRIP__=1` forms. There are no errors + and no warnings on riscv64. On aarch64 there is one warning: + `homa_metrics.c:442: the frame size of 2112 bytes is larger than 2048 bytes` + (`homa_metrics_print`, which the stripped build leaves out). The same function stays under + the limit on x86. +- **Bug, riscv clock rate:** in the normal build, `homa_clock()` is `get_cycles()`. On riscv + that reads the `time` CSR, which ticks at the platform timebase: 10 MHz on QEMU `virt`, and + 1 MHz on some boards. `homa_clock_khz()` (`homa_impl.h:767`) has cases for x86 (`tsc_khz`) + and arm64 (`arch_timer_get_cntfrq()`), and otherwise returns 1000000, i.e. 1 GHz. So on + riscv every cycles↔ns conversion is 100× to 1000× off: timeouts, RESEND intervals and + pacing. A likely fix is `riscv_timebase / 1000` under `CONFIG_RISCV` (the symbol is + exported GPL, and Homa is Dual BSD/GPL). The stripped (upstream) build uses + `ktime_get_ns()` and is correct. This was found by reading the code, since nothing runs on + riscv yet. +- **`util/`:** 0 of 19 programs build on either arch. 17 of 23 sources compile; the other 6 fail + for two reasons: + - `rdtsc()` in `util/test_utils.h:63` is x86 inline asm ("impossible constraint in 'asm'"). + It breaks `test_utils.cc`, which every program links, plus `time_trace.cc`, `dist_test.cc`, + `homa_test.cc` and `smi.cc`. `util/inc_tput.cc:26` has a second, unused copy. + - The `cp_node.cc:3662` crash handler reads `uc_mcontext.gregs[REG_RIP]`, which is x86_64-only. +- **Follow-up:** ask John whether non-x86 `util/` matters (#14). If it does, the fix is small: + `__builtin_ia32_rdtsc` on x86 with `clock_gettime(CLOCK_MONOTONIC_RAW)` elsewhere, and + `REG_RIP` behind `#ifdef __x86_64__`. + +### F10: 1 ms of delay with no loss makes Homa resend about a quarter of its data + +In `homa-pair-lifecycle` phase 8.1 (`netem delay 1ms` on both nodes, so a 2 ms RTT, no loss), +node1 sent 225 RESENDs in 6 s, and node0 resent 15870 packets: 23 % of its 68855 DATA packets, +about 70 packets per RESEND. Most of them arrived after their RPC had finished: node1 counted +12202 `unknown_rpcs` and 3668 `resent_discards`. 6.12 shows the same (254 RESENDs, 16827 +packets). With 1 % loss and no delay, resends match the drops (2080 RESENDs, 1776 packets). + +- A receiver requests a resend after `resend_ticks` (5) ticks of 1 ms (`homa_plumbing.c:2180`) + with nothing heard on an RPC that still expects granted data (`homa_timer.c:155`). A 2 ms + RTT is well inside 5 ms, so the RTT alone shouldn't trigger it; a grant waits 1 ms each way + as well. +- **Follow-up (#16):** sweep `resend_ticks` with netem delay (homa-pair-exp style), and look at + which RPCs request the resends and for what ranges. Then decide whether this is worth asking + John about. The 2 ms RTT is long for a datacenter, but cross-rack or virtualized paths can + see it. + +### F11: Homa's qdisc stalls all traffic on NICs that report no link speed (fix: fork #23) + +Round-4 candidate 5, now reproduced in VMs. +- **Cause:** virtio-net reports `SPEED_UNKNOWN` (-1). `homa_qdev_update_sysctl` + (`homa_qdisc.c:1404`) takes it as `link_mbps`, so `max_nic_queue_bytes` is negative. +- **Effect:** every packet the qdisc may defer (Homa DATA in messages of 1000 bytes or more, + and TCP packets of 1000 bytes or more) is deferred, and the pacer never sends it. With + `homa-qdisc on`, echo, cp_node Homa and TCP all stall (phases 10–11 above). +- **Wider reach:** any NIC whose link is down or whose speed is unknown, and a speed of 0 + would divide by zero. +- **Fix:** keep `homa->link_mbps` unless the speed is valid. + [randomizedcoder/HomaModule#23](https://github.com/randomizedcoder/HomaModule/pull/23) + - Test-first: `homa_qdev_update_sysctl__link_speed_unknown` fails on `main` (`link_mbps` + -1) and passes with the fix. + - 885/885 in IPv6 and `--ipv4` modes. + - The pair lifecycle is green on 7.3 and 6.12. + +--- + +### F12: The upstream-shaped source does not build in a kernel tree (`analysis-netdev`) + +`analysis-netdev` puts the files `make net-next` writes into the 7.3 tree as `net/homa` and +builds them as netdev would (defconfig + `HOMA=m`, `W=1`; gcc + sparse, clang, i386). All +three builds fail, for two reasons: + +- **`util/strip.py` keeps both branches of `#ifndef __UNIT_TEST__ … #else … #endif`.** After + `#ifndef` it is in "keep" mode, and `#else /* __UNIT_TEST__ */` sets "keep" again rather + than switching to "drop" (`strip.py:331`). In `homa_tx_pool.c` both definitions of + `homa_tx_pool_mem_nid()` survive: `redefinition of 'homa_tx_pool_mem_nid'`. checkpatch's + "open brace" error is the unit-test stub. Fix: toggle on `#else`. +- **`homa_rpc_alloc_server()` reads `hsk->sock.sk_memcg` directly** (`homa_rpc.c:143`, from + 719ea29 "Add cgroup memory accounting"). `struct sock` has `sk_memcg` only under + `CONFIG_MEMCG`, so a kernel without memcg (x86_64 and i386 defconfig) does not build. + The out-of-tree build only passes because distro kernels enable memcg. Fix: + `mem_cgroup_from_sk(&hsk->sock)`, which `` stubs to NULL without memcg. + +The rest is cleanup a netdev submission would need: 8 kernel-doc warnings (parameters +renamed without their `@` lines: `homa_pool_unlock`, `__homa_tx_pool_alloc_frag`, +`homa_tx_pool_alloc_page`; missing `Return:`), and 31 checkpatch findings (`kzalloc_obj` +and `kmalloc_obj` suggestions from the 7.x rules, two typos, `jiffies` compared without +`time_after`, line lengths, comment alignment). + +### F13: Lock-order inversion between `peertab->lock` and the route table's `ht->lock` (lockdep) + +Found by the first run of `homa-microvm-test-debug` with lockdep on (2026-10-30), during the +smoke test's netns phase, with or without the F8/F11 fixes: + +``` +WARNING: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected +kworker (cleanup_net) is trying to acquire &ht->lock at rhashtable_walk_enter +and this task is already holding &peertab->lock at homa_peer_free_net +``` + +- `peertab->lock` is taken in softirq: `homa_gro_receive → homa_softirq → + homa_dispatch_pkts → homa_rpc_alloc_server → homa_route_get` (`spin_lock_bh`). +- `homa_peer_free_net()` holds it and calls `rhashtable_walk_enter()`, which takes the + table's `ht->lock` with plain `spin_lock`. The rhashtable rehash worker + (`rht_deferred_worker`) holds `ht->lock` with softirqs enabled. +- **The deadlock:** CPU0's rehash worker holds `ht->lock`, and a Homa packet's softirq on + CPU0 then spins on `peertab->lock`. Meanwhile CPU1, in netns exit, holds `peertab->lock` + and spins on `ht->lock`. Neither CPU can make progress. It is rare, since it needs a rehash, + a netns exit and Homa traffic at the same moment, but it is a hard hang. +- `homa_route_pick_victims()` (route GC, under `peertab->lock`) calls `rhashtable_walk_enter()` + the same way (`homa_peer.c:644`). +- Fix direction: walk the table without holding `peertab->lock` (`rhashtable_walk_enter`, + `_start` and `_stop` all take `ht->lock`), and take it only around each removal and count + update, as the rest of net/ walks its rhashtables. + +KASAN also reports F8 by name on `main`: `slab-use-after-free in homa_route_free`, from +an RCU callback. + +### F14: Unannotated lockless read of a deferred RPC's packet queue (KCSAN) + +Found by `homa-pair-lifecycle-kcsan` (with the F8/F11 fixes) on its second run, in the Homa +qdisc phase with the pacer on. The first run, all 39 phases, reported nothing: the race needs +the pacer and the timer to touch the same RPC at the same moment. + +``` +BUG: KCSAN: data-race in homa_qdisc_get_deferred_homa [homa] / homa_rpc_tx_end [homa] +write (marked) ... homa_qdisc_get_deferred_homa <- homa_qdisc_xmit_deferred_homa <- homa_qdisc_pacer +read ... homa_rpc_tx_end <- homa_timer_check_rpc <- homa_timer +value changed: 0x00000086 -> 0x00000084 +``` + +- The pacer dequeues a deferred packet under `qdev->defer_lock` (`__skb_dequeue`, which + writes `qlen` with `WRITE_ONCE`). The timer reads the same queue's length with no lock, in + `homa_qdisc_deferred_offset()` (`homa_qdisc.h:489`, `skb_queue_len()`), and then + `qrpc->tx_left`, which the pacer writes with a plain store (`homa_qdisc.c:953`). +- The comment there says the race is intended ("the worst that can happen is to return a + slightly out-of-date result"), so this is not a logic bug. But the reads are plain loads, + which the compiler may tear or re-fetch, and netdev expects such races to be annotated. +- Fix: `skb_queue_len_lockless()` for the length, and `READ_ONCE`/`WRITE_ONCE` on `tx_left`. + Small, and it makes the KCSAN pair lifecycle clean. + +KCSAN reports nothing else in Homa's functions, in the smoke test or in either pair-lifecycle +run (netem loss and delay, the Homa qdisc and pacer, rmmod). + +### The pair lifecycle on KASAN kernels + +`homa-pair-lifecycle-debug` runs all 39 phases (2026-10-31, with the F8/F11 fixes). KASAN, +KFENCE, KMEMLEAK and UBSAN report nothing; lockdep reports F13 from both sides (node1's report +is `homa_route_free`, in an RCU callback, taking `peertab->lock` after the walk learned +`peertab->lock -> ht->lock`). So the run fails at dmesg and serial-log until F13 is fixed. + +- **Echo 5/6 on the default qdisc, 6/6 on Homa's.** These guests send far slower than the + `net.homa.link_mbps` of 25000 that Homa assumes. Without Homa's qdisc and pacer, the + client's fq_codel dropped 20032 Homa packets in the baseline alone (275 `data_xmit_errors`), + and the 1 MB echo ended in `rpc_timeouts`. With Homa's qdisc (phases 10 and 11) every echo + completes. This is the slow sanitizer guest, not a memory bug: the plain kernels pass the + same phases. It does show what happens when `link_mbps` overstates the link. +- **Boot.** On this host (load about 40), a KASAN guest takes up to 13 minutes before udev + has announced hvc0 and eth1. So the sanitizer guests wait up to 600 s for devices + (`systemd.default_device_timeout_sec`), the boot phase gets `bootTimeoutDebug` (1200 s), + and every lifecycle's virtio phase now waits for `systemctl is-system-running` and + `udevadm settle` before running anything. + +## Follow-up actions + +| # | Action | Why | Effort | Who | +|---|---|---|---|---| +| 1 | ~~F2 experiments~~ ✅ done: QEMU `socket` backend stall, pair tests moved to `dgram` | — | — | — | +| 2 | ~~PLAN-pairs phase 3: `homa-vm-matrix`~~ ✅ done; its first run found F8 | — | — | — | +| 13 | Send F8 upstream once the user has reviewed fork #22 | Use-after-free on every netns exit and rmmod | — | us → upstream | +| 3 | ~~PLAN-pairs phase 4: netem loss and delay~~ ✅ `homa-pair-lifecycle`; found F10 | — | — | — | +| 16 | F10: why 1 ms of delay triggers resends (sweep `resend_ticks` under netem delay) | A quarter of DATA resent with no loss | small | us, then maybe John | +| 4 | ~~PLAN-pairs phase 5: Homa qdisc on virtio~~ ✅ `homa-pair-lifecycle` phases 10–12; reproduced round-4 candidate 5 (F11), fix fork #23 | — | — | — | +| 17 | Send F11 upstream once the user has reviewed fork #23 | The Homa qdisc stalls all traffic on NICs that report no speed | — | us → upstream | +| 5 | ~~Run `homa-microvm-test-debug`, then a KASAN pair test~~ ✅ found F13, F14 (2026-10-31) | Memory-safety coverage of the real data path | — | — | +| 6 | IPv6 pair test (`--ipv6` for cp_node, v6 addresses on eth1) | #137/#138 were IPv6 bugs; nothing tests v6 between nodes | small | us | +| 7 | Round-4 candidates 6–8, test-first (5 is fork #23) | More upstream fixes in John's style | medium | us → upstream | +| 8 | ~~Which LTS will `main` target?~~ ✅ 7.3 or later (John, #120, 2026-10-29) | Closes F1; unit KDIR moves to 7.3 (F5) | — | — | +| 9 | Restore the `homa_drop_packet` call, or drop its sysctls? | F3 | — | ask John | +| 10 | ~~Post F2 upstream~~ not needed: VM link, not Homa | — | — | — | +| 11 | Bootable 7.0–7.2 kernels (PLAN-pairs phase 6) | Full VM coverage of the 7.x series | large (kernel builds) | optional | +| 14 | Does `util/` need to build off x86? (F9) If so, a portable `rdtsc()` and `REG_RIP` guard | riscv64/aarch64 `util/` builds fail | small | ask John | +| 15 | riscv `homa_clock_khz()`: `riscv_timebase / 1000` (F9) | Timers 100× off on riscv in the normal build | small | us → upstream (if John wants riscv) | +| 19 | F13: walk the route table outside `peertab->lock` | A possible hard hang at netns exit; lockdep fails the KASAN VM until it is fixed | small | us → upstream | +| 18 | F12: `strip.py` `#else` toggle and `mem_cgroup_from_sk()`, test-first | The net-next files must build in-tree before any submission | small | us → upstream | +| 20 | F14: `skb_queue_len_lockless()` and `READ_ONCE`/`WRITE_ONCE` on `tx_left` | KCSAN fails the pair lifecycle until the intended race is annotated | small | us → upstream | +| 12 | Keep `nix/fuzz/known-findings.txt` in step with upstream | Entries come out as fixes land, so the fuzz-mock gate stays meaningful; currently empty | ongoing | us | + +--- + +## History + +| Date | Milestone | +|---|---| +| 2026-09-18 | Flake built in phases: design doc, module and user-space derivations, dev shell, microVM, OCI image, tiered static analysis, fuzzing. Round 1 reports in `analysis-reports/round1/` | +| 2026-10-21 | Round 2: rebased on `main`, 6.18 default, hermetic unit KDIR, real-code fuzz-mock (found #121), `analysis-reports/round2/` | +| 2026-10-22 | 7.x: default 7.3, `kernel/src.nix` (7.0–7.2 and local trees), `homa-build-matrix`, per-kernel microVMs, GRO/RPS smoke test | +| 2026-10-28 | The 7.1 patch landed upstream (#120), so `nix/patches/` is empty. Two-VM pair tests, cross-kernel pairs, this status doc. F2 latency sweep (`homa-pair-exp`): RESEND-timer stalls from QEMU's `socket` link, pair tests moved to `dgram` | +| 2026-10-29 | John: `main` will target an LTS of 7.3+ (F1 closed). Phase 3 `homa-vm-matrix`; guests boot with `init_on_free=1`; found F8 (UAF in `homa_route_free`), fix as fork #22. riscv64/aarch64 cross builds (`homa-cross-matrix`, F9). `homa-lifecycle` (pcp-style phases over ttyS0 and hvc0). `homa-pair-lifecycle`: the pair over both consoles, with `tc netem` loss and delay phases (limit 100000); found F10. PLAN-pairs phase 5: Homa's qdisc and pacer in the pair lifecycle; reproduced round-4 candidate 5 (F11), fix as fork #23 | +| 2026-10-30 | Refactor, same targets: the VM drivers in Tcl/Expect with table-driven tcltest suites (`tcl-tests`); flake-parts and one package scope; `homaLib` (source subsets with `lib.fileset`, `mkKbuild`, `perKernel`), `mkAnalysisTool`, one fuzz runner, `ports.nix`; the tree grouped by area, with a README per folder. Fixed on the way: `homa-utils-debug` did not link, and sparse/smatch reported 0 when kbuild could not run | +| 2026-10-30 | Tcl review (Ousterhout-style): a phase script's `return`/`break`/`continue` are recorded and errors keep their stack (`phase-errors.log`); `Batch` restores `HOMA_VM_LOGDIR`; modules `protocol`/`parse`/`report`; `parse netemCheck`, one-pass `delta`; `waitFor`, `Node phase`, a `Pair` class; the echo count from `constants.echoRpcs`; full proc headers checked by `style.test`. The nagelfar gate in `tcl-tests` had never failed (wrong output pattern); fixed. Driver output unchanged | +| 2026-10-30 | Nix workflows for the drivers: `homa-tcl-test` (the suite or part of it: `-file`, `-match`, `--tcl9`) and `homa-tcl-lint`, which the `tcl-tests` check now runs; `homa-vm-compare old new` compares two matrix runs phase by phase (`parse shape`, `report diff`); `nix develop` has Tcl, Expect, tcllib, nagelfar and tclint | +| 2026-10-30 | Nix review and refactor, same targets: the debug VM boots its KASAN/KMEMLEAK kernel (it booted the plain one before) with a homa.ko built for it, and passes; kernel trees keep their name; `homa-lint` is `analysis-quick`; dead code and unused exports removed (drvPaths checked against a baseline); pair tests build their link port from their name and share one cp_node command line; stale comments fixed | +| 2026-10-30 | Demo evidence: `analysis-netdev` (netdev's checks on the `make net-next` files: found F12, they do not build in-tree), `homa-tests-coverage` (gcov: 86.7 % of lines), sanitizer VMs (`-debug`: KASAN, KFENCE, KMEMLEAK, lockdep, UBSAN; `-kcsan`) for the smoke test and the pair lifecycle (found F13 and F14; KASAN names F8), and `homa-evidence`, one HTML report of all of it. The netem drop check accepts a quarter of the configured loss (0.1 % measured 0.04 % over 82k packets) | diff --git a/nix/analysis/README.md b/nix/analysis/README.md new file mode 100644 index 00000000..d8f5e31d --- /dev/null +++ b/nix/analysis/README.md @@ -0,0 +1,50 @@ +# nix/analysis: static analysis in tiers + +Each tool is a derivation whose output is a report (`report.txt`) and a finding count +(`count.txt`). A tier runs several tools and adds a `summary.txt`. The tools report +findings; they do not fail on them. Only `analysis-quick` is a `nix flake check`. + +| Tier | Tools | Needs a kernel | +|---|---|---| +| `analysis-quick` | ruff (Python), shellcheck and shfmt (shell), clang-format | no | +| `analysis-standard` | + cppcheck, clang-tidy (from a compile database) | yes | +| `analysis-deep` | + sparse (with endianness checks), smatch | yes | +| `analysis-netdev` | netdev's checks on the code as submitted (below) | yes | + +```sh +nix build .#analysis-standard && cat result/summary.txt +nix build .#analysis-sparse && less result/report.txt # one tool +``` + +`analysis-netdev` checks what a reviewer on netdev would see: the upstream-shaped source +that `make net-next` writes (`util/strip.py`), placed in the default kernel's tree as +`net/homa`, with `net/Kconfig` and `net/Makefile` wired as an upstream patch would. Its +tools are `checkpatch.pl --strict --codespell`, `kernel-doc -Wall`, and in-tree `make W=1` +builds of `net/homa` with gcc and sparse, clang (`LLVM=1`), and 32-bit x86 (`ARCH=i386`). +A build that never compiles a `net/homa` file fails, rather than reporting zero. +`analysis-netdev-tree` is that kernel tree, for trying a fix by hand. + +| File | What it is | +|---|---| +| `default.nix` | `mkAnalysisTool`, the tiers, and the per-tool `analysis-` packages | +| `python.nix`, `shell.nix`, `clang-format.nix`, `cppcheck.nix` | one `mkAnalysisTool` call each | +| `compile-db.nix` | `compile_commands.json`, recorded with `bear` around a kbuild build | +| `clang-tidy.nix` | clang-tidy over that compile database | +| `kbuild-check.nix` | a checker kbuild runs on every file (`make C=2 CHECK=...`): sparse and smatch | +| `netdev.nix` | the `analysis-netdev` tree and its five checks | + +## Adding a tool + +```nix +mytool = mkAnalysisTool { + name = "mytool"; + tools = [ pkgs.mytool ]; + script = "mytool . > $out/report.txt"; + count = ": warning: "; # an extended regex matching one finding +}; +``` + +`script` runs in a writable copy of Homa's source (`homaLib.src.module` unless `src` says +otherwise). `kbuild = true` adds the compiler and the default kernel's build dependencies. +Each tool file is called with `lib.callPackageWith`, so it names the arguments it needs +(`mkAnalysisTool`, `pkgs`, `kernelConfig`, ...) and gets just those. diff --git a/nix/analysis/clang-format.nix b/nix/analysis/clang-format.nix new file mode 100644 index 00000000..e8ca5ff1 --- /dev/null +++ b/nix/analysis/clang-format.nix @@ -0,0 +1,25 @@ +# nix/analysis/clang-format.nix +# +# clang-format compliance check for the core C sources/headers. Homa follows Linux kernel +# style (tabs, checkpatch-strict), so we check against the built-in LinuxKernel style rather +# than imposing a new one (avoids massive churn). + +{ pkgs, mkAnalysisTool }: + +mkAnalysisTool { + name = "clang-format"; + tools = [ pkgs.clang-tools ]; + script = '' + # Core module sources/headers (top-level homa_*.c/.h + timetrace). + mapfile -t cfiles < <(find . -maxdepth 1 -type f \( -name 'homa_*.c' -o -name 'homa_*.h' \ + -o -name 'homa.h' -o -name 'timetrace.c' -o -name 'timetrace.h' \) | sort) + { + echo "# clang-format (style=LinuxKernel): ''${#cfiles[@]} files checked" + for f in "''${cfiles[@]}"; do + clang-format --style=LinuxKernel --dry-run --Werror "$f" >/dev/null 2>&1 \ + || echo "would-reformat: $f" + done + } > "$out/report.txt" + ''; + count = "^would-reformat: "; +} diff --git a/nix/analysis/clang-tidy.nix b/nix/analysis/clang-tidy.nix new file mode 100644 index 00000000..84f94e9d --- /dev/null +++ b/nix/analysis/clang-tidy.nix @@ -0,0 +1,32 @@ +# nix/analysis/clang-tidy.nix +# +# clang-tidy over the core C sources using the compile DB from compile-db.nix. The DB's +# absolute paths (from the compile-db build sandbox) are rewritten onto this copy of the +# tree. Kernel-code clang-tidy via a captured DB is inherently best-effort; if the DB is +# empty (module didn't build) this yields 0 findings, which is recorded honestly. + +{ pkgs, mkAnalysisTool, compile-db }: + +mkAnalysisTool { + name = "clang-tidy"; + tools = [ pkgs.clang-tools pkgs.jq ]; + script = '' + # Rewrite the DB's recorded source root onto our copy so clang-tidy finds the files. + srcroot=$(cat ${compile-db}/srcroot) + sed "s|$srcroot|$PWD|g" ${compile-db}/compile_commands.json > compile_commands.json + + mapfile -t files < <(jq -r '.[].file' compile_commands.json 2>/dev/null \ + | grep -E 'homa_.*\.c$' | sort -u || true) + + { + echo "# clang-tidy (bugprone-*,cert-*,clang-analyzer-*,misc-*): ''${#files[@]} file(s) in DB" + for f in "''${files[@]}"; do + [ -f "$f" ] || continue + clang-tidy -p "$PWD" \ + --checks='-*,bugprone-*,cert-*,clang-analyzer-*,misc-*' \ + "$f" 2>&1 || true + done + } > "$out/report.txt" + ''; + count = ": (warning|error): "; +} diff --git a/nix/analysis/compile-db.nix b/nix/analysis/compile-db.nix new file mode 100644 index 00000000..850fb22a --- /dev/null +++ b/nix/analysis/compile-db.nix @@ -0,0 +1,38 @@ +# nix/analysis/compile-db.nix +# +# Builds a compile_commands.json for the module by running the kbuild compile under `bear`. +# This is kernel-dependent (needs moduleBuildDependencies + KDIR) and is the input to +# clang-tidy. If the module fails to build (kernel API drift), an empty DB is emitted so the +# downstream analysis still evaluates (the drift is recorded, not fatal). +# +# Also records the build's source root ($out/srcroot) so clang-tidy can rewrite the DB's +# absolute paths onto its own copy of the tree. + +{ pkgs, homaLib, kernelConfig }: + +homaLib.mkKbuild { + pname = "homa-compile-db"; + kdir = kernelConfig.kdir; + nativeBuildInputs = [ pkgs.bear ] ++ kernelConfig.kernelPackages.kernel.moduleBuildDependencies; + + buildPhase = '' + runHook preBuild + echo "$PWD" > srcroot + bear --output compile_commands.json -- make $makeFlags || true + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + mkdir -p "$out" + cp srcroot "$out/srcroot" + if [ -f compile_commands.json ]; then + cp compile_commands.json "$out/compile_commands.json" + else + echo '[]' > "$out/compile_commands.json" + fi + runHook postInstall + ''; + + dontFixup = true; +} diff --git a/nix/analysis/cppcheck.nix b/nix/analysis/cppcheck.nix new file mode 100644 index 00000000..788a6496 --- /dev/null +++ b/nix/analysis/cppcheck.nix @@ -0,0 +1,33 @@ +# nix/analysis/cppcheck.nix +# +# cppcheck over the core C sources. Runs standalone (no compile DB needed) so it stays in +# the `standard` tier without requiring a kernel build. Kernel headers are absent, so many +# macros are unknown — we suppress missingInclude/unknownMacro noise and focus on cppcheck's +# own flow/bounds/leak checks. + +{ pkgs, mkAnalysisTool }: + +mkAnalysisTool { + name = "cppcheck"; + tools = [ pkgs.cppcheck ]; + script = '' + mapfile -t cfiles < <(find . -maxdepth 1 -type f -name 'homa_*.c' | sort) + cfiles+=("./timetrace.c") + { + echo "# cppcheck (warning,performance,portability; std=c11)" + cppcheck \ + --enable=warning,performance,portability \ + --std=c11 \ + --language=c \ + -I. \ + --suppress=missingInclude \ + --suppress=missingIncludeSystem \ + --suppress=unknownMacro \ + --inline-suppr \ + --quiet \ + --template='{file}:{line}: {severity}: {id}: {message}' \ + "''${cfiles[@]}" 2>&1 || true + } > "$out/report.txt" + ''; + count = "^[^:]+:[0-9]+: "; +} diff --git a/nix/analysis/default.nix b/nix/analysis/default.nix new file mode 100644 index 00000000..d021a1c0 --- /dev/null +++ b/nix/analysis/default.nix @@ -0,0 +1,134 @@ +# nix/analysis/default.nix +# +# Tiered static analysis (xdp2/rocm pattern). Each tool is one mkAnalysisTool call (below) +# that writes $out/report.txt and $out/count.txt; the tier targets gather those into one +# summary.txt with per-tool finding counts. +# +# analysis-quick python (ruff) + shell (shellcheck/shfmt) + clang-format (no kernel) +# analysis-standard + cppcheck + clang-tidy (compile DB) +# analysis-deep + sparse (endianness) + smatch +# analysis-netdev netdev's checks on the code as submitted (netdev.nix): checkpatch +# --strict, kernel-doc, in-tree W=1 builds (gcc + sparse, clang, i386) +# +# Per-tool packages are also exposed (analysis-ruff, analysis-cppcheck, ...). nix/scope.nix +# calls this file with callPackage; each tool file is called the same way, and gets +# mkAnalysisTool. + +{ pkgs +, lib +, homaLib +, kernelConfig +}: + +let + # mkAnalysisTool { name, tools, script, count, src?, kbuild? }: copies the source (by + # default Homa's module sources), runs script, which writes "$out/report.txt", then + # writes into count.txt how many report lines match the extended regex count. + # kbuild = true adds what a build against the default kernel needs, a C compiler + # included (runCommandCC; plain runCommand has none). + mkAnalysisTool = { name, tools, script, count, src ? homaLib.src.module, kbuild ? false }: + (if kbuild then pkgs.runCommandCC else pkgs.runCommand) "homa-analysis-${name}" + { + # stdenv already has coreutils, findutils, grep, sed and awk on PATH. + nativeBuildInputs = tools + ++ lib.optionals kbuild + ([ pkgs.gnumake ] ++ kernelConfig.kernelPackages.kernel.moduleBuildDependencies); + hardeningDisable = lib.optional kbuild "all"; + } '' + cp -r ${src} src && chmod -R +w src && cd src + mkdir -p "$out" + ${script} + count=$(grep -cE '${count}' "$out/report.txt" || true) + echo "''${count:-0}" > "$out/count.txt" + ''; + + # callPackageWith: like callPackage, with these added to what a file can name. + call = lib.callPackageWith (pkgs // { inherit homaLib kernelConfig mkAnalysisTool; }); + kbuildCheck = call ./kbuild-check.nix { }; + + python = call ./python.nix { }; + shell = call ./shell.nix { }; + clang-format = call ./clang-format.nix { }; + cppcheck = call ./cppcheck.nix { }; + compile-db = call ./compile-db.nix { }; + clang-tidy = call ./clang-tidy.nix { inherit compile-db; }; + # sparse: the kernel's own semantic checker, with endianness checks on. It flags + # mismatched __be16/__be32/__be64 handling and __force casts in the wire code, which + # fuzzing can miss. + sparse = kbuildCheck { + name = "sparse"; + checker = "${pkgs.sparse}/bin/sparse"; + flags = "-D__CHECK_ENDIAN__ -Wsparse-all"; + count = ":[0-9]+:[0-9]+: (warning|error):"; + }; + # smatch: kernel-oriented flow analysis, when nixpkgs has it. + smatch = kbuildCheck { + name = "smatch"; + checker = "${pkgs.smatch}/bin/smatch -p=kernel"; + count = ":[0-9]+:[0-9]+: (warn|error):"; + }; + hasSmatch = pkgs ? smatch; + netdev = call ./netdev.nix { }; + + # A tier is a runCommand that gathers named tool reports into one directory + summary. + mkTier = tierName: tools: + let + links = lib.concatMapStringsSep "\n" + (t: '' + cp "${t.drv}/report.txt" "$out/${t.name}.report.txt" + c=$(cat "${t.drv}/count.txt") + printf ' %-14s %s finding(s)\n' "${t.name}:" "$c" >> "$out/summary.txt" + total=$((total + c)) + '') + tools; + in + pkgs.runCommand "homa-${tierName}" { } '' + mkdir -p "$out" + total=0 + echo "=== Homa ${tierName} ===" > "$out/summary.txt" + ${links} + echo "" >> "$out/summary.txt" + echo "Total: $total finding(s)" >> "$out/summary.txt" + echo "$total" > "$out/count.txt" + cat "$out/summary.txt" + ''; + + quickTools = [ + { name = "ruff"; drv = python; } + { name = "shell"; drv = shell; } + { name = "clang-format"; drv = clang-format; } + ]; + standardTools = quickTools ++ [ + { name = "cppcheck"; drv = cppcheck; } + { name = "clang-tidy"; drv = clang-tidy; } + ]; + deepTools = standardTools ++ [ + { name = "sparse"; drv = sparse; } + ] ++ lib.optional hasSmatch { name = "smatch"; drv = smatch; }; + + analysis-quick = mkTier "analysis-quick" quickTools; + analysis-standard = mkTier "analysis-standard" standardTools; + analysis-deep = mkTier "analysis-deep" deepTools; + + # Not a superset of the others: the checks netdev runs on a submission (netdev.nix). + netdevTools = map (name: { inherit name; drv = netdev.${name}; }) + [ "checkpatch" "kdoc" "gcc-w1" "clang-w1" "i386-w1" ]; + analysis-netdev = mkTier "analysis-netdev" netdevTools; +in +{ + packages = { + inherit analysis-quick analysis-standard analysis-deep analysis-netdev; + analysis-netdev-tree = netdev.tree; + analysis-ruff = python; + analysis-shell = shell; + analysis-clang-format = clang-format; + analysis-cppcheck = cppcheck; + analysis-clang-tidy = clang-tidy; + analysis-compile-db = compile-db; + analysis-sparse = sparse; + } // lib.optionalAttrs hasSmatch { analysis-smatch = smatch; } + // lib.listToAttrs (map (t: lib.nameValuePair "analysis-netdev-${t.name}" t.drv) netdevTools); + + # `analysis-quick` is kernel-independent, so it makes a good `nix flake check` gate. + checks = { inherit analysis-quick; }; +} diff --git a/nix/analysis/kbuild-check.nix b/nix/analysis/kbuild-check.nix new file mode 100644 index 00000000..2b7fff32 --- /dev/null +++ b/nix/analysis/kbuild-check.nix @@ -0,0 +1,36 @@ +# nix/analysis/kbuild-check.nix +# +# A checker that kbuild runs on every source file (make C=2 CHECK=...), against the default +# kernel: sparse and smatch (deep tier). Best-effort: if the module does not build against +# that kernel, the report still holds what the checker and kbuild printed. But it fails if +# kbuild never ran the checker at all (no "CHECK" line), so a broken setup cannot pass as +# zero findings. +# +# kbuildCheck { name, checker, flags ? "", count } +# checker the command kbuild runs per file (CHECK=) +# flags passed to it (CF=) +# count extended regex for one finding in its output + +{ kernelConfig, mkAnalysisTool }: + +{ name, checker, flags ? "", count }: + +mkAnalysisTool { + inherit name count; + tools = [ ]; + kbuild = true; + script = '' + # C=2 checks every file, not just those make rebuilds. + make KDIR=${kernelConfig.kdir} MY_CFLAGS='-O2' \ + C=2 CHECK="${checker}" CF="${flags}" > checker.stdout.log 2> checker.stderr.log || true + { + echo "# ${name} ${flags}" + cat checker.stderr.log checker.stdout.log + } > "$out/report.txt" + if ! grep -qE '^ +CHECK ' checker.stdout.log; then + cat "$out/report.txt" + echo "${name}: kbuild never ran the checker" + exit 1 + fi + ''; +} diff --git a/nix/analysis/netdev.nix b/nix/analysis/netdev.nix new file mode 100644 index 00000000..c380ba06 --- /dev/null +++ b/nix/analysis/netdev.nix @@ -0,0 +1,124 @@ +# nix/analysis/netdev.nix +# +# The checks a Homa patch series meets on netdev, run on the code as it is submitted: the +# upstream-shaped source that `make net-next` writes (util/strip.py), inside a real kernel +# tree (the default kernel's), with net/Kconfig and net/Makefile wired as an upstream patch +# would. Each check is one mkAnalysisTool report: +# +# checkpatch scripts/checkpatch.pl --file --strict --codespell, the flags of Homa's own +# `make checkpatch-net-next` +# kdoc scripts/kernel-doc -none -Wall +# gcc-w1 an in-tree x86_64 build of net/homa with W=1, and sparse (C=1) +# clang-w1 the same with LLVM=1 +# i386-w1 a 32-bit build (ARCH=i386) with W=1: 64-bit divisions, pointer casts, ... +# +# A build check fails (rather than reporting zero findings) if kbuild never compiled a +# net/homa file. + +{ pkgs +, lib +, homaLib +, kernelConfig +, mkAnalysisTool +}: + +let + kernel = kernelConfig.kernelPackages.kernel; + + # The kernel source with Homa's stripped files in net/homa and include/uapi/linux/homa.h. + tree = pkgs.runCommand "homa-netdev-tree-${kernel.version}" + { nativeBuildInputs = with pkgs; [ gnumake perl python3 ]; } '' + mkdir "$out" + tar xf ${kernel.src} -C "$out" --strip-components=1 + patchShebangs "$out/scripts" + cp -r ${homaLib.src.util} homa && chmod -R +w homa + patchShebangs homa/util/strip.py + make -C homa net-next LINUX_SRC_DIR="$out" + sed -i '/^source "net\/sctp\/Kconfig"/a source "net/homa/Kconfig"' "$out/net/Kconfig" + echo 'obj-$(CONFIG_HOMA) += homa/' >> "$out/net/Makefile" + grep -q 'net/homa/Kconfig' "$out/net/Kconfig" + ''; + + # Relative to the tree, where each check runs, so its report shows short paths. + homaFiles = "net/homa/*.[ch] include/uapi/linux/homa.h"; + + checkpatch = mkAnalysisTool { + name = "netdev-checkpatch"; + tools = [ pkgs.perl pkgs.codespell ]; + script = '' + dict=$(find ${pkgs.codespell} -name dictionary.txt -path '*codespell_lib/data/*' | head -n 1) + { + echo "# checkpatch.pl --file --strict --codespell (net/homa as submitted)" + cd ${tree} + scripts/checkpatch.pl --file --strict --codespell \ + --codespellfile "$dict" --color=never ${homaFiles} 2>&1 || true + } > "$out/report.txt" + ''; + count = "^(ERROR|WARNING|CHECK):"; + }; + + kdoc = mkAnalysisTool { + name = "netdev-kdoc"; + tools = [ pkgs.perl pkgs.python3 ]; + script = '' + cd ${tree} + # -none prints nothing for clean files, so first check that it parses Homa's at all. + python3 scripts/kernel-doc -rst net/homa/homa_rpc.c | grep -q homa_rpc_alloc_server \ + || { echo "kernel-doc found no documentation in net/homa/homa_rpc.c"; exit 1; } + { + echo "# kernel-doc -none -Wall (net/homa as submitted)" + # It prints some warnings twice (once per pass); keep the first. + python3 scripts/kernel-doc -none -Wall ${homaFiles} 2>&1 | awk '!seen[$0]++' || true + } > "$out/report.txt" + ''; + count = "^(Warning|warning|error)"; + }; + + # An in-tree build of net/homa: defconfig, HOMA=m, then the directory alone. + inTreeBuild = { name, makeFlags ? "", defconfig ? "defconfig", tools ? [ ] }: mkAnalysisTool { + inherit name; + kbuild = true; + tools = with pkgs; [ bc bison flex perl python3 openssl elfutils ] ++ tools; + script = '' + mk() { make -C ${tree} O="$PWD/build" ${makeFlags} "$@"; } + mk ${defconfig} > config.log 2>&1 + ${tree}/scripts/config --file build/.config -e IPV6 -m HOMA + mk olddefconfig >> config.log 2>&1 + grep -q '^CONFIG_HOMA=m' build/.config || { cat config.log; echo "HOMA not enabled"; exit 1; } + # -k: a file that fails to compile must not hide the other files' warnings. + mk -k -j"$NIX_BUILD_CORES" W=1 net/homa/ > build.log 2>&1 || echo "make failed" >> build.log + { + echo "# make W=1 ${makeFlags} net/homa/ (${defconfig})" + grep -E 'net/homa/|include/uapi/linux/homa\.h|make failed' build.log \ + | grep -vE '^ +(CC|CHECK) |\*\*\*|not remade' | sed "s|${tree}/||g" || true + } > "$out/report.txt" + if ! grep -qE '^ +CC +\[?M?\]? *net/homa/' build.log; then + tail -n 40 build.log + echo "${name}: kbuild never compiled net/homa" + exit 1 + fi + ''; + count = "(warning|error):|make failed"; + }; + + gcc-w1 = inTreeBuild { + name = "netdev-gcc-w1"; + makeFlags = "C=1 CHECK=${pkgs.sparse}/bin/sparse"; + tools = [ pkgs.sparse ]; + }; + clang-w1 = inTreeBuild { + name = "netdev-clang-w1"; + # The kernel passes its own --target, which nixpkgs' clang wrapper does not allow; host + # tools still use the wrapped clang (LLVM=1 sets HOSTCC=clang). + makeFlags = "LLVM=1 CC=${pkgs.llvmPackages.clang-unwrapped}/bin/clang"; + tools = with pkgs; [ clang lld llvm ]; + }; + i386-w1 = inTreeBuild { + name = "netdev-i386-w1"; + makeFlags = "ARCH=i386"; + defconfig = "i386_defconfig"; + }; +in +{ + inherit tree checkpatch kdoc gcc-w1 clang-w1 i386-w1; +} diff --git a/nix/analysis/python.nix b/nix/analysis/python.nix new file mode 100644 index 00000000..f2dfd24c --- /dev/null +++ b/nix/analysis/python.nix @@ -0,0 +1,28 @@ +# nix/analysis/python.nix +# +# Python static analysis with ruff. Homa's scripts include EXTENSIONLESS python programs +# (util/cp_* with a `#!/usr/bin/python3` shebang), so files are discovered by shebang as +# well as by `.py` extension — a plain `*.py` glob would miss them. + +{ pkgs, homaLib, mkAnalysisTool }: + +mkAnalysisTool { + name = "python"; + tools = [ pkgs.ruff ]; + src = homaLib.src.all; + script = '' + export RUFF_CACHE_DIR="$TMPDIR/ruff" + # A shebang must be the first line (grep -l would match a "#!" on any line). + { find . -type f -exec awk 'FNR == 1 && /^#!.*python/ { print FILENAME } { nextfile }' {} + + find . -name '*.py'; } | sort -u > "$out/files.txt" + mapfile -t pyfiles < "$out/files.txt" + { + echo "# ruff check ($(wc -l < "$out/files.txt") files)" + if [ "''${#pyfiles[@]}" -gt 0 ]; then + ruff check --output-format=concise "''${pyfiles[@]}" 2>&1 || true + fi + } > "$out/report.txt" + ''; + # ruff's concise format: path:line:col: code message + count = "^[^:]+:[0-9]+:[0-9]+:"; +} diff --git a/nix/analysis/shell.nix b/nix/analysis/shell.nix new file mode 100644 index 00000000..696dc548 --- /dev/null +++ b/nix/analysis/shell.nix @@ -0,0 +1,32 @@ +# nix/analysis/shell.nix +# +# Shell static analysis with shellcheck + shfmt. Homa has NO *.sh files — its shell scripts +# are extensionless (util/send_many, util/get_traces, cloudlab/bin/*), so discovery is by +# shebang. + +{ pkgs, homaLib, mkAnalysisTool }: + +mkAnalysisTool { + name = "shell"; + tools = [ pkgs.shellcheck pkgs.shfmt ]; + src = homaLib.src.all; + script = '' + # The shebang must be the first line (grep -l would match a "#!" on any line). + find . -type f -exec awk 'FNR == 1 && /^#!.*(bash|\/bin\/sh|env sh)/ { print FILENAME } + { nextfile }' {} + | sort -u > "$out/files.txt" + mapfile -t shfiles < "$out/files.txt" + { + echo "# shellcheck ($(wc -l < "$out/files.txt") files)" + if [ "''${#shfiles[@]}" -gt 0 ]; then + shellcheck --severity=warning "''${shfiles[@]}" 2>&1 || true + fi + echo "" + echo "# shfmt -d (formatting diffs)" + if [ "''${#shfiles[@]}" -gt 0 ]; then + shfmt -d "''${shfiles[@]}" 2>&1 || true + fi + } > "$out/report.txt" + ''; + # One per shellcheck finding: "In line N:" + count = "^In .* line [0-9]+:"; +} diff --git a/nix/build/README.md b/nix/build/README.md new file mode 100644 index 00000000..4779b3fb --- /dev/null +++ b/nix/build/README.md @@ -0,0 +1,35 @@ +# nix/build: homa.ko, the user-space tools, the unit tests, the OCI image + +| File | Outputs | +|---|---| +| `module.nix` | `homa-module` (the default kernel), `homa-module-debug` (`-g -O1`), `homa-module-` for every nixpkgs kernel, `homa-module-local` (your tree, `--impure`), and `homa-build-matrix`, which tries every kernel and writes `summary.md` | +| `userspace.nix` | `homa-utils[-debug]` (`util/`; `-debug` adds ASan) and `homa-tests`, which builds and runs the unit tests in `test/` | +| `cross.nix` | `homa-module-{riscv64,aarch64}[-strip]`, `homa-utils-{riscv64,aarch64}`, `linux-kdir--` and `homa-cross-matrix` (x86_64 hosts only) | +| `oci.nix` | `homa-image[-debug]`: the user-space tools in an OCI image (`docker load < result`) | + +```sh +nix build .#homa-module # result/lib/modules//homa.ko +nix build .#homa-build-matrix && cat result/summary.md +nix build .#homa-tests # fails if a unit test fails +``` + +## How the module build works + +Homa's own `Makefile` does the work, as it would by hand: `make KDIR=`. +`homaLib.mkKbuild` ([`../lib`](../lib)) wraps that in a derivation: the source is +`homaLib.src.module`, the kernel tree comes from [`../kernel`](../kernel), and kbuild's +own flags replace nixpkgs' hardening flags. A build is one function call per kernel: + +```nix +buildModule { kernelPackages = pkgs.linuxPackages_6_12; enableDebug = true; } +``` + +`homa-build-matrix` does not fail when a kernel fails. Its job is the table: which kernels +Homa builds on today. + +## Cross builds + +`cross.nix` uses nixpkgs' `pkgsCross.`, a copy of nixpkgs whose compilers target +another architecture. Calling `userspace.nix` through `pkgsCross.riscv64.callPackage` is +all it takes to cross-compile the tools; the module builds against a riscv64 or aarch64 +kernel tree prepared from source. diff --git a/nix/build/cross.nix b/nix/build/cross.nix new file mode 100644 index 00000000..5d06e941 --- /dev/null +++ b/nix/build/cross.nix @@ -0,0 +1,90 @@ +# nix/build/cross.nix +# +# Cross-compile checks (xdp2 pattern): build Homa for other architectures on an x86_64 +# host, with nixpkgs' cached cross toolchains (pkgsCross), and no emulation. +# +# homa-module- homa.ko for the default kernel (7.3), cross-compiled against a +# prepared tree (kernel/src.nix mkTree) using nixpkgs' +# kernel config; checks homa.ko's ELF machine +# homa-module--strip ... the same for the upstream-shaped source (__STRIP__=1) +# homa-utils- util/ (cp_node, demo_*, ...) built with the cross toolchain +# homa-cross-matrix every arch: module results plus a `make -k` report of util/, +# in summary.md; fails only if a homa.ko does not build +# +# is riscv64 or aarch64. The kernel trees take about 2 minutes each (modules_prepare, +# no vmlinux), versus about 30 for a full cross kernel build. +# +# nix/scope.nix calls this file with callPackage, passing buildAgainstTree from module.nix. + +{ pkgs +, lib +, kernelConfig +, kernelSrc +, homaLib +, unitKdir +, buildAgainstTree +}: + +let + # arch: the kernel's ARCH=; machine: what readelf -h prints for the target. + targets = { + riscv64 = { cross = pkgs.pkgsCross.riscv64; arch = "riscv"; machine = "RISC-V"; }; + aarch64 = { cross = pkgs.pkgsCross.aarch64-multiplatform; arch = "arm64"; machine = "AArch64"; }; + }; + + # Same source as the default x86 kernel, so every arch builds the same kernel version. + kernel = kernelConfig.kernelPackages.kernel; + series = kernelConfig.seriesOf kernelConfig.kernelPackages; + + perTarget = name: t: + let + cc = t.cross.stdenv.cc; + tree = kernelSrc.mkTree { + name = "${name}-${series}"; + inherit (kernel) version src; + configfile = t.cross.linuxPackages_latest.kernel.configfile; + makeFlags = [ "ARCH=${t.arch}" "CROSS_COMPILE=${cc.targetPrefix}" ]; + extraInputs = [ cc ]; + }; + module = buildAgainstTree name tree { inherit (t) machine; }; + moduleStrip = buildAgainstTree "${name}-strip" tree { inherit (t) machine; strip = true; }; + # The same file as the native tools, called from the cross package set, so its + # stdenv is the cross one. + userspace = t.cross.callPackage ./userspace.nix { inherit homaLib unitKdir; }; + utils = userspace.homa-utils; + utilsReport = userspace.buildUtils { report = true; }; + in + { inherit tree module moduleStrip utils utilsReport; inherit (t) machine; }; + + all = lib.mapAttrs perTarget targets; + + homa-cross-matrix = pkgs.runCommand "homa-cross-matrix" { } '' + mkdir -p "$out" + { + echo "# Cross-compile matrix (Linux ${kernel.version})" + ${lib.concatMapStrings (name: let a = all.${name}; in '' + echo + echo "## ${name}" + echo + echo "- homa.ko: PASS (${a.machine}), ${a.module}" + echo "- homa.ko (__STRIP__=1): PASS (${a.machine}), ${a.moduleStrip}" + echo "- compiler warnings in homa.ko: $(grep -cE "^[^ ]+:[0-9]+:[0-9]+: warning:" ${a.module}/build.log || true)" + echo + cat ${a.utilsReport}/report.md + cp ${a.utilsReport}/build.log "$out/utils-${name}.log" + '') (lib.attrNames all)} + } > "$out/summary.md" + cat "$out/summary.md" + ''; +in +{ + packages = lib.concatMapAttrs + (name: a: { + "homa-module-${name}" = a.module; + "homa-module-${name}-strip" = a.moduleStrip; + "homa-utils-${name}" = a.utils; + "linux-kdir-${name}-${series}" = a.tree; + }) + all + // { inherit homa-cross-matrix; }; +} diff --git a/nix/build/module.nix b/nix/build/module.nix new file mode 100644 index 00000000..c0f7a4c2 --- /dev/null +++ b/nix/build/module.nix @@ -0,0 +1,156 @@ +# nix/build/module.nix +# +# The homa.ko out-of-tree kernel module, built with the standard nixpkgs kbuild idiom +# (kernel.moduleBuildDependencies + KDIR pointing at the kernel's build tree). +# +# Debug vs non-debug is a single boolean (`enableDebug`), mirroring xdp2's `enableAsserts`: +# release: MY_CFLAGS='-O2' +# debug: MY_CFLAGS='-g -O1 -fno-omit-frame-pointer' +# Homa has no formal DEBUG macro, so at the C level "debug" means opt-level/symbols only. +# Real memory-safety coverage comes from the microVM's KASAN kernel (see docs/DESIGN.md §4). +# +# `buildModule` is also reused by nix/microvms/ to build the .ko against the SAME +# kernelPackages the guest boots (co-pinning avoids vermagic drift). +# +# Usage (nix/scope.nix calls this file with callPackage): +# modulePkgs.homa-module # release .ko (default kernel) +# modulePkgs.packages # { homa-module, homa-module-debug, homa-module-..., +# # homa-build-matrix } +# modulePkgs.buildModule { kernelPackages = ; enableDebug = false; } +# +# Per-kernel targets: +# homa-module- nixpkgs kernels (kernel/config.nix matrix): a real .ko, loadable in the +# matching microVM. +# homa-module- kernel-src trees (7.0-7.2, local): compile check only, built with +# KBUILD_MODPOST_WARN=1 because those trees have no Module.symvers. +# homa-build-matrix one derivation that tries every kernel above and writes a +# pass/fail table plus logs; it never fails itself. + +{ pkgs +, lib +, homaLib +, kernelConfig +, kernelSrc +}: + +let + inherit (homaLib) mkKbuild perKernel; + + buildModule = { kernelPackages, enableDebug ? false }: + let + kernel = kernelPackages.kernel; + modDirVersion = kernel.modDirVersion; + myCflags = if enableDebug then "-g -O1 -fno-omit-frame-pointer" else "-O2"; + in + mkKbuild { + pname = "homa-module${lib.optionalString enableDebug "-debug"}"; + # The top Makefile's default target `all` runs: + # $(MAKE) -C $(KDIR) M=$(shell pwd) modules + # so we only need to point KDIR at this kernel's build tree. + kdir = kernelConfig.kdirOf kernelPackages; + nativeBuildInputs = kernel.moduleBuildDependencies; + env.MY_CFLAGS = myCflags; + + installPhase = '' + runHook preInstall + mkdir -p "$out/lib/modules/${modDirVersion}" + cp homa.ko "$out/lib/modules/${modDirVersion}/" + # Record what this .ko was built against, for the microVM / debugging. + { + echo "modDirVersion=${modDirVersion}" + echo "enableDebug=${lib.boolToString enableDebug}" + echo "MY_CFLAGS=${myCflags}" + } > "$out/lib/modules/${modDirVersion}/homa.ko.buildinfo" + runHook postInstall + ''; + + # Never strip or patchelf a kernel module. + dontStrip = true; + dontPatchELF = true; + + meta = homaLib.meta // { + description = "Homa transport protocol Linux kernel module (homa.ko)" + + lib.optionalString enableDebug " [debug]"; + }; + }; + + homa-module = buildModule { inherit (kernelConfig) kernelPackages; }; + homa-module-debug = buildModule { inherit (kernelConfig) kernelPackages; enableDebug = true; }; + + # Compile-test matrix: homa-module- against each nixpkgs kernel (fallback for + # API drift on the newest kernel). + matrixPkgs = perKernel "homa-module" + (_: kp: buildModule { kernelPackages = kp; }) + kernelConfig.matrix; + + # Compile-only builds against kernels prepared from source (nix/kernel/src.nix). A cross + # tree (cross.nix) brings its ARCH=/CROSS_COMPILE= flags and its cross gcc along. + # strip: build the upstream-shaped source (make __STRIP__=1) + # machine: if set, readelf must report this e_machine for homa.ko (e.g. "RISC-V") + buildAgainstTree = name: tree: { strip ? false, machine ? null }: mkKbuild { + pname = "homa-module-${name}"; + kdir = "${tree}/build"; + nativeBuildInputs = tree.nativeBuildInputs; + buildInputs = tree.buildInputs; + makeFlags = [ "KBUILD_MODPOST_WARN=1" ] ++ tree.makeFlags + ++ lib.optional strip "__STRIP__=1"; + # Keep the log, so homa-cross-matrix can count compiler warnings. + buildPhase = '' + runHook preBuild + make -j"$NIX_BUILD_CORES" $makeFlags 2>&1 | tee build.log + runHook postBuild + ''; + installPhase = '' + ${lib.optionalString (machine != null) '' + readelf -h homa.ko | grep -q 'Machine: *${machine}' \ + || { readelf -h homa.ko; echo "homa.ko is not ${machine}"; exit 1; } + ''} + mkdir -p "$out" + cp homa.ko build.log "$out/" + ''; + dontFixup = true; + }; + srcPkgs = perKernel "homa-module" + (name: tree: buildAgainstTree name tree { }) + kernelSrc.trees; + + # Every kernel, one report. Each build runs in its own copy of the source; a failure is + # recorded, not propagated, so one broken kernel does not hide the others. + matrixKdirs = + lib.mapAttrsToList (n: kp: { name = n; version = kp.kernel.version; kdir = kernelConfig.kdirOf kp; extra = ""; }) + kernelConfig.matrix + ++ lib.mapAttrsToList (n: t: { name = n; version = t.version; kdir = "${t}/build"; extra = "KBUILD_MODPOST_WARN=1"; }) + kernelSrc.trees; + + homa-build-matrix = mkKbuild { + pname = "homa-build-matrix"; + nativeBuildInputs = kernelConfig.kernelPackages.kernel.moduleBuildDependencies + ++ (with pkgs; [ elfutils openssl ]); + buildPhase = '' + mkdir -p "$out/logs" + printf '| Kernel | Version | Result |\n|---|---|---|\n' > "$out/summary.md" + fail=0 + ${lib.concatMapStrings (k: '' + rm -rf "../w-${k.name}"; cp -r . "../w-${k.name}"; chmod -R u+w "../w-${k.name}" + if make -C "../w-${k.name}" KDIR=${k.kdir} ${k.extra} \ + -j"$NIX_BUILD_CORES" > "$out/logs/${k.name}.log" 2>&1; then + r=PASS + else + r=FAIL; fail=$((fail + 1)) + fi + echo "| ${k.name} | ${k.version} | $r |" >> "$out/summary.md" + rm -rf "../w-${k.name}" + '') matrixKdirs} + echo "$fail" > "$out/failures" + cat "$out/summary.md" + ''; + dontInstall = true; + dontFixup = true; + }; +in +{ + inherit buildModule buildAgainstTree homa-module; + packages = { + inherit homa-module homa-module-debug homa-build-matrix; + } // srcPkgs // matrixPkgs; +} diff --git a/nix/build/oci.nix b/nix/build/oci.nix new file mode 100644 index 00000000..bdbc84cc --- /dev/null +++ b/nix/build/oci.nix @@ -0,0 +1,52 @@ +# nix/build/oci.nix +# +# A lightweight OCI image bundling Homa's user-space tools + Python tooling (secondary to +# the microVM, which is the real kernel-module test vehicle). Uses dockerTools.buildLayeredImage +# (reproducible, layer-cached) per the xdp2 pattern. +# +# nix build .#homa-image && docker load < result +# nix build .#homa-image-debug && docker load < result +# +# In the scope (nix/scope.nix) as ociPkgs. + +{ pkgs +, lib +, pythonEnv +, homa-utils +, homa-utils-debug +}: + +let + mkImage = { name, utils }: + pkgs.dockerTools.buildLayeredImage { + inherit name; + tag = "latest"; + created = "1970-01-01T00:00:00Z"; # reproducible + + contents = [ + utils + pythonEnv + pkgs.bashInteractive + pkgs.coreutils + ]; + + config = { + Cmd = [ "/bin/bash" ]; + Env = [ "MPLBACKEND=Agg" ]; + Labels = { + "org.opencontainers.image.title" = name; + "org.opencontainers.image.description" = + "Homa user-space tools and Python analysis tooling"; + "org.opencontainers.image.source" = + "https://github.com/PlatformLab/HomaModule"; + "org.opencontainers.image.licenses" = "BSD-2-Clause"; + }; + }; + }; + + homa-image = mkImage { name = "homa-image"; utils = homa-utils; }; + homa-image-debug = mkImage { name = "homa-image-debug"; utils = homa-utils-debug; }; +in +{ + packages = { inherit homa-image homa-image-debug; }; +} diff --git a/nix/build/userspace.nix b/nix/build/userspace.nix new file mode 100644 index 00000000..1b1cd54f --- /dev/null +++ b/nix/build/userspace.nix @@ -0,0 +1,195 @@ +# nix/build/userspace.nix +# +# User-space derivations: +# homa-utils - the util/ benchmark/test binaries (release, mirrors util/Makefile -O3) +# homa-utils-debug - the same, with AddressSanitizer injected via the cc-wrapper +# homa-tests - build + run the test/ kselftest-style unit harness (ASan) +# homa-tests-coverage - the same run built with gcov: line/branch coverage of Homa's sources +# +# The test harness compiles the kernel C files in user space against the pinned kernel's +# headers, so it needs KDIR + ARCH=x86 (test/Makefile hardcodes x86 include paths). This is +# kernel-version sensitive; if it fails against the newest kernel that is recorded in +# nix/STATUS.md and the static-analysis report (see docs/DESIGN.md §5). +# +# nix/scope.nix calls this file with callPackage (and cross.nix from each cross package set). + +{ pkgs +, lib +, homaLib +, unitKdir +}: + +let + inherit (homaLib) version; + + # report = true (cross.nix) runs `make -k` and never fails: $out/report.md says which + # util/ objects and programs built, with each failing file's first error. + buildUtils = { debug ? false, report ? false }: + pkgs.stdenv.mkDerivation { + pname = "homa-utils${lib.optionalString debug "-debug"}${lib.optionalString report "-report"}"; + inherit version; + src = homaLib.src.util; + + nativeBuildInputs = [ pkgs.gnumake ]; + hardeningDisable = [ "all" ]; + + # Inject debug/ASan without editing util/Makefile (the cc-wrapper honours these). + # NIX_CFLAGS_LINK, not NIX_LDFLAGS: these go to gcc when it links, NIX_LDFLAGS to ld, + # which rejects -fsanitize. + env = lib.optionalAttrs debug { + NIX_CFLAGS_COMPILE = "-g -fno-omit-frame-pointer -fsanitize=address"; + NIX_CFLAGS_LINK = "-fsanitize=address"; + }; + + dontConfigure = true; + + # util/Makefile calls g++ by name. When cross-compiling (cross.nix) that must be the + # target compiler, so put a g++ that is $CXX first on PATH. + buildPhase = '' + runHook preBuild + ${lib.optionalString (pkgs.stdenv.hostPlatform != pkgs.stdenv.buildPlatform) '' + mkdir -p .cross-bin + ln -s "$(command -v "$CXX")" .cross-bin/g++ + ln -s "$(command -v "$CC")" .cross-bin/gcc + ln -s "$(command -v "$CC")" .cross-bin/cc + export PATH="$PWD/.cross-bin:$PATH" + ''} + ${if report then '' + mkdir -p "$out" + make -C util -k -j"$NIX_BUILD_CORES" all > "$out/build.log" 2>&1 || true + { + echo "| File | Result | First error |" + echo "|---|---|---|" + for f in util/*.c util/*.cc homa_receiver.cc; do + o="util/$(basename "''${f%.*}").o" + if [ -f "$o" ]; then + echo "| $f | PASS | |" + else + # Rebuild it alone: with -j the errors interleave, and some are in headers. + err=$(make -C util "$(basename "$o")" 2>&1 \ + | grep -m 1 -E '^[^ ]+:[0-9]+:[0-9]+: error:' | sed 's/|/\\|/g' || true) + echo "| $f | FAIL | $err |" + fi + done + echo + bins=$(make -s -C util -p -n all 2>/dev/null | sed -n 's/^BINS := //p') + built=0 + for b in $bins; do [ -x "util/$b" ] && built=$((built + 1)); done + echo "Programs built: $built of $(echo "$bins" | wc -w)" + } > "$out/report.md" + '' else '' + make -C util -j"$NIX_BUILD_CORES" all + ''} + runHook postBuild + ''; + + dontInstall = report; + installPhase = '' + runHook preInstall + mkdir -p "$out/bin" + # Install every executable the Makefile produced under util/. + for f in util/*; do + if [ -f "$f" ] && [ -x "$f" ]; then + install -m755 "$f" "$out/bin/" + fi + done + runHook postInstall + ''; + + meta = homaLib.meta // { + description = "Homa user-space test/benchmark tools (util/)" + + lib.optionalString debug " [ASan]"; + }; + }; + + homa-utils = buildUtils { }; + homa-utils-debug = buildUtils { debug = true; }; + + homa-tests = pkgs.stdenv.mkDerivation { + pname = "homa-tests"; + inherit version; + src = homaLib.src.test; + + # test/linux_objs (picks the vendored rbtree/rhashtable/xxhash for the kernel version) + # is a #!/bin/bash script, which does not exist inside the build sandbox. + postPatch = "patchShebangs test/linux_objs"; + + nativeBuildInputs = [ pkgs.gnumake pkgs.perl pkgs.bash ]; + hardeningDisable = [ "all" ]; + + dontConfigure = true; + + buildPhase = '' + runHook preBuild + # unitKdir merges nixpkgs' split source/ + build/ trees and adapts the kernel + # config to the mock layer (see nix/kernel/unit-kdir/default.nix). + make -C test \ + KDIR=${unitKdir} \ + CC="gcc ${unitKdir.extraCFlags}" \ + -j"$NIX_BUILD_CORES" unit + runHook postBuild + ''; + + # Run the unit tests as the check step (default IPv6 path). + doCheck = true; + checkPhase = '' + runHook preCheck + (cd test && ./unit) + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + mkdir -p "$out/bin" + install -m755 test/unit "$out/bin/homa-unit" + runHook postInstall + ''; + + meta = homaLib.meta // { + description = "Homa kselftest-style unit tests (built + run under AddressSanitizer)"; + }; + }; + + # The same unit tests built with gcov, run once; gcovr then reports line, branch and + # function coverage of Homa's own sources (not the tests or the mock layer): + # unit.log the test run + # lines.txt, summary.txt line and branch coverage per file, plus the total + # coverage.json the totals, for scripts (homa-evidence) + # html/ per-line annotated source + homa-tests-coverage = homa-tests.overrideAttrs (old: { + pname = "homa-tests-coverage"; + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.gcovr ]; + env = { + NIX_CFLAGS_COMPILE = "--coverage"; + NIX_CFLAGS_LINK = "--coverage"; + }; + # Keep the run's output: its last lines count the tests that passed. + checkPhase = '' + runHook preCheck + (cd test && ./unit) 2>&1 | tee unit.log + grep -q '^\[ PASSED \]' unit.log + runHook postCheck + ''; + installPhase = '' + runHook preInstall + mkdir -p "$out/html" + cp unit.log "$out/" + # Homa's sources are at the top of the tree, the harness in test/. + gcov=(--gcov-executable "${lib.getBin pkgs.stdenv.cc.cc}/bin/gcov" --root . + --filter '^(homa_[a-z_]+|timetrace)\.[ch]$' --exclude-unreachable-branches) + gcovr "''${gcov[@]}" --txt "$out/summary.txt" --txt-metric branch \ + --json-summary "$out/coverage.json" --html-details "$out/html/index.html" \ + test + gcovr "''${gcov[@]}" --txt "$out/lines.txt" test + cat "$out/lines.txt" + runHook postInstall + ''; + meta = homaLib.meta // { + description = "Line and branch coverage of Homa's sources by the unit tests"; + }; + }); +in +{ + inherit buildUtils homa-utils homa-utils-debug; + packages = { inherit homa-utils homa-utils-debug homa-tests homa-tests-coverage; }; +} diff --git a/nix/dev/README.md b/nix/dev/README.md new file mode 100644 index 00000000..babe45e3 --- /dev/null +++ b/nix/dev/README.md @@ -0,0 +1,22 @@ +# nix/dev: the development shell + +`nix develop` opens a bash shell with every tool on `PATH` and `$KDIR` pointing at the +default kernel's build tree. Nothing is installed on the host; leaving the shell leaves +no trace. + +| File | In the scope as | What it is | +|---|---|---| +| `packages.nix` | `packagesModule` | the tool lists: build inputs, analysis and debugging tools, `pythonEnv` (python3 with the packages Homa's scripts import), and the Tcl drivers' tools (Tcl 8.6, Expect, tcllib, nagelfar, tclint; Tcl's setup hook sets `TCLLIBPATH`) | +| `env-vars.nix` | `envVars` | shell code that exports `KDIR`, `LINUX_VERSION`, `ARCH`, and puts `pythonEnv` first on `PATH` | +| `shell-functions/` | `shellFunctions` | `homa-build`, `homa-test`, `homa-clean`, `homa-checkpatch`, `homa-lint` | +| `shell.nix` | `devShell` | the `mkShell` that puts them together | + +```sh +nix develop +homa-build # make against $KDIR +homa-test # the unit tests +homa-checkpatch # the kernel's checkpatch.pl +homa-lint # the analysis-quick tier: ruff, shellcheck, clang-format +``` + +`nix fmt` formats Nix files with `nixpkgs-fmt` ([`../parts/dev.nix`](../parts/dev.nix)). diff --git a/nix/dev/env-vars.nix b/nix/dev/env-vars.nix new file mode 100644 index 00000000..67f2c179 --- /dev/null +++ b/nix/dev/env-vars.nix @@ -0,0 +1,26 @@ +# nix/dev/env-vars.nix +# +# A shell-script fragment (returned as a string) that the dev shell splices into its +# shellHook. Sets the environment Homa's Makefiles expect. +# +# In the scope (nix/scope.nix) as envVars; dev/shell.nix puts it in the shellHook. + +{ kernelConfig +, pythonEnv +}: + +'' + # Homa's test/Makefile hardcodes x86 include paths. + export ARCH="''${ARCH:-x86}" + + # Default KDIR to the flake-pinned kernel, but let callers override it (e.g. to build + # against the running host kernel: make KDIR=/lib/modules/$(uname -r)/build). + export KDIR="''${KDIR:-${kernelConfig.kdir}}" + export LINUX_VERSION="''${LINUX_VERSION:-${kernelConfig.modDirVersion}}" + + # Headless matplotlib for the plotting scripts. + export MPLBACKEND="''${MPLBACKEND:-Agg}" + + # Make the pinned python (with matplotlib/numpy) the one on PATH inside the shell. + export PATH="${pythonEnv}/bin:$PATH" +'' diff --git a/nix/dev/packages.nix b/nix/dev/packages.nix new file mode 100644 index 00000000..a5b71502 --- /dev/null +++ b/nix/dev/packages.nix @@ -0,0 +1,52 @@ +# nix/dev/packages.nix +# +# The dev shell's tools, grouped by role: +# pythonEnv python3 + the only third-party deps Homa's scripts use +# allPackages everything, for the dev shell +# +# In the scope (nix/scope.nix) as packagesModule. + +{ pkgs +, lib +, kernelPackages +}: + +let + # Homa's Python tooling imports only matplotlib and numpy beyond the stdlib. + pythonEnv = pkgs.python3.withPackages (p: [ p.matplotlib p.numpy ]); + + # Out-of-tree kernel module build dependencies (the key kbuild idiom) + make/perl for the + # test harness dependency generation (test/mergedep.pl). + buildTools = [ + pkgs.gnumake + pkgs.perl + pkgs.bc + ] ++ kernelPackages.kernel.moduleBuildDependencies; + + # C/C++ static-analysis + formatting tools. `smatch` is not in every nixpkgs, so guard it. + cTools = [ + pkgs.clang-tools # clang-tidy, clang-format + pkgs.cppcheck + pkgs.sparse + pkgs.bear # compile_commands.json generation + pkgs.gdb + ] ++ lib.optional (pkgs ? smatch) pkgs.smatch; + + pyTools = [ pkgs.ruff pkgs.mypy pythonEnv ]; + shTools = [ pkgs.shellcheck pkgs.shfmt ]; + docTools = [ pkgs.groff pkgs.ghostscript ]; # man/ PDF generation + + # The VM test drivers (nix/microvms/tcl): Tcl 8.6 with Expect and tcllib, and the two + # linters. Tcl's setup hook puts expect's and tcllib's lib on TCLLIBPATH. + tclTools = [ + pkgs.tcl + pkgs.expect + pkgs.tclPackages.tcllib + pkgs.nagelfar + pkgs.tclint + ]; +in +{ + inherit pythonEnv; + allPackages = buildTools ++ cTools ++ pyTools ++ shTools ++ docTools ++ tclTools; +} diff --git a/nix/dev/shell-functions/default.nix b/nix/dev/shell-functions/default.nix new file mode 100644 index 00000000..3eecc24a --- /dev/null +++ b/nix/dev/shell-functions/default.nix @@ -0,0 +1,41 @@ +# nix/dev/shell-functions/default.nix +# +# Bash helper functions spliced into the dev shell's shellHook (xdp2 pattern). Returns one +# concatenated string of function definitions. +# +# In the scope (nix/scope.nix) as shellFunctions; dev/shell.nix puts it in the shellHook. + +{}: + +'' + # Build homa.ko against $KDIR (env-vars.nix defaults KDIR to the pinned kernel; override + # it to retarget, e.g. KDIR=/lib/modules/$(uname -r)/build homa-build). + homa-build() { + make KDIR="''${KDIR}" "$@" + } + + # Build and run the unit tests (AddressSanitizer) against $KDIR. + homa-test() { + make -C test KDIR="''${KDIR}" LINUX_VERSION="''${LINUX_VERSION}" ARCH="''${ARCH}" "$@" \ + && ./test/unit + } + + homa-clean() { + make clean 2>/dev/null || true + make -C test clean 2>/dev/null || true + make -C util clean 2>/dev/null || true + } + + # Kernel checkpatch (needs a Linux source tree; set LINUX_SRC_DIR). + homa-checkpatch() { + make checkpatch "$@" + } + + # Quick static analysis: the analysis-quick tier (ruff, shellcheck, clang-format in the + # kernel's style), so the shell and the flake report the same findings. + homa-lint() { + local out + out=$(nix build "$(git rev-parse --show-toplevel)#analysis-quick" --no-link --print-out-paths) \ + && cat "$out/summary.txt" && echo "reports: $out" + } +'' diff --git a/nix/dev/shell.nix b/nix/dev/shell.nix new file mode 100644 index 00000000..8e982d87 --- /dev/null +++ b/nix/dev/shell.nix @@ -0,0 +1,41 @@ +# nix/dev/shell.nix +# +# The Homa development shell. Composes the toolchain from packages.nix, the environment +# fragment from env-vars.nix, and the bash helper functions from shell-functions/*.nix. +# +# In the scope (nix/scope.nix) as devShell; nix/parts/dev.nix makes it devShells.default. + +{ pkgs +, packagesModule +, envVars +, shellFunctions # bash function definitions (shell-functions/) +}: + +pkgs.mkShell { + packages = packagesModule.allPackages; + + # Disable hardening in the shell so an in-shell `make` matches the derivations. + hardeningDisable = [ "all" ]; + + shellHook = '' + ${envVars} + ${shellFunctions} + + cat <<'EOF' + Homa dev shell. Kernel module + userspace + analysis toolchain is on PATH. + + homa-build build homa.ko against $KDIR (override KDIR to retarget) + homa-test build and run the unit tests (ASan) + homa-clean make clean + homa-checkpatch run kernel checkpatch (needs a Linux source tree) + homa-lint the analysis-quick tier: ruff, shellcheck, clang-format + + The VM test drivers (nix/microvms/tcl): tclsh with Expect and tcllib, nagelfar, tclint. + nix run .#homa-tcl-test -- -file wait.test their tests, or some of them + nix run .#homa-tcl-lint their lint + nix run .#homa-vm-compare -- old new two matrix runs' logs, phase by phase + + Docs: nix/docs/DESIGN.md Status: nix/STATUS.md + EOF + ''; +} diff --git a/nix/docs/DESIGN.md b/nix/docs/DESIGN.md new file mode 100644 index 00000000..1b9a6263 --- /dev/null +++ b/nix/docs/DESIGN.md @@ -0,0 +1,255 @@ +# Homa Nix Flake — Design + +> Status: **living design document**. This is the durable *why* and *how* behind the +> `nix/` tree. For a quick-start command list see [`../README.md`](../README.md); for +> live build progress see [`../STATUS.md`](../STATUS.md). + +## 1. Goal + +`HomaModule` is an out-of-tree Linux kernel transport module (`homa.ko`) with C++ +user-space tools (`util/`), a kselftest-style unit-test harness (`test/`, built with +AddressSanitizer), and ~23 Python analysis scripts. Historically it ships only hand-written +GNU Makefiles — no Nix, no CI, no linter config. + +This flake adds a **reproducible, modular Nix layer** on top, giving a fork of the repo: + +1. A **dev shell** with the full toolchain (kernel build deps, gcc/g++, Python + matplotlib/numpy, static-analysis tools). +2. **Debug + release derivations** of both the kernel module and the user-space tools. +3. A **microVM** that boots a matching kernel and actually `insmod`s and smoke-tests `homa.ko`. +4. A **tiered static-analysis** framework (Python/shell/format → clang-tidy/cppcheck → sparse/smatch). +5. A **fuzzing** framework focused on Homa's wire-format **(de)serialization** paths. +6. Two generated reports: a **static-analysis report** and a **fuzzing report**. + +Everything is **purely additive** — no existing repo file is modified (except an optional +`.gitignore` line for `result*` symlinks). + +## 2. Reference patterns adapted + +This design does not invent conventions; it adapts three existing, proven Nix setups: + +| Concern | Reference project | What we take | +|---|---|---| +| Modular `nix/` layout, debug/release switch, OCI image | `xdp2` | one-concern-per-file modules wired by explicit `import`; a boolean `enableDebug` on a shared derivation; `dockerTools.buildLayeredImage` | +| microVM that loads an out-of-tree `.ko` | `uds-rdma-proxy` | guest `nixosSystem` + `microvm.nixosModules.microvm`, **co-pinned** `kernelPackages` for guest boot and module build, 9p `/nix/store` share, `insmod $KO` driven over an `expect` console, KASAN debug variant | +| Userspace libFuzzer framework | `rocm-systems` | `-fsanitize=fuzzer,address,undefined`, `hardeningDisable=["all"]`, attrset-of-recipes harnesses, a `run-fuzzers` wrapper, `summary.txt`/`count.txt` output, `fuzz-deep` crash dedup, `fuzz-cov`, a `fuzz-selftest` canary gate | + +## 3. Module map & composition + +Three layers, each with one job: + +1. **`flake.nix` + `nix/parts/`** (flake-parts). `flake.nix` names the inputs (nixpkgs, + flake-parts, microvm.nix) and the systems, and imports one part per area. A part maps + scope values to flake outputs and holds no build logic. flake-parts merges the parts' + `perSystem` results, so `packages` can come from several files, and a name defined twice + is an error rather than a silent `//` override. +2. **`nix/scope.nix`** (`lib.makeScope`). Every Homa value lives in one package scope. + Each file is called with `callPackage`, so it takes only the arguments it names, from + the scope first, then from nixpkgs. Nothing threads `pkgs lib kernelConfig src` through + by hand, and `homa.overrideScope` swaps a piece (another kernel config, say) and rebuilds + whatever depends on it. Files never read flake inputs: they stay callable on their own. +3. **The area files** (`kernel/`, `build/`, `analysis/`, `fuzz/`, `microvms/`, `dev/`). + Each returns `{ packages; checks?; ... }`. What several areas share lives in + `nix/lib` (`homaLib`): the source subsets, `mkKbuild`, `perKernel`. + +``` +flake.nix # inputs, systems, imports nix/parts/*; usage header +nix/ + README.md STATUS.md # quick start and commands; status, findings, follow-ups + docs/DESIGN.md # this file + parts/ # scope.nix (builds the scope, passes it as `homa`), build, analysis, + # fuzz, vms, dev + scope.nix # the package scope + lib/ # homaLib: src.{module,util,test,fuzz,all} (lib.fileset + patches/), + # version, meta, mkKbuild, perKernel + patches/ # upstream-pending Homa fixes (a list; currently empty) + kernel/ # config.nix (default kernel, KDIR, matrix, vmSeries), src.nix + # (trees from tarballs or a local checkout), unit-kdir/ + build/ # module.nix, userspace.nix, cross.nix, oci.nix + dev/ # shell.nix, packages.nix, env-vars.nix, shell-functions/ + analysis/ # default.nix (mkAnalysisTool, tiers), one file per tool, + # kbuild-check.nix (sparse, smatch) + fuzz/ # default.nix, tools/ (run-fuzzers.sh and the larger builds), + # harness/, dict/, corpus-seeds/ + microvms/ + default.nix # every VM target + constants.nix # sizes, timeouts, addresses, netem phases, oopsRegex + ports.nix # every host port from one table, asserted unique + guest/ # mkVm.nix, smoke.nix, pair.nix, metrics.nix, qdisc.nix + host/ # smoke-test.nix, pair-test.nix (bash); lifecycle, pair-lifecycle, + # matrix (Tcl drivers, configured as JSON); tcl.nix + tcl/ # the homa::lc Tcl/Expect package, its drivers and tcltest suite +analysis-reports/ # the reports from the analysis and fuzzing rounds (fork only) +``` + +Choices behind this layout: + +- **Source subsets, not `self`.** A derivation depends on every byte of its source. Each + build gets only what it reads (`homaLib.src.*`, via `lib.fileset`), so editing a README or + a `.nix` file outside a build's subset leaves it cached. The version string leaves out the + git revision for the same reason. +- **One helper per repeated shape.** kbuild derivations (`mkKbuild`), analysis tools + (`mkAnalysisTool`), fuzz runs (one `run-fuzzers.sh`), and per-kernel package sets + (`perKernel`) are each written once. +- **Checked invariants at evaluation time.** `ports.nix` asserts that no two VM tests bind + the same host port; the `tcl-tests` check asserts that the host's oops regex is the + guests'. +- **Tcl/Expect for console driving.** The lifecycle drivers talk to guest shells over TCP + consoles, which is Expect's job; see `nix/microvms/tcl/README.md`. Scripts that run in + the guests stay bash, shellchecked by `writeShellApplication`. + +## 4. Debug vs non-debug model + +A single derivation module is instantiated twice via a boolean, mirroring xdp2's +`enableAsserts`: + +- **Kernel module** (`module.nix`, `enableDebug ? false`): release builds with + `MY_CFLAGS='-O2'`; debug builds with `MY_CFLAGS='-g -O1 -fno-omit-frame-pointer'`. Homa's + Makefile already appends `-g` and has no formal `DEBUG` macro, so at the C level "debug" + means optimization/symbols. **Real memory-safety coverage comes from the microVM's + KASAN/KMEMLEAK debug kernel** (§6), not from a module CFLAG. This is stated honestly in + the module header so nobody over-reads what `homa-module-debug` provides. +- **User-space** (`userspace.nix`): release mirrors `util/Makefile` (`-O3 -Wall -Werror`); + debug mirrors `test/Makefile` (`-fsanitize=address -g`), including the documented + `rhashtable.o -O2 -fno-sanitize=address` special case. + +## 5. Kernel pinning strategy + +`kernel/config.nix` is the single source of truth. nixpkgs is pinned to `nixos-unstable`. The +default `kernelPackages = pkgs.linuxPackages_latest` (7.3), overridable, because Homa is +being ported forward to 7.x. Every derivation builds a subset of the source from +`homaLib.src` (`nix/lib`) rather than `self`, with `nix/patches/` applied: they can carry +an upstream-pending fix that the default kernel needs. It carried +the 7.1 port (fork draft #12) until upstream landed it as `e65a5c0`, and is empty now. A +patch that upstream has absorbed stops applying, which fails the build at patchPhase: that +is the cue to delete it. + +Kernels come from two places: + +- **nixpkgs** (`kernelConfig.matrix`, keyed by series: `6_1 6_6 6_12 6_18 7_3`). Each + series probed with `tryEval`, so EOL removals drop out instead of breaking evaluation. + These get a real `homa-module-` and a bootable `homa-microvm-test-`. +- **Source** (`kernel/src.nix`): series nixpkgs has already dropped (7.0, 7.1, 7.2 from + pinned kernel.org tarballs) and, with `--impure` and `HOMA_LINUX_SRC=/path/to/linux`, a + local checkout (`local`). Each is `olddefconfig` + `modules_prepare` in an O= tree using + nixpkgs' newest kernel config: about 2.5 minutes and 450 MB, versus a full kernel build. + There is no `vmlinux` or `Module.symvers`, so `homa-module-` against these is a + compile check built with `KBUILD_MODPOST_WARN=1`, not a loadable module. + +`homa-build-matrix` builds Homa against every kernel above in one derivation and writes +`summary.md` plus per-kernel logs. It records failures rather than failing, so one broken +kernel never hides the others. + +The unit harness and kernel-mock fuzzers stay on 6.18 (`unitKernelPackages`, +`nix/kernel/unit-kdir`): `test/mock.c` has to support the kernel's headers, and that lags. + +## 6. microVM approach + +For a kernel module, a container is the wrong unit — you need a real kernel. The microVM +(astro/microvm.nix, adapted from `uds-rdma-proxy` and xtcp2) provides one per kernel: + +- `mkVm.nix` builds a guest `nixosSystem` for a given `kernelPackages` and loads a + `homa.ko` built against that same set (avoids vermagic drift). It uses qemu's `pc` + machine: on the default `microvm` machine the guest's 8250 UART never registers as + ttyS0 and the console goes silent after early boot. +- The host `/nix/store` is mounted into the guest over **9p**, so the exact `.ko` store + path is visible; it is exported to the guest as `environment.variables.HOMA_KO`. +- **The test runs inside the guest** (xtcp2 pattern). With `autoTest = true`, `smoke.nix` + runs as a systemd oneshot with `StandardOutput=journal+console`: `insmod` → + `/proc/modules` → loopback echo RPCs (`demo_server1` ← `demo_client1`, 6 messages from + 100 B to 1 MB, through sendmsg and recvmsg) → the same echo between two network + namespaces over veth with GRO on, TSO off (veth only uses GRO then) and + `rps_sock_flow_entries` set, with kprobes requiring calls to `homa_gro_receive` and + `homa_set_softirq_cpu` (the RPS flow-table code that 7.1 changed) → `rmmod` → dmesg gate. It prints + `HOMA_SMOKE__OK` lines, then `HOMA_SMOKE_PASS` or `HOMA_SMOKE_FAIL:`, and + powers the VM off. +- `smoke-test.nix` on the host only boots the runner under `timeout`, captures the console, and + greps the sentinels. There is no expect script, no TCP serial port and no shell prompt to + match, so tests for different kernels can run in parallel. +- **Sanitizer variants** (`sanitizer`, kernels in `guest/debug-kernels.nix`) boot a debug + kernel with a homa.ko built for it, allot more RAM, inflate the timeouts + (`constants.sanitizerSlowdown`), and add a `SANITIZER` phase that fails on any KASAN, + KCSAN, UBSAN or kmemleak report: + - `debug`: KASAN (inline), KFENCE, KMEMLEAK, lockdep (`PROVE_LOCKING`), + `DEBUG_ATOMIC_SLEEP`, UBSAN (bounds, shift), `DEBUG_LIST`. + - `kcsan`: KCSAN, off at boot. When homa.ko loads, a udev rule (`guest/kcsan.nix`) + limits reports to Homa's functions and turns it on, so races elsewhere in the kernel + stay quiet. + +Exposed: `homa-microvm[-|-debug|-kcsan]` (interactive runner, root autologin), +`homa-microvm-test[-|-debug|-kcsan]` (`nix run` verdicts), and +`homa-pair-lifecycle-{debug,kcsan}` (the pair lifecycle on those kernels). + +## 7. Fuzzing — the (de)serialization focus + +Homa's core job is moving packets, so **wire-format serialization/deserialization is the +primary fuzzing focus** — the code most likely to hide bounds, endianness, and emit/ingest +asymmetry bugs. Kernel code is fuzzed **in user space** with libFuzzer by reusing the +existing `test/mock.c` kernel-mock layer (which already provides `mock_skb_alloc` and +userspace substitutes for kernel functions) and the self-contained userspace dissector. + +Wire types live in `homa_wire.h`; the packet-type byte sits at offset 13 in every header +(`enum homa_packet_type`: `DATA=0x10, GRANT=0x11, RESEND=0x12, RPC_UNKNOWN=0x13, +BUSY=0x14, CUTOFFS=0x15, FREEZE=0x16, NEED_ACK=0x17, ACK=0x18`). + +**Focus A — deserialization (parse), one harness per packet type** (`harness/deser/`), so a +crash pinpoints the exact parser: `fuzz_data_pkt`→`homa_data_pkt`, `fuzz_grant_pkt`, +`fuzz_resend_pkt`, `fuzz_ack_pkt`, `fuzz_cutoffs_pkt`, `fuzz_need_ack_pkt`, +`fuzz_rpc_unknown_pkt`; plus `fuzz_dispatch`→`homa_dispatch_pkts` (whole-packet entry) and +`fuzz_gro`→`homa_gro_receive`/`homa_gso_segment` (the highest-risk `skb_transport_header`→ +`homa_data_hdr` / `__skb_pull` bounds casts). + +**Focus B — serialization + roundtrip differential** (`harness/roundtrip/`): +- `fuzz_roundtrip` — parse → re-emit via `homa_xmit_*`/`homa_message_out_init` → assert + parse∘emit invariants (type, RPC id via `homa_local_id`, offsets/lengths survive a + `__be*` roundtrip). Catches byte-order and asymmetry bugs a one-direction parse misses. +- `fuzz_differential` — same bytes into the kernel parse path and the userspace dissector + (`dissect_homa`); divergence on type or key fields = a real discrepancy. + +**Focus C — userspace dissector** (`harness/fuzz_dissector.c`) over `dissect_homa` alone: +the easy first win (no kernel mock) and the oracle for `fuzz_differential`. + +**Static complement:** the `analysis` deep tier runs `sparse` with `-D__CHECK_ENDIAN__` to +statically vet the `__be16/__be32/__be64` handling in the same (de)serialization code. + +**Targets:** `fuzz` (build all), `fuzz-run` (bounded run → `summary.txt`/`count.txt`/ +symbolized `crashes//repro-.txt`, one per sanitizer signature), `fuzz-deep` +(`nix run`: fork mode + dictionary, `FUZZ_DEEP_TIME`/`FORKS`/`MAXLEN`), +`fuzz-cov` (llvm-cov replay, scoped to the wire functions), `fuzz-selftest` (injected- +overflow canary gate). Runner honors `FUZZ_TIME`/`FUZZ_WORKDIR`. + +## 8. Static analysis — tiers + +`analysis/default.nix` exposes `analysis-quick|standard|deep`, each a `runCommand` that +links per-tool outputs and writes a `summary.txt` with finding counts. + +- **quick** (no kernel): `ruff` + `mypy` (shebang-aware discovery — Homa has extensionless + `util/cp_*` Python scripts), `shellcheck` + `shfmt` (shebang-detected shell), `clang-format` check. +- **standard**: + a `bear`-built `compile_commands.json` → `clang-tidy` (bugprone/cert/ + clang-analyzer) + `cppcheck --enable=all --std=c11`. +- **deep**: + `sparse` (endianness on) + `smatch`. +- **netdev** (`analysis/netdev.nix`): the checks netdev runs on a submission, on the + upstream-shaped source (`make net-next`) placed in a kernel tree as `net/homa`: + `checkpatch --strict --codespell`, `kernel-doc -Wall`, and in-tree `W=1` builds with + gcc + sparse, clang, and `ARCH=i386`. The other tiers check the out-of-tree source; this + one checks what John would send. + +Unit-test coverage is `homa-tests-coverage` (`build/userspace.nix`): the unit tests built +with `--coverage`, then `gcovr` line and branch percentages per file, and an HTML report. + +## 8a. Evidence report + +`evidence/default.nix` gathers every result into one report, for reviewers and demos: +`homa-evidence` (no VM: build and cross matrices, unit tests and coverage, `analysis-deep`, +`analysis-netdev`, fuzzing) writes `evidence.md` and `evidence.html`; `homa-evidence-full` +adds `homa-vm-matrix` and, with `HOMA_EVIDENCE_SANITIZERS=1`, the pair lifecycle on the +KASAN and KCSAN kernels. It only gathers: each number comes from a flake output that can be +built by itself. + +## 9. Deliverables & reports + +The two reports are **separate documents** under `analysis-reports/` (on the fork's `nix-flake` +branch, https://github.com/randomizedcoder/HomaModule/tree/nix-flake/analysis-reports): +- `static-analysis-report.md` — per-tool findings, severity counts, which tiers ran vs were + skipped (and why, e.g. kernel API drift). +- `fuzzing-report.md` — organized around the Focus A/B/C areas: per-type parse results, GRO/ + GSO casts, roundtrip/differential findings, endianness observations, coverage, reproducers. diff --git a/nix/evidence/default.nix b/nix/evidence/default.nix new file mode 100644 index 00000000..b5e8e186 --- /dev/null +++ b/nix/evidence/default.nix @@ -0,0 +1,195 @@ +# nix/evidence/default.nix +# +# One report of everything the flake checks, for reviewers and demos: +# +# homa-evidence nix build .#homa-evidence: every check that needs no VM (kernel +# build matrix, cross builds, unit tests and their coverage, static +# analysis including netdev's checks, fuzzing), gathered into +# evidence.md and evidence.html, with each check's own output beside +# them +# homa-evidence-full nix run .#homa-evidence-full -- [outdir]: that report, plus the VM +# matrix (homa-vm-matrix, needs KVM) and, with HOMA_EVIDENCE_SANITIZERS=1, +# the pair lifecycle on the KASAN and KCSAN kernels +# +# The report only gathers results; each section comes from a derivation that the flake +# also exposes on its own, so any number in it can be rebuilt and checked by itself. + +{ pkgs +, lib +, modulePkgs +, crossPkgs +, userspacePkgs +, analysisPkgs +, fuzzPkgs +, microvmPkgs +}: + +let + hasCross = pkgs.stdenv.hostPlatform.system == "x86_64-linux"; + m = modulePkgs.packages; + a = analysisPkgs.packages; + f = fuzzPkgs.packages; + v = microvmPkgs.packages; + coverage = userspacePkgs.packages.homa-tests-coverage; + + # Markdown to a standalone page (light and dark). + toHtml = '' + to_html() { + { + printf '%s\n' '' \ + '' \ + 'Homa evidence' + cmark-gfm -e table "$1" + echo '' + } > "$2" + } + ''; + + homa-evidence = pkgs.runCommand "homa-evidence" + { nativeBuildInputs = with pkgs; [ jq cmark-gfm ]; } '' + mkdir -p "$out" + cov=${coverage}/coverage.json + pct() { jq -r ".$1" "$cov"; } + # kernels : rows of the build matrix that match. + kernels() { grep -cE "^\| $1[^|]* \|[^|]*\| ($2) \|" ${m.homa-build-matrix}/summary.md || true; } + units=$(grep -oE '[0-9]+ / [0-9]+ tests passed' ${coverage}/unit.log | tail -n 1) + # "ruff 1973, shell 4, ..." from a tier's summary.txt. + tools() { awk '/^ +[a-z].*finding/ { sub(":", "", $1); printf "%s%s %s", sep, $1, $2; sep = ", " }' "$1"; } + crashes=$(sed -nE 's/^Total crash inputs: ([0-9]+).*/\1/p' ${f.fuzz-run}/summary.txt) + harnesses=$(grep -cF 'crash input(s)' ${f.fuzz-run}/summary.txt || true) + + { + echo "# Homa: evidence report" + echo + echo "Built by \`nix build .#homa-evidence\` from the flake's pinned inputs. Every section" + echo "is also a flake output of its own (named in its heading), so each result can be" + echo "rebuilt and checked by itself." + echo + echo "## At a glance" + echo + echo "| Area | Result |" + echo "|---|---|" + echo "| homa.ko builds, 7.x kernels (\`main\` targets 7.3+) | $(kernels 7 PASS) of $(kernels 7 'PASS|FAIL') |" + echo "| homa.ko builds, older kernels (API drift expected) | $(kernels 6 PASS) of $(kernels 6 'PASS|FAIL') |" + ${lib.optionalString hasCross '' + echo "| homa.ko builds (architectures) | x86_64, aarch64, riscv64 |" + ''} + echo "| Unit tests | $units |" + echo "| Unit-test coverage of Homa's sources | lines $(pct line_percent) %, branches $(pct branch_percent) %, functions $(pct function_percent) % |" + echo "| Static analysis, raw findings (\`analysis-deep\`) | $(tools ${a.analysis-deep}/summary.txt) |" + echo "| netdev checks on the code as submitted (\`analysis-netdev\`) | $(tools ${a.analysis-netdev}/summary.txt) |" + echo "| Fuzzing (\`fuzz-run\`) | ''${crashes:-?} crash inputs from $harnesses harnesses; $(cut -d: -f1 ${f.fuzz-selftest}/verdict.txt) (the canary finds an injected overflow) |" + echo + echo "## Kernels (\`homa-build-matrix\`)" + echo + cat ${m.homa-build-matrix}/summary.md + ${lib.optionalString hasCross '' + echo + echo "## Architectures (\`homa-cross-matrix\`)" + echo + sed 's/^# /### /; s/^## /#### /' ${crossPkgs.packages.homa-cross-matrix}/summary.md + ''} + echo + echo "## Unit tests and coverage (\`homa-tests-coverage\`)" + echo + echo "Line coverage per file; html/ has every source line annotated." + echo + echo '```' + tail -n 4 ${coverage}/unit.log + echo + cat ${coverage}/lines.txt + echo '```' + echo + echo "## Static analysis (\`analysis-deep\`)" + echo + echo "Raw counts, before triage: most are style (ruff on util/'s Python, clang-tidy's" + echo "readability checks) or sparse's notes on kernel headers. The triaged findings, and" + echo "the upstream fixes they led to, are in analysis-reports/ on the fork." + echo + echo '```' + cat ${a.analysis-deep}/summary.txt + echo '```' + echo + echo "## netdev checks (\`analysis-netdev\`)" + echo + echo "checkpatch --strict, kernel-doc and in-tree W=1 builds (gcc with sparse, clang," + echo "32-bit) of the files \`make net-next\` writes, as a submission would carry them." + echo + echo '```' + cat ${a.analysis-netdev}/summary.txt + echo + cat ${a.analysis-netdev}/gcc-w1.report.txt + echo + cat ${a.analysis-netdev}/kdoc.report.txt + echo '```' + echo + echo "checkpatch's report: analysis-netdev/checkpatch.report.txt." + echo + echo "## Fuzzing (\`fuzz-run\`, \`fuzz-cov\`, \`fuzz-selftest\`)" + echo + echo '```' + cat ${f.fuzz-run}/summary.txt + echo + tail -n 5 ${f.fuzz-cov}/summary.txt + echo + cat ${f.fuzz-selftest}/verdict.txt + echo '```' + } > "$out/evidence.md" + + ${toHtml} + to_html "$out/evidence.md" "$out/evidence.html" + + # Each check's own output, for the details. + ln -s ${coverage}/html "$out/coverage-html" + ln -s ${a.analysis-deep} "$out/analysis-deep" + ln -s ${a.analysis-netdev} "$out/analysis-netdev" + ln -s ${f.fuzz-run} "$out/fuzz-run" + ln -s ${m.homa-build-matrix} "$out/build-matrix" + cat "$out/evidence.md" + ''; + + homa-evidence-full = pkgs.writeShellApplication { + name = "homa-evidence-full"; + runtimeInputs = with pkgs; [ coreutils cmark-gfm ]; + text = '' + out=$(realpath -m "''${1:-homa-evidence-out}") + mkdir -p "$out" + cp -rL --no-preserve=mode ${homa-evidence}/. "$out/" + md="$out/evidence.md" + # vm <command> <logdir>: run one VM test command, append its summary. + vm() { + local status=0 + echo "== $1" + HOMA_VM_LOGDIR="$3" "$2" || status=$? + { + echo + echo "## $1 (\`$(basename "$2")\`)" + echo + if [ -f "$3/summary.md" ]; then cat "$3/summary.md"; fi + echo + echo "Exit status: $status. Logs: $3" + } >> "$md" + } + vm "VM tests: every kernel, pairs across kernels" ${lib.getExe v.homa-vm-matrix} "$out/vm" + if [ "''${HOMA_EVIDENCE_SANITIZERS:-0}" = 1 ]; then + vm "Pair lifecycle on a KASAN/lockdep/UBSAN kernel" \ + ${lib.getExe v.homa-pair-lifecycle-debug} "$out/vm-kasan" + vm "Pair lifecycle on a KCSAN kernel" \ + ${lib.getExe v.homa-pair-lifecycle-kcsan} "$out/vm-kcsan" + fi + ${toHtml} + to_html "$md" "$out/evidence.html" + echo "Report: $out/evidence.html" + ''; + }; +in +{ + packages = { inherit homa-evidence homa-evidence-full; }; +} diff --git a/nix/fuzz/README.md b/nix/fuzz/README.md new file mode 100644 index 00000000..ed9d1d29 --- /dev/null +++ b/nix/fuzz/README.md @@ -0,0 +1,37 @@ +# nix/fuzz: libFuzzer harnesses for Homa's wire format + +The harnesses feed random bytes to the code that decodes Homa packets, built with +libFuzzer, AddressSanitizer and UndefinedBehaviorSanitizer. [`../docs/DESIGN.md`](../docs/DESIGN.md) +§7 explains what each one covers and why. + +| Output | What it is | +|---|---| +| `fuzz` | the standalone harnesses (`fuzz_dissector`, `fuzz_selftest`), which need no kernel, plus `bin/run-fuzzers` | +| `fuzz-run` | a 20 s run of each: `summary.txt`, `count.txt`, `crashes/` | +| `fuzz-deep` | a command: fork mode and the dictionary, for `FUZZ_DEEP_TIME` seconds per harness | +| `fuzz-cov` | llvm-cov coverage of the wire decoder | +| `fuzz-selftest` | a canary: fails unless the harness finds a planted bug (a `nix flake check`) | +| `fuzz-mock` | the kernel-mock harnesses: Homa's real receive path on the unit tests' mock kernel, plus `bin/run-mock-fuzzers` | + +```sh +nix build .#fuzz && FUZZ_TIME=300 ./result/bin/run-fuzzers out/ +nix run .#fuzz-deep -- out/ +nix build .#fuzz-mock && ./result/bin/run-mock-fuzzers 600 out/ +``` + +| Path | What it is | +|---|---| +| `default.nix` | the outputs above, and the helpers they share: `compileHarness`, `mkFuzzDrv`, `launcher` | +| `tools/run-fuzzers.sh` | the one runner every output uses: runs each harness, reproduces each crash, keeps one per sanitizer signature | +| `tools/fuzz-{cov,selftest,mock}.nix` | the builds with more to them than one compile | +| `tools/build-mock-fuzzers.sh`, `tools/gen-mock-seeds.py` | the kernel-mock build and its seed corpus; both run by hand too | +| `harness/` | the harness sources; `harness/mock/` for the kernel-mock ones | +| `corpus-seeds/`, `dict/` | starting inputs and the libFuzzer dictionary | +| `known-findings.txt` | sanitizer reports `fuzz-mock` accepts while their fix is pending | + +## The runner's output + +`run-fuzzers [outdir]` leaves, for each harness, its working corpus and log, and under +`crashes/<harness>/` one `crash-<sig>` input with its symbolized `repro-<sig>.txt` per +distinct crash. `summary.txt` has a line per harness. The environment sets the run: +`FUZZ_TIME` (seconds per harness), `FUZZ_FORKS`, `FUZZ_MAXLEN`, `FUZZ_DICT`. diff --git a/nix/fuzz/corpus-seeds/dissector/seed0 b/nix/fuzz/corpus-seeds/dissector/seed0 new file mode 100644 index 00000000..1ba57cfc Binary files /dev/null and b/nix/fuzz/corpus-seeds/dissector/seed0 differ diff --git a/nix/fuzz/default.nix b/nix/fuzz/default.nix new file mode 100644 index 00000000..2c2c35f8 --- /dev/null +++ b/nix/fuzz/default.nix @@ -0,0 +1,117 @@ +# nix/fuzz/default.nix +# +# Fuzzing composition root (rocm-systems pattern). Focus on Homa's wire-format +# (de)serialization — see docs/DESIGN.md §7. +# +# fuzz standalone wire-decoder + selftest harnesses (kernel-free, always builds), +# with bin/run-fuzzers (FUZZ_TIME=300 ./result/bin/run-fuzzers [outdir]) +# fuzz-run a 20 s run of each of those -> summary.txt / count.txt / crashes +# fuzz-deep nix run .#fuzz-deep -- [outdir]: fork mode + dictionary, for as long as +# FUZZ_DEEP_TIME says +# fuzz-cov llvm-cov of the wire decoder +# fuzz-selftest canary gate (fails if the harness is vacuous) +# fuzz-mock kernel-mock harnesses: real homa_softirq receive path + per-type pinned +# variants on the unit-test mock layer (gate: all build + seeds run clean), +# with bin/run-mock-fuzzers [seconds] [outdir] +# +# fuzz, fuzz-run, fuzz-deep and fuzz-mock's runner all run harnesses through one script, +# tools/run-fuzzers.sh, which also sorts the crashes by sanitizer signature. + +{ pkgs +, lib +, homaLib +, unitKdir +}: + +let + src = homaLib.src.fuzz; + sanFlags = "-std=c++17 -g -O1 -fno-omit-frame-pointer -fsanitize=fuzzer,address,undefined"; + symbolizer = "${lib.getBin pkgs.llvm}/bin/llvm-symbolizer"; + + # homa-fuzz <bindir> <corpus> [outdir]: see tools/run-fuzzers.sh. + runner = pkgs.writeShellApplication { + name = "homa-fuzz"; + runtimeInputs = with pkgs; [ coreutils findutils gnugrep gnused ]; + runtimeEnv.ASAN_SYMBOLIZER_PATH = symbolizer; + text = builtins.readFile ./tools/run-fuzzers.sh; + }; + + # A command for a build's bin/: runs the harnesses installed next to it, seeded from the + # build's corpus/. setup runs first, and may set FUZZ_* and the arguments ("$@" is the + # outdir). + launcher = name: setup: pkgs.writeShellApplication { + inherit name; + text = '' + here=$(dirname "$(readlink -f "$0")") + ${setup} + exec ${lib.getExe runner} "$here" "$here/../corpus" "$@" + ''; + }; + + # The clang++ command for one harness in nix/fuzz/harness. + compileHarness = { name, out ? "out/bin/${name}", flags ? "" }: + "clang++ ${sanFlags} ${flags} -I nix/fuzz/harness nix/fuzz/harness/${name}.cc -o ${out}"; + + # A derivation over the fuzz sources, built with clang (then args' own + # nativeBuildInputs) and no hardening flags (they clash with the sanitizers). + mkFuzzDrv = args: pkgs.stdenv.mkDerivation ({ + version = "0"; + inherit src; + hardeningDisable = [ "all" ]; + dontConfigure = true; + dontFixup = true; + } // args // { + nativeBuildInputs = [ pkgs.clang ] ++ args.nativeBuildInputs or [ ]; + }); + + # callPackageWith: like callPackage, with these added to what a file can name. + call = file: lib.callPackageWith (pkgs // { inherit mkFuzzDrv compileHarness launcher unitKdir; }) + file + { }; + + # The kernel-independent harnesses: the standalone wire-format spec decoder and the + # self-test canary. They need no kernel headers, so this always builds. + standaloneNames = [ "fuzz_selftest" "fuzz_dissector" ]; + fuzz = mkFuzzDrv { + pname = "homa-fuzzers"; + buildPhase = '' + runHook preBuild + mkdir -p out/bin out/corpus + ${lib.concatMapStrings (name: '' + ${compileHarness { inherit name; }} + mkdir -p out/corpus/${name} + cp nix/fuzz/corpus-seeds/${lib.removePrefix "fuzz_" name}/* out/corpus/${name}/ \ + 2>/dev/null || true + '') standaloneNames} + cp ${launcher "run-fuzzers" ""}/bin/run-fuzzers out/bin/ + runHook postBuild + ''; + installPhase = ''cp -r out "$out"''; + }; + + fuzz-run = pkgs.runCommand "homa-fuzz-run" { } '' + FUZZ_TIME=20 ${lib.getExe runner} ${fuzz}/bin ${fuzz}/corpus "$out" + ''; + + # A command, not a build, so that its time limit can be set when it runs. + fuzz-deep = pkgs.writeShellApplication { + name = "fuzz-deep"; + text = '' + # fuzz-deep [outdir]: FUZZ_DEEP_TIME (60 s per harness), FUZZ_DEEP_FORKS (2), + # FUZZ_DEEP_MAXLEN (4096). + export FUZZ_TIME=''${FUZZ_DEEP_TIME:-60} FUZZ_FORKS=''${FUZZ_DEEP_FORKS:-2} \ + FUZZ_MAXLEN=''${FUZZ_DEEP_MAXLEN:-4096} FUZZ_DICT=${src}/nix/fuzz/dict/homa_wire.dict + exec ${lib.getExe runner} ${fuzz}/bin ${fuzz}/corpus "''${1:-fuzz-deep-out}" + ''; + }; + + fuzz-cov = call ./tools/fuzz-cov.nix; + fuzz-selftest = call ./tools/fuzz-selftest.nix; + fuzz-mock = call ./tools/fuzz-mock.nix; +in +{ + packages = { inherit fuzz fuzz-run fuzz-deep fuzz-cov fuzz-selftest fuzz-mock; }; + + # fuzz-selftest is a real gate and kernel-free, so it belongs in `nix flake check`. + checks = { inherit fuzz-selftest; }; +} diff --git a/nix/fuzz/dict/homa_wire.dict b/nix/fuzz/dict/homa_wire.dict new file mode 100644 index 00000000..fb49dd92 --- /dev/null +++ b/nix/fuzz/dict/homa_wire.dict @@ -0,0 +1,17 @@ +# nix/fuzz/dict/homa_wire.dict — libFuzzer dictionary for Homa wire formats. +# The packet-type byte (enum homa_packet_type) lives at offset 11 of every header; feeding +# these values steers the fuzzer toward exercising each per-type parser. +type_data="\x10" +type_grant="\x11" +type_resend="\x12" +type_rpc_unknown="\x13" +type_busy="\x14" +type_cutoffs="\x15" +type_freeze="\x16" +type_need_ack="\x17" +type_ack="\x18" +# doff for a standard 5-word TCP-style header (non-DATA packets expect 5<<4). +doff_five="\x50" +# Common sentinel offsets/lengths that hit boundary conditions. +zero32="\x00\x00\x00\x00" +neg32="\xff\xff\xff\xff" diff --git a/nix/fuzz/harness/fuzz_dissector.cc b/nix/fuzz/harness/fuzz_dissector.cc new file mode 100644 index 00000000..a771eb2f --- /dev/null +++ b/nix/fuzz/harness/fuzz_dissector.cc @@ -0,0 +1,49 @@ +/* nix/fuzz/harness/fuzz_dissector.cc — Focus C (kernel-independent) + * + * A standalone Homa wire-format decoder driven by libFuzzer. It classifies the packet type, + * validates the per-type header length (mirroring homa_plumbing.c's header_lengths[]), and + * for DATA packets walks the trailing segment headers doing the same offset/length + * arithmetic the kernel does — all under ASan/UBSan, so an out-of-bounds read or overflow in + * that arithmetic is caught. This needs no kernel headers or Wireshark, so it always builds, + * and it is the reference oracle used by fuzz_differential. + */ +#include "homa_wire_spec.h" +#include <cstdint> +#include <cstring> + +using namespace homa_spec; + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + Decoded d = classify(data, size); + if (!d.valid_type || !d.long_enough) + return 0; /* the kernel would drop these early; nothing more to decode */ + + /* Read the common header safely (we know size >= header_length(type) >= sizeof(common)). */ + common_hdr common; + memcpy(&common, data, sizeof(common)); + + if (d.type == DATA) { + data_hdr dh; + memcpy(&dh, data, sizeof(dh)); + + /* message_length / segment walk (big-endian on the wire). Convert defensively. */ + auto be32 = [](uint32_t v) -> uint32_t { + return ((v & 0x000000ffu) << 24) | ((v & 0x0000ff00u) << 8) | + ((v & 0x00ff0000u) >> 8) | ((v & 0xff000000u) >> 24); + }; + uint32_t message_length = be32(dh.message_length); + (void)message_length; + + /* Walk segment headers in the tail, bounded strictly by `size`. */ + size_t pos = sizeof(data_hdr) - sizeof(seg_hdr); /* first seg is inside data_hdr */ + while (pos + sizeof(seg_hdr) <= size) { + seg_hdr seg; + memcpy(&seg, data + pos, sizeof(seg)); + uint32_t off = be32(seg.offset); + (void)off; + /* Advance by a seg header + at least one byte to guarantee progress. */ + pos += sizeof(seg_hdr) + 1; + } + } + return 0; +} diff --git a/nix/fuzz/harness/fuzz_guard.h b/nix/fuzz/harness/fuzz_guard.h new file mode 100644 index 00000000..1eab17ab --- /dev/null +++ b/nix/fuzz/harness/fuzz_guard.h @@ -0,0 +1,40 @@ +/* nix/fuzz/harness/fuzz_guard.h + * + * Sanitizer-independent overflow canary (adapted from the rocm-systems fuzzing design). + * A GuardedBuffer<N> places known canary words immediately before and after an N-byte + * buffer; Check() aborts (so libFuzzer records the input) if either canary was smashed. + * Used by fuzz_selftest to prove the harness framework is not vacuous. + */ +#ifndef HOMA_FUZZ_GUARD_H +#define HOMA_FUZZ_GUARD_H + +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> + +namespace homa_fuzz { + +static constexpr uint32_t kCanaryLo = 0xDEADBEEFu; +static constexpr uint32_t kCanaryHi = 0xCAFEBABEu; + +[[noreturn]] inline void Fail(const char *why) { + fprintf(stderr, "canary smashed: %s\n", why); + std::abort(); +} + +template <size_t N> +struct GuardedBuffer { + uint32_t lo = kCanaryLo; + uint8_t data[N] = {0}; + uint32_t hi = kCanaryHi; + + void Check() const { + if (lo != kCanaryLo) Fail("underflow (lo canary)"); + if (hi != kCanaryHi) Fail("overflow (hi canary)"); + } +}; + +} // namespace homa_fuzz + +#endif /* HOMA_FUZZ_GUARD_H */ diff --git a/nix/fuzz/harness/fuzz_selftest.cc b/nix/fuzz/harness/fuzz_selftest.cc new file mode 100644 index 00000000..484e48c2 --- /dev/null +++ b/nix/fuzz/harness/fuzz_selftest.cc @@ -0,0 +1,30 @@ +/* nix/fuzz/harness/fuzz_selftest.cc + * + * A deliberately fixed-size copy loop guarded by fuzz_guard.h. This harness is + * self-contained (no kernel/mock/Wireshark deps) so it ALWAYS builds — it exists to prove + * the libFuzzer + sanitizer toolchain and the canary net actually catch an overflow. + * + * The `nix build .#fuzz-selftest` gate injects a regression (widening the copy past the + * buffer) and asserts this harness aborts. In its shipped form the bound is correct, so a + * normal fuzz run finds nothing. + */ +#include "fuzz_guard.h" +#include <cstddef> +#include <cstdint> + +/* SELFTEST_COPY_LEN is the number of bytes copied into an 8-byte buffer. Shipped value (8) + * is safe; the selftest gate rewrites it larger to confirm the canary fires. */ +#ifndef SELFTEST_COPY_LEN +#define SELFTEST_COPY_LEN 8 +#endif + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + homa_fuzz::GuardedBuffer<8> buf; + size_t n = size < (size_t)SELFTEST_COPY_LEN ? size : (size_t)SELFTEST_COPY_LEN; + /* Intentionally copy up to SELFTEST_COPY_LEN bytes; if the gate widens that past 8 it + * overruns buf.data and smashes buf.hi. */ + for (size_t i = 0; i < n; i++) + buf.data[i] = data[i]; + buf.Check(); + return 0; +} diff --git a/nix/fuzz/harness/homa_wire_spec.h b/nix/fuzz/harness/homa_wire_spec.h new file mode 100644 index 00000000..f668662e --- /dev/null +++ b/nix/fuzz/harness/homa_wire_spec.h @@ -0,0 +1,129 @@ +/* nix/fuzz/harness/homa_wire_spec.h + * + * A dependency-free (no kernel, no Wireshark) mirror of Homa's on-wire header layout, taken + * verbatim from homa_wire.h (non-__STRIP__ variant). It lets the dissector/spec harness and + * the differential reference side decode raw bytes and reason about per-type header lengths + * exactly as homa_plumbing.c's header_lengths[] does. All multi-byte fields are big-endian + * on the wire; this decoder only needs the type byte and struct sizes, so it does not byte-swap. + * + * Kept in lock-step with homa_wire.h by construction (same field order/sizes + a static_assert + * on the type-byte offset). + */ +#ifndef HOMA_WIRE_SPEC_H +#define HOMA_WIRE_SPEC_H + +#include <cstddef> +#include <cstdint> + +namespace homa_spec { + +/* enum homa_packet_type (homa_wire.h) */ +enum PacketType : uint8_t { + DATA = 0x10, + GRANT = 0x11, + RESEND = 0x12, + RPC_UNKNOWN = 0x13, + BUSY = 0x14, + CUTOFFS = 0x15, + FREEZE = 0x16, + NEED_ACK = 0x17, + ACK = 0x18, + MIN_TYPE = 0x10, + MAX_TYPE = 0x18, +}; + +#define HOMA_MAX_PRIORITIES 8 +#define HOMA_MAX_ACKS_PER_PKT 5 + +#pragma pack(push, 1) + +struct common_hdr { + uint16_t sport; + uint16_t dport; + uint32_t sequence; + char ack[3]; + uint8_t type; /* offset 11 */ + uint8_t doff; + uint8_t flags; + uint16_t window; + uint16_t checksum; + uint16_t urgent; + uint64_t sender_id; +}; + +struct ack_t { + uint64_t client_id; + uint16_t server_port; +}; + +struct seg_hdr { uint32_t offset; }; + +struct data_hdr { + common_hdr common; + uint32_t message_length; + uint32_t incoming; + ack_t ack; + uint16_t cutoff_version; + uint8_t retransmit; + char pad[3]; + seg_hdr seg; +}; + +struct grant_hdr { common_hdr common; uint32_t offset; uint8_t priority; }; +struct resend_hdr { common_hdr common; uint32_t offset; uint32_t length; uint8_t priority; }; +struct rpc_unknown_hdr { common_hdr common; }; +struct busy_hdr { common_hdr common; }; +struct cutoffs_hdr { common_hdr common; uint32_t unsched_cutoffs[HOMA_MAX_PRIORITIES]; uint16_t cutoff_version; }; +struct freeze_hdr { common_hdr common; }; +struct need_ack_hdr { common_hdr common; }; +struct ack_hdr { common_hdr common; uint16_t num_acks; ack_t acks[HOMA_MAX_ACKS_PER_PKT]; }; + +#pragma pack(pop) + +static_assert(offsetof(common_hdr, type) == 11, "type byte must be at offset 11"); + +/* Minimum header length required for each type (mirrors header_lengths[] indexed by + * type - DATA in homa_plumbing.c). */ +inline size_t header_length(uint8_t type) { + switch (type) { + case DATA: return sizeof(data_hdr); + case GRANT: return sizeof(grant_hdr); + case RESEND: return sizeof(resend_hdr); + case RPC_UNKNOWN: return sizeof(rpc_unknown_hdr); + case BUSY: return sizeof(busy_hdr); + case CUTOFFS: return sizeof(cutoffs_hdr); + case FREEZE: return sizeof(freeze_hdr); + case NEED_ACK: return sizeof(need_ack_hdr); + case ACK: return sizeof(ack_hdr); + default: return 0; + } +} + +struct Decoded { + bool valid_type; /* type byte is a known Homa packet type */ + uint8_t type; + bool long_enough; /* buffer >= header_length(type) */ + size_t need; /* required header length for this type */ +}; + +/* Classify a raw packet the way the kernel's early parse does: read the type byte, look up + * the required header length, and check the buffer is long enough. */ +inline Decoded classify(const uint8_t *data, size_t size) { + Decoded d{}; + if (size <= offsetof(common_hdr, type)) { + d.valid_type = false; + d.type = 0; + d.long_enough = false; + d.need = 0; + return d; + } + d.type = data[offsetof(common_hdr, type)]; + d.valid_type = (d.type >= MIN_TYPE && d.type <= MAX_TYPE); + d.need = header_length(d.type); + d.long_enough = d.valid_type && size >= d.need; + return d; +} + +} // namespace homa_spec + +#endif /* HOMA_WIRE_SPEC_H */ diff --git a/nix/fuzz/harness/mock/fuzz_compat.c b/nix/fuzz/harness/mock/fuzz_compat.c new file mode 100644 index 00000000..aa6bff55 --- /dev/null +++ b/nix/fuzz/harness/mock/fuzz_compat.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: BSD-2-Clause or GPL-2.0+ +/* + * fuzz_compat.c - link-time glue for the kernel-mock fuzzers. + * + * With -fsanitize=address and CONFIG_GENERIC_ENTRY (but no KASAN), the + * kernel's fortify-string.h renames memcpy/memset/memmove to the KASAN + * software-tag symbols. Forward them to libc, where ASan intercepts them. + */ + +#include <string.h> + +void *__hwasan_memcpy(void *p, const void *q, size_t size) +{ + return memcpy(p, q, size); +} + +void *__hwasan_memset(void *p, int c, size_t size) +{ + return memset(p, c, size); +} + +void *__hwasan_memmove(void *p, const void *q, size_t size) +{ + return memmove(p, q, size); +} diff --git a/nix/fuzz/harness/mock/fuzz_softirq.c b/nix/fuzz/harness/mock/fuzz_softirq.c new file mode 100644 index 00000000..90b0f19f --- /dev/null +++ b/nix/fuzz/harness/mock/fuzz_softirq.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: BSD-2-Clause or GPL-2.0+ +/* + * fuzz_softirq.c - libFuzzer harness that feeds packets through Homa's real + * receive path (homa_softirq -> homa_dispatch_pkts -> homa_*_pkt) on top of + * the unit-test mock layer (test/mock.c, test/utils.c). + * + * Each input sets up a small amount of RPC state (mirroring the + * homa_incoming unit-test fixture), then delivers up to MAX_PKTS packets + * whose bytes come from the fuzzer. Afterwards everything is torn down and + * the mock layer's leak checks (skbs, kmallocs, pages, locks, ...) are run; + * any FAIL() from those checks, or from a sanitizer, aborts the run. + * + * Input format: + * byte 0 scenario (see setup_scenario) + * byte 1 flags: bit 0 = aim sport/dport/sender_id at the scenario's + * RPC; bit 1 = IPv4 instead of IPv6; bit 2 = with scenario 6, + * aim at the server RPC instead of the client RPC + * then repeated [u16 little-endian length][length bytes] packets + * + * Build with -DPIN_TYPE=<opcode> to force every packet's type byte to a + * single opcode, so that the fuzzer spends its time inside one handler. + */ + +#define KSELFTEST_NOT_MAIN 1 +#include "homa_impl.h" +#include "homa_grant.h" +#include "homa_qdisc.h" +#include "homa_rpc.h" +#include "homa_sock.h" +#include "homa_wire.h" +#include "kselftest_harness.h" +#include "ccutils.h" +#include "mock.h" +#include "utils.h" + +#define MAX_PKTS 8 + +/* test/mock.c reports leaks and lock imbalances with FAIL(), which + * clears __current_test->passed. main.c normally owns this storage. + */ +static struct __test_metadata fuzz_meta = {.name = "fuzz"}; +struct __test_metadata *__current_test = &fuzz_meta; +struct __test_metadata *__test_list; +unsigned int __test_count; +unsigned int __fixture_count; +int __constructor_order; + +struct fuzz_env { + struct in6_addr client_ip; + struct in6_addr server_ip; + struct homa homa; + struct homa_net *hnet; + struct homa_sock hsk; + struct homa_sock hsk2; + + /* Packets are aimed at this RPC when flags bit 0 is set. */ + struct in6_addr *saddr; + struct in6_addr *daddr; + u16 sport; + u16 dport; + u64 sender_id; +}; + +enum { + CLIENT_PORT = 40000, + SERVER_PORT = 99, + CLIENT_ID = 1234, + SERVER_ID = 1235, +}; + +static void aim_at_client(struct fuzz_env *env) +{ + env->saddr = &env->server_ip; + env->daddr = &env->client_ip; + env->sport = SERVER_PORT; + env->dport = env->hsk.port; + env->sender_id = SERVER_ID; +} + +static void aim_at_server(struct fuzz_env *env) +{ + env->saddr = &env->client_ip; + env->daddr = &env->server_ip; + env->sport = CLIENT_PORT; + env->dport = SERVER_PORT; + env->sender_id = CLIENT_ID; +} + +static void setup_scenario(struct fuzz_env *env, u8 scenario, u8 flags) +{ + aim_at_server(env); + switch (scenario % 8) { + case 0: + /* No RPCs at all: exercises the unknown-RPC paths. */ + break; + case 1: + unit_client_rpc(&env->hsk, UNIT_OUTGOING, &env->client_ip, + &env->server_ip, SERVER_PORT, CLIENT_ID, 2000, + 10000); + aim_at_client(env); + break; + case 2: + unit_client_rpc(&env->hsk, UNIT_RCVD_ONE_PKT, &env->client_ip, + &env->server_ip, SERVER_PORT, CLIENT_ID, 2000, + 10000); + aim_at_client(env); + break; + case 3: + unit_server_rpc(&env->hsk2, UNIT_RCVD_ONE_PKT, &env->client_ip, + &env->server_ip, CLIENT_PORT, SERVER_ID, 10000, + 100); + break; + case 4: + unit_server_rpc(&env->hsk2, UNIT_IN_SERVICE, &env->client_ip, + &env->server_ip, CLIENT_PORT, SERVER_ID, 10000, + 100); + break; + case 5: + unit_server_rpc(&env->hsk2, UNIT_OUTGOING, &env->client_ip, + &env->server_ip, CLIENT_PORT, SERVER_ID, 100, + 20000); + break; + case 6: + unit_client_rpc(&env->hsk, UNIT_OUTGOING, &env->client_ip, + &env->server_ip, SERVER_PORT, CLIENT_ID, 20000, + 1000); + unit_server_rpc(&env->hsk2, UNIT_RCVD_ONE_PKT, &env->client_ip, + &env->server_ip, CLIENT_PORT, SERVER_ID + 2, + 10000, 100); + if (flags & 4) + env->sender_id = CLIENT_ID + 2; + else + aim_at_client(env); + break; + case 7: + unit_client_rpc(&env->hsk, UNIT_RCVD_MSG, &env->client_ip, + &env->server_ip, SERVER_PORT, CLIENT_ID, 2000, + 1000); + aim_at_client(env); + break; + } +} + +static void deliver(struct fuzz_env *env, const u8 *data, size_t len, + u8 flags) +{ + struct homa_common_hdr *h; + struct sk_buff *skb; + + skb = mock_skb_alloc(env->saddr, env->daddr, NULL, len, 0); + h = skb_put(skb, len); + memcpy(h, data, len); + if (len >= sizeof(*h)) { +#ifdef PIN_TYPE + h->type = PIN_TYPE; +#endif + if (flags & 1) { + h->sport = htons(env->sport); + h->dport = htons(env->dport); + h->sender_id = cpu_to_be64(env->sender_id); + } + } + + /* FREEZE is a debugging aid that freezes the timetrace; it isn't + * part of the protocol, so keep the fuzzer out of it. + */ + if (len > offsetof(struct homa_common_hdr, type) && h->type == FREEZE) + h->type = DATA; + homa_softirq(skb); +} + +int LLVMFuzzerTestOneInput(const u8 *data, size_t size) +{ + static struct fuzz_env env; + u8 scenario, flags; + int npkts; + + if (size < 2) + return 0; + scenario = data[0]; + flags = data[1]; + data += 2; + size -= 2; + + memset(&env, 0, sizeof(env)); + fuzz_meta.passed = 1; + mock_ipv6_default = !(flags & 2); + mock_ipv6 = mock_ipv6_default; + env.client_ip = unit_get_in_addr("196.168.0.1"); + env.server_ip = unit_get_in_addr("1.2.3.4"); + homa_init(&env.homa); + env.hnet = mock_hnet(0, &env.homa); + env.homa.num_priorities = 1; + env.homa.poll_cycles = 0; + env.homa.qshared->fifo_fraction = 0; + env.homa.unsched_bytes = 10000; + env.homa.grant->window = 10000; + mock_sock_init(&env.hsk, env.hnet, 0); + mock_sock_init(&env.hsk2, env.hnet, SERVER_PORT); + setup_scenario(&env, scenario, flags); + unit_log_clear(); + + for (npkts = 0; npkts < MAX_PKTS && size >= 2; npkts++) { + size_t len = data[0] | (data[1] << 8); + + data += 2; + size -= 2; + if (len > size) + len = size; + if (len > 1500) + len = 1500; + deliver(&env, data, len, flags); + data += len; + size -= len; + } + + homa_destroy(&env.homa); + unit_teardown(); + if (!fuzz_meta.passed) + abort(); + return 0; +} diff --git a/nix/fuzz/known-findings.txt b/nix/fuzz/known-findings.txt new file mode 100644 index 00000000..abec9452 --- /dev/null +++ b/nix/fuzz/known-findings.txt @@ -0,0 +1,4 @@ +# Sanitizer SUMMARY locations already triaged (see https://github.com/randomizedcoder/HomaModule/blob/nix-flake/analysis-reports/round2/fuzzing-report.md). +# The fuzz-mock smoke gate tolerates these; any other finding fails the build. +# One "file:line" per line, matched against the sanitizer SUMMARY line. +# (none: homa_incoming.c:63, small_msg_bytes[-1], fixed upstream by 1d38026 / PR #121) diff --git a/nix/fuzz/tools/build-mock-fuzzers.sh b/nix/fuzz/tools/build-mock-fuzzers.sh new file mode 100755 index 00000000..2f13d76e --- /dev/null +++ b/nix/fuzz/tools/build-mock-fuzzers.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# nix/fuzz/tools/build-mock-fuzzers.sh +# +# Builds the kernel-mock libFuzzer harnesses (nix/fuzz/harness/mock/*.c) by +# compiling Homa and the unit-test mock layer exactly the way test/Makefile +# does, but with clang + libFuzzer + ASan/UBSan. +# +# Usage: build-mock-fuzzers.sh <src-dir> <kdir> <out-dir> [extra-cflags] +# <kdir> a merged KDIR such as nix/kernel/unit-kdir produces +# [extra-cflags] e.g. unitKdir.extraCFlags +# +# Writes <out-dir>/bin/<harness>, <out-dir>/status.txt and <out-dir>/build.log. +# Exits non-zero if the shared objects or any harness fail to build. + +set -u +SRC=$(cd "$1" && pwd) +KDIR=$2 +OUT=$3 +EXTRA=${4:-} +CC=${CC:-clang} +CXX=${CXX:-clang++} +JOBS=${JOBS:-$(nproc)} + +mkdir -p "$OUT/bin" "$OUT/obj" +OBJ="$OUT/obj" +LOG="$OUT/build.log" +STATUS="$OUT/status.txt" +: > "$LOG" +: > "$STATUS" + +INC="-I$SRC/test -I$SRC \ + -I$KDIR/arch/x86/include -I$KDIR/arch/x86/include/generated \ + -I$KDIR/include -I$KDIR/arch/x86/include/uapi \ + -I$KDIR/arch/x86/include/generated/uapi -I$KDIR/include/uapi \ + -I$KDIR/include/generated/uapi -include $KDIR/include/linux/kconfig.h" +DEFS="-D__KERNEL__ -D__UNIT_TEST__ -DKBUILD_MODNAME='\"homa\"' -DCC_USING_FENTRY" +# alignment is off: list_for_each_entry() on an empty list forms container_of(head) for +# cacheline-aligned types, and mock kmalloc() is plain malloc(). +SAN="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize=alignment -fno-sanitize-recover=undefined" +# clang makes this an error by default; it is the 6.18 recvmsg() signature change. +CFLAGS="-std=gnu11 -fms-extensions -fno-pie -g -O1 -fno-omit-frame-pointer -w \ + -Wno-error=incompatible-function-pointer-types $SAN $DEFS $INC $EXTRA" +CXXFLAGS="-std=c++11 -g -O1 -fno-omit-frame-pointer -w $SAN $DEFS -I$SRC/test -I$SRC" + +HOMA_SRCS="homa_devel homa_interest homa_incoming homa_outgoing homa_peer + homa_pool homa_plumbing homa_rpc homa_sock homa_timer homa_tx_pool + homa_utils timetrace homa_grant homa_hijack homa_metrics homa_offload + homa_qdisc" +LINUX_OBJS=$(bash "$SRC/test/linux_objs" "$KDIR/include/generated/uapi/linux/version.h") + +compile() { # <flags> <src> <obj> + eval "$CC -c $1 \"$2\" -o \"$3\"" >> "$LOG" 2>&1 || + { echo "FAILED $2" >> "$STATUS"; return 1; } +} + +pids="" +shared="" +for m in $HOMA_SRCS; do + compile "$CFLAGS" "$SRC/$m.c" "$OBJ/$m.o" & pids="$pids $!" + shared="$shared $OBJ/$m.o" + while [ "$(jobs -rp | wc -l)" -ge "$JOBS" ]; do wait -n; done +done +for o in $LINUX_OBJS; do + b=${o%.o} + case $b in + # As in test/Makefile: these need -O2 and no ASan. + rhashtable*|xxhash*) f="$CFLAGS -O2 -fno-sanitize=address" ;; + *) f="$CFLAGS" ;; + esac + compile "$f" "$SRC/test/$b.c" "$OBJ/$o" & pids="$pids $!" + shared="$shared $OBJ/$o" +done +for m in mock utils; do + compile "$CFLAGS" "$SRC/test/$m.c" "$OBJ/$m.o" & pids="$pids $!" + shared="$shared $OBJ/$m.o" +done +# Plain C, no kernel headers. +$CC -c -O1 -g "$SRC/nix/fuzz/harness/mock/fuzz_compat.c" -o "$OBJ/fuzz_compat.o" \ + >> "$LOG" 2>&1 || echo "FAILED fuzz_compat.c" >> "$STATUS" +shared="$shared $OBJ/fuzz_compat.o" +eval "$CXX -c $CXXFLAGS \"$SRC/test/ccutils.cc\" -o \"$OBJ/ccutils.o\"" \ + >> "$LOG" 2>&1 || echo "FAILED ccutils.cc" >> "$STATUS" +shared="$shared $OBJ/ccutils.o" + +ok=1 +for p in $pids; do wait "$p" || ok=0; done +grep -q FAILED "$STATUS" && ok=0 +if [ $ok != 1 ]; then + echo "shared objects failed; see build.log" >> "$STATUS" + cat "$STATUS" + grep -m20 -E 'error|fatal' "$LOG" + exit 1 +fi +echo "shared objects OK ($(basename "$KDIR"))" >> "$STATUS" + +# name:source:extra-defines. The PIN_TYPE variants keep the fuzzer inside a +# single packet handler. +HARNESSES=" +fuzz_softirq:fuzz_softirq.c: +fuzz_data_pkt:fuzz_softirq.c:-DPIN_TYPE=DATA +fuzz_grant_pkt:fuzz_softirq.c:-DPIN_TYPE=GRANT +fuzz_resend_pkt:fuzz_softirq.c:-DPIN_TYPE=RESEND +fuzz_rpc_unknown_pkt:fuzz_softirq.c:-DPIN_TYPE=RPC_UNKNOWN +fuzz_busy_pkt:fuzz_softirq.c:-DPIN_TYPE=BUSY +fuzz_cutoffs_pkt:fuzz_softirq.c:-DPIN_TYPE=CUTOFFS +fuzz_need_ack_pkt:fuzz_softirq.c:-DPIN_TYPE=NEED_ACK +fuzz_ack_pkt:fuzz_softirq.c:-DPIN_TYPE=ACK +fuzz_start_msg_pkt:fuzz_softirq.c:-DPIN_TYPE=START_MSG +" +rc=0 +for h in $HARNESSES; do + name=${h%%:*}; rest=${h#*:}; file=${rest%%:*}; defs=${rest#*:} + if compile "$CFLAGS $defs" "$SRC/nix/fuzz/harness/mock/$file" "$OBJ/$name.o" && + $CXX -no-pie -fsanitize=fuzzer,address,undefined $shared "$OBJ/$name.o" \ + -o "$OUT/bin/$name" >> "$LOG" 2>&1; then + echo "BUILT $name" >> "$STATUS" + else + echo "FAILED $name" >> "$STATUS" + rc=1 + fi +done +cat "$STATUS" +exit $rc diff --git a/nix/fuzz/tools/fuzz-cov.nix b/nix/fuzz/tools/fuzz-cov.nix new file mode 100644 index 00000000..e90a3f03 --- /dev/null +++ b/nix/fuzz/tools/fuzz-cov.nix @@ -0,0 +1,40 @@ +# nix/fuzz/tools/fuzz-cov.nix +# +# Source-based coverage of the standalone wire decoder (fuzz_dissector), replayed over its +# corpus, reported with llvm-cov. Gives a concrete coverage number for the wire +# (de)serialization decode logic. Kernel-free. + +{ pkgs, mkFuzzDrv, compileHarness }: + +mkFuzzDrv { + pname = "homa-fuzz-cov"; + nativeBuildInputs = [ pkgs.llvm ]; + + buildPhase = '' + runHook preBuild + mkdir -p "$out" + ${compileHarness { + name = "fuzz_dissector"; + out = "fuzz_dissector_cov"; + flags = "-fprofile-instr-generate -fcoverage-mapping"; + }} + + work="$PWD/work"; mkdir -p "$work" + cp nix/fuzz/corpus-seeds/dissector/* "$work/" 2>/dev/null || true + LLVM_PROFILE_FILE="$PWD/dissector.profraw" \ + ./fuzz_dissector_cov -runs=200000 -max_total_time=20 "$work" > "$out/cov.log" 2>&1 || true + + llvm-profdata merge -sparse dissector.profraw -o dissector.profdata 2>/dev/null || true + { + echo "# fuzz-cov: coverage of the Homa wire decoder (fuzz_dissector)" + llvm-cov report ./fuzz_dissector_cov -instr-profile=dissector.profdata 2>&1 || true + } > "$out/summary.txt" + llvm-cov show ./fuzz_dissector_cov -instr-profile=dissector.profdata \ + > "$out/dissector.show.txt" 2>&1 || true + cp dissector.profdata "$out/" 2>/dev/null || true + tail -n 20 "$out/summary.txt" + runHook postBuild + ''; + + dontInstall = true; +} diff --git a/nix/fuzz/tools/fuzz-mock.nix b/nix/fuzz/tools/fuzz-mock.nix new file mode 100644 index 00000000..ad80e93c --- /dev/null +++ b/nix/fuzz/tools/fuzz-mock.nix @@ -0,0 +1,68 @@ +# nix/fuzz/tools/fuzz-mock.nix +# +# Builds the KERNEL-MOCK libFuzzer harnesses (nix/fuzz/harness/mock/*.c). These feed fuzzer +# bytes through Homa's real receive path (homa_softirq -> homa_dispatch_pkts -> homa_*_pkt) +# on top of the unit-test mock layer (test/mock.c), compiled with clang + libFuzzer + +# ASan/UBSan against the same merged KDIR the unit tests use (nix/kernel/unit-kdir). +# +# The build logic lives in build-mock-fuzzers.sh so it can also be run by hand. This target +# is a gate: it fails unless every harness builds and runs its seed corpus cleanly. +# +# Output: $out/bin/<harness>, $out/corpus/ (generated seeds), $out/status.txt, and +# $out/bin/run-mock-fuzzers [seconds-per-harness] [outdir] for a time-boxed campaign (fork mode, +# one job per core). + +{ pkgs, unitKdir, mkFuzzDrv, launcher }: + +let + runner = launcher "run-mock-fuzzers" '' + FUZZ_TIME=''${1:-60} + FUZZ_FORKS=''${FUZZ_FORKS:-$(nproc)} + export FUZZ_TIME FUZZ_FORKS FUZZ_MAXLEN=4096 + set -- "''${2:-mock-fuzz-out}" + ''; +in +mkFuzzDrv { + pname = "homa-fuzzers-mock"; + nativeBuildInputs = [ pkgs.llvmPackages.compiler-rt pkgs.python3 ]; + + buildPhase = '' + runHook preBuild + CC=clang CXX=clang++ JOBS=$NIX_BUILD_CORES \ + bash nix/fuzz/tools/build-mock-fuzzers.sh . ${unitKdir} out "${unitKdir.extraCFlags}" + python3 nix/fuzz/tools/gen-mock-seeds.py out/corpus + runHook postBuild + ''; + + # Smoke gate: every harness must get through the seeds with no sanitizer report and no + # mock-layer leak, except findings already listed in nix/fuzz/known-findings.txt. + doCheck = true; + checkPhase = '' + runHook preCheck + known=$(grep -v '^#' nix/fuzz/known-findings.txt | grep . || true) + for f in out/bin/fuzz_*; do + name=$(basename "$f") + if "$f" -runs=0 out/corpus > "smoke-$name.log" 2>&1; then + echo "smoke: $name OK" + continue + fi + summary=$(grep -m1 SUMMARY "smoke-$name.log" || true) + if [ -n "$summary" ] && [ -n "$known" ] && echo "$summary" | grep -qF "$known"; then + echo "smoke: $name known finding: $summary" + else + cat "smoke-$name.log" + echo "smoke: $name FAILED" + exit 1 + fi + done + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + mkdir -p "$out" + cp -r out/bin out/corpus out/status.txt out/build.log "$out/" + cp ${runner}/bin/run-mock-fuzzers "$out/bin/" + runHook postInstall + ''; +} diff --git a/nix/fuzz/tools/fuzz-selftest.nix b/nix/fuzz/tools/fuzz-selftest.nix new file mode 100644 index 00000000..5d2ba61c --- /dev/null +++ b/nix/fuzz/tools/fuzz-selftest.nix @@ -0,0 +1,43 @@ +# nix/fuzz/tools/fuzz-selftest.nix +# +# Canary gate (rocm-systems idea): compile fuzz_selftest with the copy bound deliberately +# widened past the 8-byte guarded buffer, feed it an overflowing input, and REQUIRE that it +# aborts (via the fuzz_guard canary or ASan). If it does not, the harness/toolchain is +# vacuous and this build fails. This is the one fuzz target that is a real pass/fail gate. + +{ mkFuzzDrv, compileHarness }: + +mkFuzzDrv { + pname = "homa-fuzz-selftest"; + + buildPhase = '' + runHook preBuild + mkdir -p "$out" + # Inject the regression: widen the copy to 32 bytes into an 8-byte buffer. + ${compileHarness { + name = "fuzz_selftest"; + out = "selftest_broken"; + flags = "-DSELFTEST_COPY_LEN=32"; + }} + head -c 32 /dev/zero | tr '\0' 'A' > overflow.bin + + set +e + ASAN_OPTIONS=abort_on_error=1 ./selftest_broken overflow.bin > "$out/selftest.log" 2>&1 + rc=$? + set -e + + if [ "$rc" -eq 0 ]; then + echo "SELFTEST FAILED: harness did NOT catch the injected overflow" | tee "$out/verdict.txt" >&2 + exit 1 + fi + if grep -qiE 'canary smashed|AddressSanitizer|stack-buffer-overflow|global-buffer-overflow' "$out/selftest.log"; then + echo "SELFTEST PASS: injected overflow detected (rc=$rc)" | tee "$out/verdict.txt" + else + echo "SELFTEST FAILED: process aborted (rc=$rc) but no overflow signature found" | tee "$out/verdict.txt" >&2 + exit 1 + fi + runHook postBuild + ''; + + dontInstall = true; +} diff --git a/nix/fuzz/tools/gen-mock-seeds.py b/nix/fuzz/tools/gen-mock-seeds.py new file mode 100644 index 00000000..fea958e8 --- /dev/null +++ b/nix/fuzz/tools/gen-mock-seeds.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Generate seed inputs for the kernel-mock fuzzers (nix/fuzz/harness/mock). + +Usage: gen-mock-seeds.py <out-dir> + +Each seed is: scenario byte, flags byte, then [u16 LE length][packet] records +(see fuzz_softirq.c). Flags bit 0 is set so the harness aims the packets at +the scenario's RPC; only the type-specific fields need to be plausible. +""" + +import os +import struct +import sys + +DATA, GRANT, RESEND, RPC_UNKNOWN, BUSY, CUTOFFS = 0x10, 0x11, 0x12, 0x13, 0x14, 0x15 +NEED_ACK, ACK, START_MSG = 0x17, 0x18, 0x19 + + +def common(ptype): + # sport, dport, sequence, ack[3], type, doff, flags, window, checksum, + # urgent, sender_id (the harness overwrites ports and sender_id). + return struct.pack(">HHI3sBBBHHHQ", 0, 0, 0, b"\0\0\0", ptype, 0, 0, 0, 0, 0, 0) + + +def homa_ack(client_id=0, port=0): + return struct.pack(">QH", client_id, port) + + +def data(msg_length, offset, payload=1000, retransmit=0): + hdr = (common(DATA) + struct.pack(">II", msg_length, 0) + homa_ack() + + struct.pack(">HHB3s", 0, 0, retransmit, b"\0\0\0") + + struct.pack(">I", offset)) + return hdr + bytes((i & 0xff for i in range(payload))) + + +def grant(offset, priority=0): + return common(GRANT) + struct.pack(">IB", offset, priority) + + +def resend(offset, length, priority=0): + return common(RESEND) + struct.pack(">IIB", offset, length, priority) + + +def start_msg(msg_length): + return common(START_MSG) + struct.pack(">I", msg_length) + + +def ack(acks): + body = b"".join(homa_ack(c, p) for c, p in acks) + body += homa_ack() * (5 - len(acks)) + return common(ACK) + struct.pack(">H", len(acks)) + body + + +def cutoffs(version=1): + vals = [1000000, 12288, 8192, 4096, 2048, 1024, 512, 256] + return common(CUTOFFS) + struct.pack(">8IH", *vals, version) + + +def seed(scenario, *pkts, flags=1): + out = bytes([scenario, flags]) + for p in pkts: + out += struct.pack("<H", len(p)) + p + return out + + +def main(): + out = sys.argv[1] + os.makedirs(out, exist_ok=True) + seeds = { + # Client RPCs (scenarios 1, 2, 7): response packets from the server. + "c_resp_data": seed(1, data(10000, 0), data(10000, 1000)), + "c_resp_start": seed(1, start_msg(3000), data(3000, 0)), + "c_grant": seed(1, grant(5000, 1), grant(20000)), + "c_resend": seed(1, resend(0, 1000), resend(500, 100000)), + "c_busy": seed(2, common(BUSY)), + "c_unknown": seed(1, common(RPC_UNKNOWN)), + "c_cutoffs": seed(1, cutoffs()), + "c_need_ack": seed(7, common(NEED_ACK)), + "c_retransmit": seed(2, data(10000, 1400, retransmit=1)), + # Server RPCs (scenarios 3-5): request packets from the client. + "s_req_data": seed(3, data(10000, 1400), data(10000, 2800)), + "s_resend": seed(5, resend(0, 1400), resend(1400, 1400)), + "s_grant": seed(5, grant(15000)), + "s_ack": seed(4, ack([(1234, 99)])), + "s_need_ack": seed(4, common(NEED_ACK)), + # No RPC: new incoming request and stray packets. + "n_new_req": seed(0, data(5000, 0), data(5000, 1000)), + "n_new_start": seed(0, start_msg(2000)), + "n_stray": seed(0, grant(100), resend(0, 10), common(BUSY)), + # Both a client and a server RPC. + "both": seed(6, data(1000, 0), grant(30000), flags=1), + "both_srv": seed(6, data(10000, 1400), flags=5), + "ipv4": seed(3, data(10000, 1400), flags=3), + } + for name, blob in seeds.items(): + with open(os.path.join(out, name), "wb") as f: + f.write(blob) + + +if __name__ == "__main__": + main() diff --git a/nix/fuzz/tools/run-fuzzers.sh b/nix/fuzz/tools/run-fuzzers.sh new file mode 100644 index 00000000..1b7be717 --- /dev/null +++ b/nix/fuzz/tools/run-fuzzers.sh @@ -0,0 +1,83 @@ +# homa-fuzz <bindir> <corpus> [outdir] +# +# Runs every fuzz_* harness in bindir, one after another, seeded from corpus/<harness>/ +# (or from corpus/ itself when it has no directory for that harness), then reproduces each +# crash it found and keeps one per sanitizer SUMMARY line. Every fuzz target uses this: +# run-fuzzers, run-mock-fuzzers, fuzz-run and fuzz-deep (nix/fuzz/default.nix). +# +# outdir (default $FUZZ_WORKDIR, else a new temporary directory) gets: +# <harness>/ the harness's working corpus, and the crash-* inputs it found +# <harness>.log libFuzzer's output +# crashes/<harness>/ crash-<sig> and repro-<sig>.txt for each distinct crash +# summary.txt per harness: coverage, crash inputs, distinct crashes +# count.txt crash inputs in all; distinct_count.txt: distinct crashes +# +# Environment: +# FUZZ_TIME seconds per harness (60) +# FUZZ_FORKS libFuzzer -fork= jobs, with -ignore_crashes=1 (0: run in-process) +# FUZZ_MAXLEN libFuzzer -max_len= +# FUZZ_DICT a libFuzzer dictionary + +bindir=$1 +corpus=$2 +out=${3:-${FUZZ_WORKDIR:-$(mktemp -d)}} +time=${FUZZ_TIME:-60} +forks=${FUZZ_FORKS:-0} + +args=(-max_total_time="$time") +if [ "$forks" -gt 0 ]; then + args+=(-fork="$forks" -ignore_crashes=1) +fi +if [ -n "${FUZZ_MAXLEN:-}" ]; then + args+=(-max_len="$FUZZ_MAXLEN") +fi +if [ -n "${FUZZ_DICT:-}" ]; then + args+=(-dict="$FUZZ_DICT") +fi +export ASAN_OPTIONS=abort_on_error=0:detect_leaks=0:symbolize=1 +export UBSAN_OPTIONS=print_stacktrace=1 + +mkdir -p "$out/crashes" +echo "=== Homa fuzzers: ${time} s per harness, ${forks} fork(s) ===" >"$out/summary.txt" +total=0 +distinct=0 +for bin in "$bindir"/fuzz_*; do + name=$(basename "$bin") + work=$out/$name + seeds=$corpus/$name + [ -d "$seeds" ] || seeds=$corpus + mkdir -p "$work" + cp -rn "$seeds"/. "$work/" 2>/dev/null || true + "$bin" "${args[@]}" -artifact_prefix="$work/" "$work" >"$out/$name.log" 2>&1 || true + + # One crash kept per signature: the SUMMARY line, without addresses. + declare -A seen=() + crashes=0 + kept=0 + while IFS= read -r crash; do + crashes=$((crashes + 1)) + sig=$("$bin" "$crash" 2>&1 | grep -m 1 -oE 'SUMMARY: .*' | sed 's/0x[0-9a-f]*//g' || true) + sig=${sig:-unknown-$crashes} + hash=$(printf '%s' "$sig" | cksum | cut -d ' ' -f 1) + if [ -z "${seen[$hash]:-}" ]; then + seen[$hash]=1 + kept=$((kept + 1)) + mkdir -p "$out/crashes/$name" + cp "$crash" "$out/crashes/$name/crash-$hash" + "$bin" "$crash" >"$out/crashes/$name/repro-$hash.txt" 2>&1 || true + fi + done < <(find "$work" -maxdepth 1 -name 'crash-*') + unset seen + + cov=$(grep -oE 'cov: [0-9]+' "$out/$name.log" | tail -n 1 || true) + printf ' %-20s %-10s %s crash input(s), %s distinct\n' \ + "$name:" "${cov:-cov: ?}" "$crashes" "$kept" | tee -a "$out/summary.txt" + total=$((total + crashes)) + distinct=$((distinct + kept)) +done +{ + echo + echo "Total crash inputs: $total Distinct crashes: $distinct (in $out)" +} | tee -a "$out/summary.txt" +echo "$total" >"$out/count.txt" +echo "$distinct" >"$out/distinct_count.txt" diff --git a/nix/kernel/README.md b/nix/kernel/README.md new file mode 100644 index 00000000..e8d43a32 --- /dev/null +++ b/nix/kernel/README.md @@ -0,0 +1,20 @@ +# nix/kernel: which kernels Homa is built and tested against + +| File | In the scope as | What it is | +|---|---|---| +| `config.nix` | `kernelConfig` | the default kernel (nixpkgs `linuxPackages_latest`), its `kdir`, the per-series `matrix` of nixpkgs kernels, and `vmMatrix`, the series the microVMs boot (those in `vmSeries`) | +| `src.nix` | `kernelSrc` | kernel trees prepared from kernel.org tarballs (7.0 to 7.2) or a local checkout (`HOMA_LINUX_SRC`, with `--impure`), for compile checks; its `mkTree` also makes the riscv64 and aarch64 trees in [`../build/cross.nix`](../build/cross.nix) | +| `unit-kdir/` | `unitKdir` | one merged tree for the unit tests (`test/`) and the kernel-mock fuzzers, with the compiler flags they need on a nixpkgs kernel | + +To change the default kernel, edit `kernelPackages` in `config.nix`; every module build, +VM and test follows. To add a kernel to the VM tests, add its series to `vmSeries`. The +VM targets (`homa-microvm-test-<series>` and so on) and the port allocation follow from +that list. + +## Why the trees need preparing + +A kernel module builds against a kernel's build tree (`KDIR`): its headers, its `.config` +and the generated files. nixpkgs ships this for the kernels it packages, as +`kernel.dev`. For a kernel nixpkgs does not package, `src.nix` runs +`make olddefconfig modules_prepare` on the source itself. nixpkgs splits a tree into `source/` and `build/`; Homa's +`test/Makefile` expects one tree, which `unit-kdir` provides as a tree of symlinks. diff --git a/nix/kernel/config.nix b/nix/kernel/config.nix new file mode 100644 index 00000000..f6c46d28 --- /dev/null +++ b/nix/kernel/config.nix @@ -0,0 +1,60 @@ +# nix/kernel/config.nix +# +# Single source of truth for kernel selection. +# +# Returns the default kernelPackages (nixpkgs `linux_latest`, overridable), the derived +# KDIR / modDirVersion used by the out-of-tree module build, and a "matrix" of every kernel +# series nixpkgs ships (6.1 ... 7.3), used for per-kernel module builds and microVMs. +# +# In the scope (nix/scope.nix) as kernelConfig: +# kernelConfig.kernelPackages # default (nixpkgs linux_latest, 7.3) +# kernelConfig.kdir # "${kernel.dev}/lib/modules/${modDirVersion}/build" +# kernelConfig.matrix # { "6_1" = <kp>; ...; "7_3" = <kp>; } (only those present) +# kernelConfig.vmMatrix # the subset the module builds on, so microVMs are useful +# kernelConfig.unitKernelPackages # the unit harness / mock fuzzers' kernel (6.18), which +# # kernel/unit-kdir builds its KDIR from + +{ pkgs +, lib + # Override the default kernel by passing another kernelPackages set. + # + # Default = nixpkgs' newest kernel (7.3 at the time of writing): Homa is being ported + # forward to 7.x. Upstream `main` builds here since the 7.1 port landed (e65a5c0). +, kernelPackages ? pkgs.linuxPackages_latest + # The unit harness and kernel-mock fuzzers need a kernel whose headers test/mock.c + # supports; kept separate so it can lag the default. +, unitKernelPackages ? pkgs.linuxPackages_6_18 +}: + +let + # KDIR for a given kernelPackages: the build tree shipped in the kernel's `dev` output. + kdirOf = kp: "${kp.kernel.dev}/lib/modules/${kp.kernel.modDirVersion}/build"; + + # "7.3.1" -> "7_3" + seriesOf = kp: lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor kp.kernel.version); + + # Matrix: every kernel series nixpkgs still ships, keyed by series. Some attrs exist but + # throw on access (EOL kernels), so each is probed with tryEval and only usable ones kept. + # Series nixpkgs has dropped (7.0-7.2) come from kernel.org tarballs: see kernel/src.nix. + candidates = [ "6_1" "6_6" "6_12" "6_18" "latest" ]; + kpFor = name: pkgs."linuxPackages_${name}" or null; + usable = name: + let + kp = kpFor name; + probe = builtins.tryEval (kp != null && builtins.isString kp.kernel.modDirVersion); + in + probe.success && probe.value; + matrix = lib.listToAttrs + (map (n: lib.nameValuePair (seriesOf (kpFor n)) (kpFor n)) (lib.filter usable candidates)); + + # Series that get microVM and pair-test targets: those where homa.ko builds on `main`. + # 6.1/6.6 are below the supported range and 6.18-stable (6.18.40+, 6.18.57+) backported the + # 7.x socket API changes, so they stay in `matrix` (and homa-build-matrix) for visibility. + vmSeries = [ "6_12" "7_3" ]; + vmMatrix = lib.filterAttrs (s: _: lib.elem s vmSeries) matrix; +in +{ + inherit kernelPackages matrix vmMatrix kdirOf seriesOf unitKernelPackages; + inherit (kernelPackages.kernel) modDirVersion; + kdir = kdirOf kernelPackages; +} diff --git a/nix/kernel/src.nix b/nix/kernel/src.nix new file mode 100644 index 00000000..b654a03b --- /dev/null +++ b/nix/kernel/src.nix @@ -0,0 +1,106 @@ +# nix/kernel/src.nix +# +# Kernel build trees prepared from SOURCE, for compile-testing homa.ko against kernels that +# nixpkgs no longer ships (7.0-7.2 are EOL-removed) or has not released (a local checkout of +# net-next / mainline). +# +# Each tree is the kernel source plus `make olddefconfig modules_prepare` in an O= build +# directory, configured from nixpkgs' newest kernel config. That is enough for +# `make -C <tree> M=... modules`, but not a bootable kernel: there is no vmlinux and no +# Module.symvers, so module builds against it need KBUILD_MODPOST_WARN=1. About 2 minutes +# per tree, versus ~30 for a full kernel build. +# +# Sources: +# tarballs pinned kernel.org releases below (hashes checked against sha256sums.asc) +# local HOMA_LINUX_SRC=/path/to/linux, only with --impure; e.g. +# HOMA_LINUX_SRC=~/Downloads/linux nix build --impure .#homa-build-matrix +# +# In the scope (nix/scope.nix) as kernelSrc: +# kernelSrc.trees # { "7_0" = <drv>; "7_1" = <drv>; "7_2" = <drv>; [ local = <drv>; ] } +# "${kernelSrc.trees."7_1"}/build" # KDIR + +{ pkgs +, lib +}: + +let + # Config source for every tree; olddefconfig fills in or drops symbols per version. + configKernel = pkgs.linuxPackages_latest.kernel; + + tarballs = { + "7_0" = { version = "7.0.14"; hash = "sha256-3pmZt4TSKT8A05xi2PkqCKuKVLxOgP/SUKDAnLB6D5g="; }; + "7_1" = { version = "7.1.13"; hash = "sha256-YU2V+v3LXM4rZiDn7cavu2Br/9RlVAVYaBXYRoeEGtc="; }; + "7_2" = { version = "7.2.12"; hash = "sha256-uxhGio3Il4qgDKpkRudT97baP+5Wtrfzixb5QwugLU8="; }; + }; + + # Large source directories that out-of-tree module builds never read. + prune = [ "Documentation" "drivers" "sound" "samples" "fs" "LICENSES" ]; + + # mkTree is also used by cross.nix: configfile is that arch's nixpkgs kernel config, and + # makeFlags carries ARCH= and CROSS_COMPILE= (the cross gcc goes in extraInputs). + mkTree = + { name + , version + , src + , configfile ? configKernel.configfile + , makeFlags ? [ ] + , extraInputs ? [ ] + }: pkgs.stdenv.mkDerivation { + pname = "linux-kdir-${name}"; + inherit version src; + + nativeBuildInputs = with pkgs; [ bc bison flex perl python3 pkg-config ] + ++ configKernel.moduleBuildDependencies ++ extraInputs; + buildInputs = with pkgs; [ elfutils openssl zlib ]; + hardeningDisable = [ "all" ]; + dontConfigure = true; + + # The O= build tree records the absolute source path, so the source must live in $out + # too. Build into $out directly. + buildPhase = '' + runHook preBuild + mkdir -p "$out" + cp -r . "$out/source" + chmod -R u+w "$out/source" + patchShebangs "$out/source/scripts" "$out/source/tools" + cd "$out/source" + mkdir -p "$out/build" + cp ${configfile} "$out/build/.config" + make O="$out/build" ${toString makeFlags} olddefconfig + make O="$out/build" ${toString makeFlags} -j"$NIX_BUILD_CORES" modules_prepare + # Keep enough source for M= builds (headers, scripts, arch, Kbuild/Kconfig files). + rm -rf ${lib.concatMapStringsSep " " (d: "\"$out/source/${d}\"") prune} + runHook postBuild + ''; + dontInstall = true; + dontFixup = true; + + # module.nix's buildAgainstTree passes these on to make. (Not name: a passthru name + # would hide the derivation's own.) + passthru = { inherit makeFlags; }; + }; + + fromTarball = name: { version, hash }: mkTree { + inherit name version; + src = pkgs.fetchurl { + url = "https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-${version}.tar.xz"; + inherit hash; + }; + }; + + localPath = builtins.getEnv "HOMA_LINUX_SRC"; + localTree = mkTree { + name = "local"; + version = "local"; + src = builtins.path { + name = "linux-local-src"; + path = localPath; + filter = p: _: baseNameOf p != ".git"; + }; + }; +in +{ + inherit mkTree; + trees = lib.mapAttrs fromTarball tarballs + // lib.optionalAttrs (localPath != "") { local = localTree; }; +} diff --git a/nix/kernel/unit-kdir/default.nix b/nix/kernel/unit-kdir/default.nix new file mode 100644 index 00000000..3f0dd84d --- /dev/null +++ b/nix/kernel/unit-kdir/default.nix @@ -0,0 +1,54 @@ +# nix/kernel/unit-kdir/default.nix +# +# A KDIR tailored for Homa's user-space unit harness (test/) and the kernel-mock fuzzers. +# +# Homa's test/Makefile hardcodes $(KDIR)/include/... and $(KDIR)/arch/x86/include/..., but +# nixpkgs splits a kernel into source/ (checked-in headers) and build/ (generated). This +# derivation merges them into one symlink tree, and swaps in a patched copy of +# include/generated/autoconf.h so that the distro config matches what test/mock.[ch] +# assumes (see patch-autoconf.sh): no gcc named-address-space per-cpu, no memory +# allocation profiling, and CONFIG_DEBUG_PREEMPT (mock.c defines preempt_count_add()). +# +# It also exposes the extra compiler flags needed against the nixpkgs kernel: +# -fcf-protection=branch the config enables X86_KERNEL_IBT (nocf_check attributes) +# -include homa_unit_compat.h small API renames vs. the vendored test/*_v6_17_8.c sources +# -Wno-error=... diagnostics newer gcc emits on Homa that are not +# harness problems (reported separately by the analysis) +# +# Nothing in the source tree is modified: everything is passed via KDIR= and CC=. +# +# Usage: +# unitKdir (in nix/scope.nix), used by userspace.nix and fuzz/tools/fuzz-mock.nix: +# make -C test KDIR=${unitKdir} CC="gcc ${unitKdir.extraCFlags}" unit + +{ pkgs +, lib +, kernelConfig +}: + +let + unitKp = kernelConfig.unitKernelPackages; + buildK = kernelConfig.kdirOf unitKp; + compatHeader = ./homa_unit_compat.h; + + kdir = pkgs.runCommand "homa-unit-kdir-${unitKp.kernel.modDirVersion}" { } '' + srcK="$(dirname "${buildK}")/source" + mkdir -p "$out/include/generated" "$out/arch/x86/include" + cp -rs "$srcK/include/." "$out/include/" + cp -rs "$srcK/arch/x86/include/." "$out/arch/x86/include/" + cp -rs "${buildK}/include/generated/." "$out/include/generated/" + ln -sfn "${buildK}/arch/x86/include/generated" "$out/arch/x86/include/generated" + chmod -R u+w "$out/include/generated" + rm -f "$out/include/generated/autoconf.h" + ${pkgs.bash}/bin/bash ${./patch-autoconf.sh} \ + "${buildK}/include/generated/autoconf.h" "$out/include/generated/autoconf.h" + ''; +in +kdir // { + extraCFlags = lib.concatStringsSep " " [ + "-fcf-protection=branch" + "-include ${compatHeader}" + "-Wno-error=unused-but-set-variable" + "-Wno-error=incompatible-pointer-types" + ]; +} diff --git a/nix/kernel/unit-kdir/homa_unit_compat.h b/nix/kernel/unit-kdir/homa_unit_compat.h new file mode 100644 index 00000000..14bac43b --- /dev/null +++ b/nix/kernel/unit-kdir/homa_unit_compat.h @@ -0,0 +1,7 @@ +/* Force-included when building the unit harness against a newer kernel than + * the vendored test/ rhashtable sources were taken from. + */ +#ifndef kvmalloc_node_noprof +#define kvmalloc_node_noprof(_size, _flags, _node) \ + kvmalloc_node_align_noprof(_size, 1, _flags, _node) +#endif diff --git a/nix/kernel/unit-kdir/patch-autoconf.sh b/nix/kernel/unit-kdir/patch-autoconf.sh new file mode 100755 index 00000000..534c62f5 --- /dev/null +++ b/nix/kernel/unit-kdir/patch-autoconf.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# patch-autoconf.sh <in autoconf.h> <out autoconf.h> +# Adapts a distro kernel config to what Homa's user-space mock layer (test/mock.[ch]) assumes. +set -eu +awk ' +BEGIN { + split("CONFIG_USE_X86_SEG_SUPPORT CONFIG_CC_HAS_NAMED_AS CONFIG_CC_HAS_NAMED_AS_FIXED_SANITIZERS CONFIG_MEM_ALLOC_PROFILING CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT CONFIG_MEM_ALLOC_PROFILING_DEBUG", a, " ") + for (i in a) off[a[i]] = 1 +} +$1 == "#define" && ($2 in off) { print "/* unit-test KDIR: " $2 " disabled */"; next } +{ print } +END { print "#ifndef CONFIG_DEBUG_PREEMPT\n#define CONFIG_DEBUG_PREEMPT 1 /* unit-test KDIR */\n#endif" } +' "$1" > "$2" diff --git a/nix/lib/README.md b/nix/lib/README.md new file mode 100644 index 00000000..46405f2d --- /dev/null +++ b/nix/lib/README.md @@ -0,0 +1,34 @@ +# nix/lib: homaLib, what every Homa build shares + +`default.nix` is a set of values and functions, not a package. The scope +([`../scope.nix`](../scope.nix)) builds it once as `homaLib`, and any file that names +`homaLib` in its argument list gets it. + +| Name | What it is | +|---|---| +| `src.module` | Homa's top-level C sources, Makefiles and Kconfig: what kbuild reads | +| `src.util`, `src.test` | the same, plus `util/` (and `test/`) | +| `src.fuzz` | plus `util/`, `test/` and `nix/fuzz/` | +| `src.all` | the whole tree, for the linters | +| `version`, `meta` | for every Homa derivation | +| `mkKbuild { kdir, makeFlags, ... }` | a derivation that runs Homa's Makefile against a kernel tree | +| `perKernel prefix f set` | `{ "<prefix>-<series>" = f series value; }` for a set keyed by kernel series | + +## Why the source is cut into subsets + +A Nix build depends on every byte of its source. Built from the whole repository, `homa.ko` +would rebuild whenever a README changed. Each subset is made with `lib.fileset` and holds +only what one kind of build reads, so a change outside it leaves the build cached. Markdown +files are left out of every subset but `all`. For the same reason, `version` does not +contain the git revision. + +Each subset also has the patches in [`../patches`](../patches) applied. A patch is cut down +to the files in the subset first (`filterdiff`), so a fix that also adds a unit test still +applies to `src.module`, which has no `test/`. + +## Nix idioms used here + +- `lib.fileset.toSource { root; fileset; }` copies only the listed files into the store. +- `pkgs.applyPatches` is a derivation whose output is its source with patches applied. +- `mkKbuild`'s `args // { ... }` merges attribute sets: the right side wins, which is how + `makeFlags` and `nativeBuildInputs` are appended to instead of replaced. diff --git a/nix/lib/default.nix b/nix/lib/default.nix new file mode 100644 index 00000000..2a750a72 --- /dev/null +++ b/nix/lib/default.nix @@ -0,0 +1,101 @@ +# nix/lib/default.nix +# +# homaLib: what the module, userspace, analysis and fuzz builds share (see README.md). +# +# version, meta for every Homa derivation. The version leaves out the git revision: +# with it, every commit would rebuild everything, even one that only +# edits a README. +# src.<subset> Homa's source cut down to what one kind of build reads, with the +# patches in ../patches applied: +# module the top-level C sources, Makefiles and Kconfig +# util + util/ +# test + util/ and test/ +# fuzz + util/, test/ and nix/fuzz/ +# all the whole tree (the linters scan everything) +# So editing a README, or anything under nix/, does not rebuild homa.ko. +# mkKbuild a derivation that runs Homa's Makefile (kbuild) against a kernel tree +# perKernel { "<prefix>-<series>" = f series value; } for a set keyed by series + +{ pkgs, lib }: + +let + root = ../..; + + # Top-level files only: Homa's sources sit at the root next to its subdirectories. + topLevel = lib.fileset.unions (map (name: root + "/${name}") (lib.attrNames + (lib.filterAttrs + (name: type: type == "regular" + && (lib.any (ext: lib.hasSuffix ext name) [ ".c" ".h" ".cc" ] + || lib.elem name [ "Makefile" "Makefile.upstream" "Kconfig" ])) + (builtins.readDir root)))); + + # A patch touches Homa's C sources: the top level, test/ and util/. Each subset gets the + # parts of each patch that are inside it (filterdiff), so a fix that also adds a unit test + # still applies to the module's source. + patches = import ../patches; + patchDirs = [ "test" "util" ]; + patchFor = dirs: patch: pkgs.runCommand "${baseNameOf patch}" + { nativeBuildInputs = [ pkgs.patchutils ]; } '' + filterdiff -p1 ${lib.concatMapStringsSep " " (d: "-x '${d}/*'") + (lib.subtractLists dirs patchDirs)} ${patch} > $out + ''; + + subset = name: dirs: fileset: pkgs.applyPatches { + name = "homa-src-${name}"; + src = lib.fileset.toSource { inherit root fileset; }; + patches = map (patchFor dirs) patches; + # A patch with nothing inside this subset (a util/-only fix in the module's source) has + # no hunks left after filterdiff, only its mail header, and patch rejects it: drop it. + prePatch = '' + kept="" + for p in $patches; do + if grep -q '^@@ ' "$p"; then kept="$kept $p"; fi + done + patches=$kept + ''; + }; + # A directory without its Markdown files: no build reads them. + dirNoDocs = d: lib.fileset.difference (root + "/${d}") + (lib.fileset.fileFilter (f: f.hasExt "md") (root + "/${d}")); + withDirs = name: dirs: subset name dirs + (lib.fileset.unions ([ topLevel ] ++ map dirNoDocs dirs)); + + version = "0-unstable"; + src = { + module = withDirs "module" [ ]; + util = withDirs "util" [ "util" ]; + test = withDirs "test" [ "test" "util" ]; + fuzz = withDirs "fuzz" [ "test" "util" "nix/fuzz" ]; + all = subset "all" patchDirs root; + }; +in +{ + inherit version src; + + meta = { + homepage = "https://github.com/PlatformLab/HomaModule"; + license = lib.licenses.bsd2; + platforms = lib.platforms.linux; + }; + + # mkKbuild { pname, kdir, ... }: runs `make` in Homa's module source with KDIR=kdir + # (when given) and MY_CFLAGS=-O2. Any other attribute goes to mkDerivation; makeFlags and + # nativeBuildInputs are appended to, the rest replace these defaults. + mkKbuild = { kdir ? null, makeFlags ? [ ], nativeBuildInputs ? [ ], ... }@args: + pkgs.stdenv.mkDerivation ({ + inherit version; + src = src.module; + dontConfigure = true; + # The cc-wrapper's hardening flags clash with kbuild's own. + hardeningDisable = [ "all" ]; + # Through the environment, not makeFlags: stdenv word-splits makeFlags, which would + # turn a debug build's "-g -O1" into make's own -O option. + env.MY_CFLAGS = "-O2"; + } // removeAttrs args [ "kdir" ] // { + nativeBuildInputs = [ pkgs.gnumake ] ++ nativeBuildInputs; + makeFlags = lib.optional (kdir != null) "KDIR=${kdir}" ++ makeFlags; + }); + + perKernel = prefix: f: lib.mapAttrs' (series: value: + lib.nameValuePair "${prefix}-${series}" (f series value)); +} diff --git a/nix/microvms/PLAN-pairs.md b/nix/microvms/PLAN-pairs.md new file mode 100644 index 00000000..41813104 --- /dev/null +++ b/nix/microvms/PLAN-pairs.md @@ -0,0 +1,102 @@ +# Plan: Homa between pairs of microVMs + +Goal: show that the flake can build Homa for several kernels, boot each kernel, and pass +real Homa traffic between two VMs, including across different kernel versions, with a +single `nix run` command. It doubles as a demo of the flake. + +## Already in place + +| Need | Target | Notes | +|---|---|---| +| Compile against several kernels | `homa-module-<series>`, `homa-build-matrix` | nixpkgs 6.1/6.6/6.12/6.18/latest plus kernel.org 7.0–7.2 trees (compile only) | +| One VM per kernel | `homa-microvm[-<series>]` | interactive, root autologin | +| Automated single-VM test | `homa-microvm-test[-<series>]`, `-debug` | insmod, loopback echo, netns/veth with GRO+RPS, rmmod; PASS/FAIL from the console | + +Gaps: +- No VM-to-VM traffic: the netns/veth test stays inside one kernel. +- No single command that runs every kernel and prints a table. +- VM tests are generated for 6.1/6.6/6.18, where the module doesn't build (below the supported + range; 6.18.57 has the stable backports), so they fail for reasons that aren't news. + +## Design: the uds-rdma-proxy pair pattern + +Two guests built by the existing `mkVm.nix`, each with a second NIC: + +``` + node0 (server, kernel A) node1 (client, kernel B) + eth0 user NAT (store/init) eth0 user NAT + eth1 10.88.0.1/24 ◄── QEMU -netdev dgram ──► eth1 10.88.0.2/24 + udp 127.0.0.1:<port> udp 127.0.0.1:<port>+1000 +``` + +- `-netdev dgram` (UDP between two localhost ports): a private L2 link between the two qemu + processes. No root, tap or bridge needed, so it works on any host with KVM, and in CI. The + first version used `-netdev socket,listen|connect` (a TCP stream); that stalls some packets + for milliseconds (STATUS.md F2) and is kept only as `homa-pair-exp-socket`. +- The port is derived from the pair name so different pairs can run in parallel. +- Hostnames are `node0`/`node1` with `/etc/hosts` entries, because `cp_node` addresses servers + as `node<N>`. +- The server guest's oneshot: insmod, start `demo_server1` and `cp_node server`, print + `HOMA_PAIR_SERVER_READY`, then idle. +- The client guest's oneshot runs phases. Each prints `HOMA_PAIR_<phase>_OK`, and the run ends + with `HOMA_PAIR_PASS` or `HOMA_PAIR_FAIL:<phase>`: + 1. `ping` over eth1, so a broken link isn't reported as a Homa failure; + 2. echo 100 B–1 MB across VMs (`demo_client1 10.88.0.1`), which covers multi-packet messages + and grants on a real NIC (virtio-net); + 3. a short `cp_node client --protocol homa` run (workload `w4`, a few seconds), checking that + RPCs complete and the log shows no `incomplete` warnings; + 4. the same with `--protocol tcp`, giving a baseline to print beside the Homa numbers; + 5. rmmod and a dmesg gate (`BUG|Oops|WARNING.*homa`). +- Orchestrator (`pair-test.nix`, xtcp2/uds-rdma-proxy style): + 1. start node0 and wait for READY; + 2. start node1 and wait for PASS or FAIL; + 3. power off node0, then apply the same dmesg gate to node0's console (catches oopses on the + receive side); + 4. print one verdict, with both console logs kept in `$HOMA_VM_LOGDIR`. + +## Phases + +| # | Deliverable | New targets | Done when | +|---|---|---|---| +| 1 ✅ | Pair harness, same kernel | `homa-pair-test` (default kernel) | **Done:** echo + cp_node Homa/TCP pass on 7.3 ↔ 7.3, about 35 s cached; a client with a dead link fails at `PING`. The interactive pair (`homa-pair`) moves to phase 3 | +| 2 ✅ | Cross-kernel pairs | `homa-pair-test-<A>-<B>` for supported kernels (6.12 ↔ 7.3 both ways) | **Done:** all four pairings pass; each has its own link port, and all four run in parallel in 50 s. `vmSeries` in `kernel/config.nix` limits VM targets to 6.12 and 7.3 | +| 3 ✅ | One command for everything | `homa-vm-matrix` (`matrix.nix`): every single-VM test plus every pair, run in parallel, printing a table (kernel(s), verdict, echo, GRO calls, cp_node rates) | **Done:** 6 tests in 46 s. Its first run found a use-after-free in `homa_route_free` (STATUS.md F8); guests now boot with `init_on_free=1` | +| 4 ✅ | Lossy and slow links | `homa-pair-lifecycle[-<s>]` (`pair-lifecycle.nix`): the pair driven over both guests' consoles as timed lifecycle phases, with `tc netem` on eth1 of both nodes (`delay 1ms`, `loss 0.1%`, `1%`, `delay 1ms loss 1%`, `5%`). Every netem qdisc has `limit 100000`, so no tail drops | **Done:** echo, cp_node Homa and TCP complete under every phase on 6.12 and 7.3; each node's netem drop rate matches the configured loss; Homa recovers by RESEND with no `rpc_timeouts`. Found STATUS.md F10 (delay alone triggers resends) | +| 5 ✅ | Homa qdisc on virtio | `homa-pair-lifecycle` phases 10–12: `homa-qdisc on` (`qdisc.nix`) on eth1 of both nodes, traffic, then `net.homa.link_mbps=1000` so the pacer must defer, traffic, then the default qdisc back | **Done:** reproduced round-4 candidate 5 in VMs. On `main` the qdisc takes the link speed as −1 and echo, Homa and TCP all stall (red). With fork #23 every phase passes and the pacer sends Homa packets (green). STATUS.md F11 | +| 6 (stretch) | Boot 7.0–7.2 | full kernels built from the kernel/src.nix tarballs (`buildLinux`, about 30 min each, cached) | The VM matrix covers every 7.x series, not just compile-only | + +Housekeeping, done with phase 1: +- Add a `vmSupported` list in `kernel/config.nix` (today 6.12 and 7.3), so VM and pair targets are + generated only for kernels where the module builds. +- `homa-build-matrix` still reports all kernels, so the 6.1/6.6/6.18 failures stay visible + there. + +## Files + +- `nix/microvms/constants.nix`: pair addresses, MACs, base port, server and client ready + timeouts. +- `nix/microvms/guest/mkVm.nix`: optional `pair = { role, ip, port }` argument adding eth1 (qemu + `extraArgs`), static IP and hosts; `test` selects which in-guest script runs (smoke, pair + server, pair client). +- `nix/microvms/guest/pair.nix` (new): server and client in-guest scripts, shellcheck-clean, using + the same phase-marker style as `smoke.nix`. +- `nix/microvms/host/pair-test.nix` (new): two-VM orchestrator. +- `nix/microvms/host/matrix.nix` (new): parallel runner and summary table. +- `nix/microvms/default.nix`: wires up the new targets. +- `nix/README.md`, `DESIGN.md`: a demo section. + +## Observations + +- On the original `socket` link, Homa's cp_node P50 RTT was about 5 ms against about 0.3 ms + for TCP. The latency sweep (`homa-pair-exp`) traced it to the link: QEMU's TCP stream + backend delays packets, and Homa recovers through its 5 × 1 ms RESEND timer. Over `dgram` + the two protocols match. See [`../STATUS.md`](../STATUS.md) F2. + +## Caveats + +- The emulated link is slow (a few hundred Mb/s to a few Gb/s), and the guests share host CPUs. cp_node numbers + show that traffic flows, not how Homa performs; the demo output says so. +- Everything stays on the fork's `nix-flake` branch. Nothing goes upstream. The only upstream + benefit is bugs it finds, for example phase 5. +- Resources: two guests of 2 vCPU and 2 GiB each; the full matrix at about 5 pairs needs + about 20 GiB, which this host (24 cores, 125 GiB) handles. diff --git a/nix/microvms/README.md b/nix/microvms/README.md new file mode 100644 index 00000000..3e66925a --- /dev/null +++ b/nix/microvms/README.md @@ -0,0 +1,52 @@ +# nix/microvms: Homa in real kernels, in QEMU + +Each test boots one or two small NixOS guests in QEMU, with the same kernel `homa.ko` was +built against, and loads the module there. The guests share the host's `/nix/store` +read-only, so nothing is copied into a disk image, and they need no root on the host: only +KVM (`/dev/kvm`) and some localhost ports. + +| Target | Guests | What it checks | +|---|---|---| +| `homa-microvm-test[-<series>]` | 1 | the guest runs the smoke test at boot: insmod, echo RPCs over loopback and across two netns with GRO/RPS, rmmod, dmesg | +| `homa-lifecycle[-<series>]` | 1 | the host drives the guest over its consoles, phase by phase with timeouts: boot, smoke, module load/unload cycles, dmesg, poweroff | +| `homa-pair-test[-<s>-<c>]` | 2 | Homa between two VMs over a private link: echo, then cp_node with Homa and with TCP | +| `homa-pair-lifecycle[-<series>]` | 2 | the pair, driven over both nodes' consoles: traffic, netem loss and delay, Homa's qdisc | +| `homa-pair-exp[-socket]` | 2 | a latency sweep over sysctl settings ([`../STATUS.md`](../STATUS.md) F2) | +| `homa-vm-matrix` | all | every per-kernel test and every pair at once, and a summary table | +| `homa-vm-compare old new` | — | two matrix runs' logs, test by test: phase ids, names and verdicts, `HOMA_*` lines, `VERDICT` | +| `homa-tcl-test`, `homa-tcl-lint` | — | the drivers' tests (all, or `-file`/`-match`) and lint ([`tcl/README.md`](tcl/README.md)) | +| `homa-microvm[-<series>]` | 1 | an interactive guest (root, no password) | + +`<series>` is each kernel in `vmSeries` ([`../kernel/config.nix`](../kernel/config.nix)); +without it, the default kernel. Each test prints its log directory (`HOMA_VM_LOGDIR` to +choose one) and ends with a `VERDICT: PASS` or `VERDICT: FAIL` line. + +## Layout + +| Path | Runs | What it is | +|---|---|---| +| `default.nix` | | every target above, one function call each | +| `constants.nix` | | the numbers: guest sizes, timeouts, addresses, netem phases, the oops regex | +| `ports.nix` | | every host port, from one table; evaluation fails if two tests would share one | +| [`guest/`](guest/) | in the VM | the guest's NixOS configuration and the scripts it runs | +| [`host/`](host/) | on the host | the commands that boot the guests and judge the result | +| [`tcl/`](tcl/) | on the host | the Tcl/Expect drivers that the lifecycle tests and the matrix run | +| `PLAN-pairs.md` | | the design notes for the two-VM tests | + +## Two ways to drive a guest + +The simple tests (`homa-microvm-test`, `homa-pair-test`) let the guest drive itself: a +systemd unit runs the test script at boot and prints markers such as `HOMA_SMOKE_PASS` on +the serial console, then powers off. The host waits for QEMU to exit and greps the log. + +The lifecycle tests keep the host in charge. The guest offers a root shell on a virtio +console, exposed as a localhost TCP port, and a Tcl/Expect driver sends one command at a +time and times each phase. That shows where a run failed or hung, and lets a test reach +into both nodes of a pair mid-run, to add netem loss or swap the qdisc. + +## Ports + +Tests that run at once must not share a host port. `ports.nix` hands them out from one +table: two console ports per lifecycle guest, four per pair lifecycle, and one link port +per pair (a `dgram` link also uses that port plus `pair.dgramPortOffset`). TCP and UDP +ports are separate. A clash is an evaluation error that names both owners. diff --git a/nix/microvms/constants.nix b/nix/microvms/constants.nix new file mode 100644 index 00000000..56f8d577 --- /dev/null +++ b/nix/microvms/constants.nix @@ -0,0 +1,105 @@ +# nix/microvms/constants.nix +# +# Pure data shared by the guest builder (guest/) and the host-side drivers (host/). Single +# source of truth for identity, resources, ports and timeouts (uds-rdma-proxy pattern). + +{ lib }: + +{ + hostname = "homa-vm"; + + # Guest resources. + vcpu = 2; + mem = 2048; # MiB, normal build + memDebug = 4096; # KASAN needs headroom + + # Whole-run timeouts (seconds): boot + smoke test + poweroff. The kernels are chosen in + # kernel/config.nix (default and matrix), not here. + bootTimeout = 300; + bootTimeoutDebug = 1200; # KASAN guests boot slowly + # The pair lifecycle on a sanitizer kernel: each phase's timeout, times this. + sanitizerSlowdown = 4; + + # A kernel oops, warning or sanitizer report in dmesg: every dmesg check uses this, in the + # guests (smoke.nix, pair.nix) and on the host (tcl/parse.tcl, whose copy the tcl-tests + # check compares with this one). + oopsRegex = "BUG:|Oops|WARNING:|KASAN|UBSAN|general protection fault|list_add corruption"; + + # demo_client1 sends this many echo RPCs (100 B .. 1 MB); every echo check wants them all: + # smoke.nix, pair.nix and the Tcl drivers. + echoRpcs = 6; + + stateVersion = "24.11"; + + # Lifecycle test (lifecycle.nix, pcp pattern): per-phase timeouts in seconds, and where + # ports.nix starts the guests' console ports. + lifecycle = { + portBase = 24700; + timeouts = { + start = 10; # qemu up and its console ports listening + serial = 30; # first kernel line in serial.log + virtio = 120; # a root shell answering on hvc0 (full boot) + smoke = 180; # homa-smoke (smoke.nix), run over hvc0 + cycles = 90; # insmod / netns add+del / rmmod, several times + command = 30; # any other single command + exit = 60; # poweroff until qemu exits + echo = 60; # demo_client1's six RPCs, also over a lossy link + }; + moduleCycles = 5; + + # Pair lifecycle (pair-lifecycle.nix): four console ports per pair, from here on. + pairConsoleBase = 24800; + + # Homa qdisc phases: net.homa.link_mbps for the paced phase. The qdisc uses it when the + # NIC reports no speed (virtio-net), and at 1 Gb/s a 1 MB echo is several ms of link + # time, so the pacer must defer packets. + qdisc.pacedMbps = 1000; + + # Link impairment phases: tc netem on eth1 of both nodes (so both directions), each + # followed by echo, cp_node Homa and cp_node TCP. loss is a percentage (a string, as tc + # takes it). Every netem qdisc gets this limit (packets): deep enough that the queue + # never overflows, so the only drops are the configured random loss, not tail drops + # that would skew the results. pair-lifecycle.nix checks each node's measured drop + # rate against the configured one. + netem = { + limit = 100000; + phases = [ + { name = "delay1ms"; delay = "1ms"; loss = "0"; } + { name = "loss0.1"; delay = null; loss = "0.1"; } + { name = "loss1"; delay = null; loss = "1"; } + { name = "loss1-delay1ms"; delay = "1ms"; loss = "1"; } + { name = "loss5"; delay = null; loss = "5"; } + ]; + }; + }; + + # Two-VM pair test (pair.nix, pair-test.nix; uds-rdma-proxy pattern). The guests share a + # private L2 link on eth1: a QEMU "dgram" (UDP) netdev between two host ports, so no + # root, tap or bridge is needed ("socket", a TCP stream, is kept for comparison; see + # STATUS.md F2). Hostnames follow cp_node's "node<N>" naming. + pair = { + nodes = { + server = { hostname = "node0"; ip = "10.88.0.1"; mac = "52:54:00:88:00:01"; role = "listen"; }; + client = { hostname = "node1"; ip = "10.88.0.2"; mac = "52:54:00:88:00:02"; role = "connect"; }; + }; + prefixLength = 24; + linkPortBase = 23700; # host 127.0.0.1 ports for pair links start here (ports.nix) + dgramPortOffset = 1000; # "dgram" link: node0 binds the pair's port, node1 this much higher + + # Guest TCP ports for coordination (not Homa traffic). + readyPort = 5998; # server listens once its Homa servers are up + donePort = 5999; # client sends its verdict here when finished + ctlPort = 5997; # sweep mode only: server shell for sysctls/metrics (private link) + cpNodeHomaPort = 4100; # demo_server1 already owns Homa port 4000 + + # cp_node load phase: a few seconds of workload w4 at a modest rate (the socket link + # runs at roughly 1-2 Gb/s; this shows traffic flows, not Homa's performance). + workload = "w4"; + gbps = "0.2"; + loadSeconds = 6; + + serverTimeout = 120; # client waits this long for the server's ready port + clientTimeout = 300; # server waits this long for the client's report + runTimeout = 420; # host: whole pair run + }; +} diff --git a/nix/microvms/default.nix b/nix/microvms/default.nix new file mode 100644 index 00000000..1e5df053 --- /dev/null +++ b/nix/microvms/default.nix @@ -0,0 +1,254 @@ +# nix/microvms/default.nix +# +# microVM assembly (uds-rdma-proxy pattern, one VM per kernel like xtcp2). Instantiates the +# guest (guest/mkVm.nix) and the host-side drivers (host/) and exposes: +# homa-microvm[-<series>] the QEMU runner package (nix run boots it directly) +# homa-microvm-test[-<series>] boot, run smoke.nix in the guest (insmod, loopback echo +# RPCs, rmmod), PASS/FAIL from the console log +# homa-lifecycle[-<series>] host-driven phases over ttyS0 + a virtio console, each +# timed: boot, smoke, module cycles, dmesg, poweroff, exit +# (lifecycle.nix, pcp pattern) +# homa-microvm-debug default kernel with KASAN, KMEMLEAK, lockdep, UBSAN, ... +# homa-microvm-test-debug ... the smoke test on it +# homa-pair-lifecycle-debug ... the pair lifecycle (netem, qdisc) on it +# homa-microvm[-test]-kcsan, the same with KCSAN (data races in Homa's functions) +# homa-pair-lifecycle-kcsan +# homa-pair-test[-<s>-<c>] two VMs (pair-test.nix): Homa between server <s>, client <c> +# homa-pair-lifecycle[-<series>] the pair driven over both consoles, with timed phases and +# tc netem loss/delay phases (pair-lifecycle.nix) +# homa-pair-exp[-socket] latency sweep over the pair link (STATUS.md F2) +# homa-vm-matrix every per-series smoke test, lifecycle, pair lifecycle and +# pair test, in parallel, with a summary table (matrix.nix) +# homa-vm-compare two matrix runs' logs, phase by phase +# homa-tcl-test, homa-tcl-lint the Tcl drivers' tests and lint (host/tcl.nix), which +# checks.tcl-tests also runs +# <series> is every kernel/config.nix vmMatrix entry (6_12, 7_3); the unsuffixed names use the +# default kernel. Tests share nothing on the host, so they can run in parallel. +# +# Called from nix/scope.nix. Host ports come from ports.nix, which fails evaluation if two +# tests would bind the same one. + +{ pkgs +, lib +, homaLib +, microvm +, nixpkgs +, kernelConfig +, buildModule +, homaUtils +}: + +let + buildSystem = pkgs.stdenv.hostPlatform.system; + constants = import ./constants.nix { inherit lib; }; + tclApps = import ./host/tcl.nix { inherit pkgs lib; inherit (constants) oopsRegex; }; + metrics = import ./guest/metrics.nix { inherit pkgs lib; }; + + defaultKp = kernelConfig.kernelPackages; + vmSeries = lib.attrNames kernelConfig.vmMatrix; + + # forKernels prefix f: prefix = f "default" <default kernel>, and prefix-<series> = f + # <series> <its kernel> for every vmMatrix entry. + forKernels = prefix: f: + { ${prefix} = f "default" defaultKp; } // homaLib.perKernel prefix f kernelConfig.vmMatrix; + labelOf = name: if name == "default" then "" else "-${name}"; + + # Every ordered (server, client) pair of VM kernels: same-kernel pairs and both directions + # of each mixed pair (wire compatibility between Homa builds for different kernels). + pairList = lib.concatMap (s: map (c: { server = s; client = c; }) vmSeries) vmSeries; + pairName = p: "${p.server}-${p.client}"; + + # The sanitizer VMs (guest/debug-kernels.nix), on the default kernel: name -> sanitizer. + # Each gets homa-microvm-<name>, homa-microvm-test-<name> and homa-pair-lifecycle-<name>. + debugVms = { debug = "kasan"; kcsan = "kcsan"; }; + debugNames = lib.attrNames debugVms; + + ports = import ./ports.nix { + inherit lib constants vmSeries debugNames; + linkNames = + [ + { name = "pair-test"; link = "dgram"; } + { name = "pair-test-exp"; link = "dgram"; } + { name = "pair-test-exp-socket"; link = "socket"; } + ] + ++ map (n: { name = "pair-lifecycle-${n}"; link = "dgram"; }) ([ "default" ] ++ vmSeries) + ++ map (p: { name = "pair-test-${pairName p}"; link = "dgram"; }) pairList + ++ map (n: { name = "pair-lifecycle-${n}"; link = "dgram"; }) debugNames; + }; + + mkGuest = + { kernelPackages + , sanitizer ? null + , autoTest ? false + , pairNode ? null + , pairPort ? null + , pairExperiments ? null + , pairLink ? "dgram" + , consolePorts ? null + }: import ./guest/mkVm.nix { + inherit pkgs lib microvm nixpkgs buildSystem constants buildModule homaUtils + kernelPackages sanitizer autoTest pairNode pairPort pairExperiments pairLink + consolePorts; + }; + + mkTest = { kernelPackages, label, sanitizer ? null }: import ./host/smoke-test.nix { + inherit pkgs constants label; + withSanitizers = sanitizer != null; + runner = mkGuest { inherit kernelPackages sanitizer; autoTest = true; }; + }; + + # Lifecycle test of the kernel called name ("default" or a series). + mkLifecycle = name: kernelPackages: + let consolePorts = ports.console name; in + import ./host/lifecycle.nix { + inherit constants tclApps consolePorts; + label = labelOf name; + runner = mkGuest { inherit kernelPackages consolePorts; }; + }; + + # Pair lifecycle (PLAN-pairs.md phase 4): both nodes on one kernel, driven over their + # consoles, with netem phases. With a sanitizer, both run that debug kernel, and every + # phase gets constants.sanitizerSlowdown times as long. + mkPairLifecycle = name: kernelPackages: mkPairLifecycleWith { inherit name kernelPackages; }; + mkPairLifecycleWith = { name, kernelPackages, sanitizer ? null }: + let + consoles = ports.pairConsoles name; + node = pairNode: mkGuest { + inherit kernelPackages pairNode sanitizer; + pairPort = ports.link "pair-lifecycle-${name}"; + consolePorts = consoles.${if pairNode == "server" then "node0" else "node1"}; + }; + in + import ./host/pair-lifecycle.nix { + inherit lib constants tclApps metrics; + label = labelOf name; + slowdown = if sanitizer != null then constants.sanitizerSlowdown else 1; + ports = consoles; + serverRunner = node "server"; + clientRunner = node "client"; + }; + + # Two-VM pair test: node0 (server) on serverKp, node1 (client) on clientKp, linked on the + # port of "pair-test<label>". + mkPairTest = { serverKp, clientKp, label, experiments ? null, link ? "dgram" }: + let + port = ports.link "pair-test${label}"; + node = pairNode: kernelPackages: mkGuest { + inherit kernelPackages pairNode; + autoTest = true; + pairPort = port; + pairExperiments = experiments; + pairLink = link; + }; + in + import ./host/pair-test.nix { + inherit pkgs constants label port link; + serverRunner = node "server" serverKp; + clientRunner = node "client" clientKp; + }; + + # Sweep for STATUS.md F2 (Homa's P50 RTT over the VM link): one cp_node run per entry, + # with the sysctls applied on both nodes. + latencyExperiments = + let + homa = name: workload: sysctls: { inherit name workload sysctls; proto = "homa"; gbps = "0.2"; }; + tcp = name: workload: { inherit name workload; proto = "tcp"; gbps = "0.2"; sysctls = { }; }; + in + [ + (homa "w4-default" "w4" { }) + (homa "w4-link1000" "w4" { link_mbps = 1000; }) + (homa "w4-link2000" "w4" { link_mbps = 2000; }) + (homa "w4-resend2" "w4" { resend_ticks = 2; }) + (homa "w4-resend10" "w4" { resend_ticks = 10; }) + (tcp "w4-tcp" "w4") + (homa "w1-default" "w1" { }) + (homa "w1-link1000" "w1" { link_mbps = 1000; }) + (tcp "w1-tcp" "w1") + ]; + + perPair = lib.listToAttrs (map + (p: lib.nameValuePair "homa-pair-test-${pairName p}" (mkPairTest { + serverKp = kernelConfig.vmMatrix.${p.server}; + clientKp = kernelConfig.vmMatrix.${p.client}; + label = "-${pairName p}"; + })) + pairList); + + kernelTests = + forKernels "homa-microvm" (_: kernelPackages: mkGuest { inherit kernelPackages; }) + // forKernels "homa-microvm-test" (name: kernelPackages: mkTest { + inherit kernelPackages; + label = labelOf name; + }) + // forKernels "homa-lifecycle" mkLifecycle + // forKernels "homa-pair-lifecycle" mkPairLifecycle; + + # homa-vm-matrix: every per-kernel smoke, lifecycle and pair lifecycle test and every + # pair, in parallel. + matrix = import ./host/matrix.nix { + inherit lib tclApps; + inherit (constants) echoRpcs; + tests = + lib.concatMap + (s: map + (kind: { + name = "homa-${kind}-${s}"; + kernels = s; + exe = kernelTests."homa-${kind}-${s}"; + }) [ "microvm-test" "lifecycle" "pair-lifecycle" ]) + vmSeries + ++ map + (p: { + name = "homa-pair-test-${pairName p}"; + kernels = "${p.server} → ${p.client}"; + exe = perPair."homa-pair-test-${pairName p}"; + }) + pairList; + }; +in +{ + packages = lib.concatMapAttrs + (name: sanitizer: { + "homa-microvm-${name}" = mkGuest { kernelPackages = defaultKp; inherit sanitizer; }; + "homa-microvm-test-${name}" = mkTest { + kernelPackages = defaultKp; + label = "-${name}"; + inherit sanitizer; + }; + "homa-pair-lifecycle-${name}" = mkPairLifecycleWith { + inherit name sanitizer; + kernelPackages = defaultKp; + }; + }) + debugVms // { + homa-pair-test = mkPairTest { + serverKp = defaultKp; + clientKp = defaultKp; + label = ""; + }; + homa-pair-exp = mkPairTest { + serverKp = defaultKp; + clientKp = defaultKp; + label = "-exp"; + experiments = latencyExperiments; + }; + # The same sweep over QEMU's TCP stream backend, which adds the multi-millisecond + # stalls of STATUS.md F2. + homa-pair-exp-socket = mkPairTest { + serverKp = defaultKp; + clientKp = defaultKp; + label = "-exp-socket"; + experiments = latencyExperiments; + link = "socket"; + }; + homa-vm-matrix = matrix; + # The Tcl drivers' tests and lint, for running by hand (tcl.nix, ../tcl/README.md). + homa-tcl-test = tclApps.test; + homa-tcl-lint = tclApps.lint; + # Two matrix runs' logs, phase by phase: homa-vm-compare old-logdir new-logdir. + homa-vm-compare = tclApps.mkTclApp { name = "homa-vm-compare"; driver = "vm-compare.tcl"; }; + } // kernelTests // perPair; + + # The Tcl drivers' tests and lint (tcl.nix). + checks.tcl-tests = tclApps.tests; +} diff --git a/nix/microvms/guest/README.md b/nix/microvms/guest/README.md new file mode 100644 index 00000000..bfdbdfb6 --- /dev/null +++ b/nix/microvms/guest/README.md @@ -0,0 +1,29 @@ +# nix/microvms/guest: what runs inside the VMs + +| File | What it is | +|---|---| +| `mkVm.nix` | the guest: a NixOS system built with [microvm.nix](https://github.com/microvm-nix/microvm.nix), given a kernel, its `homa.ko`, and a role (smoke test, pair node, lifecycle or interactive) | +| `smoke.nix` | `homa-smoke`: insmod, echo RPCs over loopback and across two netns, rmmod, dmesg | +| `pair.nix` | the pair test's server and client scripts, and the sweep mode of `homa-pair-exp` | +| `metrics.nix` | `homa-node-stats` (Homa's counters, summed over cores, plus NIC and qdisc drops) and the awk that diffs two of its snapshots | +| `qdisc.nix` | `homa-qdisc on/off/show`: Homa's queuing discipline on eth1 | + +The scripts are bash, built with `writeShellApplication`, which runs shellcheck on them at +build time and sets `PATH` to exactly the tools listed in `runtimeInputs`. They run in the +guest, which has no Tcl. + +## Output markers + +A guest script reports through lines on its console that the host greps for: +`HOMA_SMOKE_<PHASE>_OK`, then exactly one of `HOMA_SMOKE_PASS` and +`HOMA_SMOKE_FAIL:<phase>` (pair: `HOMA_PAIR_...` and `HOMA_PAIR_SERVER_...`). The host +side ([`../host`](../host)) and `homa-vm-matrix` parse these lines, so changing one means +changing its parser too. + +## A guest, in Nix terms + +`mkVm.nix` calls `nixpkgs.lib.nixosSystem`, as a NixOS machine's own configuration would, +with the microvm.nix module added. The result's `config.microvm.declaredRunner` is a +script that starts QEMU with that system. The guest gets its kernel from +`boot.kernelPackages`, and `homa.ko` by store path. The host's store is mounted over +9p, so the guest sees the exact files the host built. diff --git a/nix/microvms/guest/debug-kernels.nix b/nix/microvms/guest/debug-kernels.nix new file mode 100644 index 00000000..45620e2e --- /dev/null +++ b/nix/microvms/guest/debug-kernels.nix @@ -0,0 +1,56 @@ +# nix/microvms/guest/debug-kernels.nix +# +# The guest kernels of the sanitizer VMs: a VM kernel rebuilt with debug options on (heavy: +# each is a full kernel build). +# +# kasan memory errors (KASAN, KFENCE, KMEMLEAK), locking (lockdep, sleeping in atomic +# context), undefined behaviour (UBSAN) and list corruption +# kcsan data races (KCSAN). It boots with KCSAN off; once homa.ko is loaded, kcsan.nix +# limits reports to Homa's own functions and turns it on, so races elsewhere in the +# kernel cannot fail a Homa test. +# +# Every report starts with a line that constants.oopsRegex matches ("BUG: KASAN:", +# "BUG: KCSAN:", "WARNING: possible circular locking dependency", "UBSAN:", ...), so the +# tests need no sanitizer-specific checks. +# +# debugKernel "kasan" kernelPackages # a kernelPackages set, as linuxPackagesFor returns + +{ pkgs, lib }: + +let + configs = with lib.kernel; { + kasan = { + KASAN = yes; + KASAN_GENERIC = yes; + KASAN_INLINE = yes; + KFENCE = yes; + DEBUG_KMEMLEAK = yes; + KCOV = yes; + PROVE_LOCKING = yes; + DEBUG_ATOMIC_SLEEP = yes; + UBSAN = yes; + UBSAN_BOUNDS = yes; + UBSAN_SHIFT = yes; + DEBUG_LIST = yes; + }; + kcsan = { + KCSAN = yes; + KCSAN_EARLY_ENABLE = no; + # KCSAN and KFENCE both watch memory accesses; nixpkgs turns KFENCE on. + KFENCE = lib.mkForce no; + }; + }; + + # Rust is unavailable with these sanitizers, so nixpkgs' RUST = yes and the drivers that + # need it would fail the config check ("unused option"). + noRust = with lib.kernel; { + RUST = lib.mkForce unset; + NOVA_CORE = lib.mkForce unset; + DRM_NOVA = lib.mkForce unset; + DRM_PANIC_SCREEN_QR_CODE = lib.mkForce unset; + }; +in +name: kernelPackages: +pkgs.linuxPackagesFor (kernelPackages.kernel.override { + structuredExtraConfig = configs.${name} // noRust; +}) diff --git a/nix/microvms/guest/kcsan.nix b/nix/microvms/guest/kcsan.nix new file mode 100644 index 00000000..0f42188b --- /dev/null +++ b/nix/microvms/guest/kcsan.nix @@ -0,0 +1,33 @@ +# nix/microvms/guest/kcsan.nix +# +# For the KCSAN guest (debug-kernels.nix): a udev rule that runs when homa.ko is loaded. It +# limits KCSAN's reports to Homa's functions (the debugfs "whitelist" of every function +# /proc/kallsyms lists for [homa]), then turns KCSAN on. The kernel boots with KCSAN off, so +# data races in the rest of the kernel never reach the console. A reload adds the new +# addresses; KCSAN keeps the old ones, which match nothing once Homa is gone. +# +# The rule runs asynchronously, a moment after insmod, so the first few RPCs of a run may go +# unwatched. + +{ pkgs }: + +let + arm = pkgs.writeShellApplication { + name = "homa-kcsan-arm"; + runtimeInputs = with pkgs; [ coreutils gawk ]; + text = '' + ctl=/sys/kernel/debug/kcsan + echo whitelist > "$ctl" + n=0 + while read -r f; do + echo "!$f" > "$ctl" + n=$((n + 1)) + done < <(awk '$4 == "[homa]" && $2 ~ /^[tT]$/ { print $3 }' /proc/kallsyms) + echo on > "$ctl" + echo "homa-kcsan: watching $n Homa functions" > /dev/kmsg + ''; + }; +in +'' + ACTION=="add", SUBSYSTEM=="module", KERNEL=="homa", RUN+="${pkgs.lib.getExe arm}" +'' diff --git a/nix/microvms/guest/metrics.nix b/nix/microvms/guest/metrics.nix new file mode 100644 index 00000000..38cf75ef --- /dev/null +++ b/nix/microvms/guest/metrics.nix @@ -0,0 +1,94 @@ +# nix/microvms/guest/metrics.nix +# +# Counters for the pair tests: the homa-node-stats guest command, and the awk helpers that +# turn two of its outputs into "name=delta" lines. Shared by pair.nix (in the guest) and +# pair-lifecycle.nix (on the host, over the consoles). + +{ pkgs, lib }: + +rec { + # Counters reported per run (Homa's summed over cores). Lost or late packets show up as + # RESENDs and resent_packets; wait_block/fast/none show how receivers waited. The rest + # say where a lost packet went: eth1 and virtio-net drops, qdisc drops, softnet backlog + # drops, and TCP retransmits for the baseline runs. + expMetrics = [ + "packets_sent_DATA" + "packets_sent_GRANT" + "packets_sent_RESEND" + "packets_rcvd_RESEND" + "packets_rcvd_DATA" + "packets_rcvd_BUSY" + "packets_sent_BUSY" + "requests_received" + "responses_received" + "resent_packets_used" + "unknown_rpcs" + "malformed_packets" + "data_xmit_errors" + "control_xmit_errors" + "handoffs_thread_waiting" + "gro_data_bypasses" + "linux_softirqd_actions" + "resent_packets" + "resent_discards" + "rpc_timeouts" + "packet_discards" + "gaps_created" + "dropped_data_no_bufs" + "wait_none" + "wait_fast" + "wait_block" + "eth1_rx_packets" + "eth1_tx_packets" + "eth1_rx_dropped" + "eth1_tx_dropped" + "virtio_drops" + "qdisc_dropped" + "softnet_dropped" + "tcp_retrans" + # Homa qdisc (homa-qdisc on): TCP packets it saw, and packets its pacer sent after + # deferring them. + "qdisc_tcp_packets" + "pacer_homa_packets" + "pacer_tcp_packets" + ]; + + # Prints /proc/net/homa_metrics plus "name value" lines for the NIC and stack counters + # above. Both guests share the host's /nix/store, so the client runs it on the server too. + nodeStats = pkgs.writeShellApplication { + name = "homa-node-stats"; + runtimeInputs = with pkgs; [ coreutils gawk iproute2 ethtool ]; + text = '' + cat /proc/net/homa_metrics + for f in rx_packets tx_packets rx_dropped tx_dropped; do + echo "eth1_$f $(cat "/sys/class/net/eth1/statistics/$f")" + done + ethtool -S eth1 | awk '/drops:/ { s += $2 } END { print "virtio_drops", s + 0 }' + tc -s qdisc show dev eth1 \ + | awk '$6 == "(dropped" { sub(",", "", $7); s += $7 } END { print "qdisc_dropped", s + 0 }' + awk '{ s += strtonum("0x" $2) } END { print "softnet_dropped", s + 0 }' /proc/net/softnet_stat + awk '$1 == "Tcp:" { if (!n) { for (i = 1; i <= NF; i++) c[$i] = i; n = 1 } else print "tcp_retrans", $c["RetransSegs"] }' /proc/net/snmp + ''; + }; + + # Bash functions for the guest-side pair test (pair.nix): snap (homa-node-stats on stdin + # -> "name=value ...", summed over cores) and delta <snap> <snap> (the non-zero + # differences). The host-side lifecycle drivers do the same in Tcl (parse snap/delta). + snapDelta = '' + snap() { + awk -v keys="${lib.concatStringsSep " " expMetrics}" ' + BEGIN { n = split(keys, k, " "); for (i = 1; i <= n; i++) want[k[i]] = 1 } + ($1 in want) { sum[$1] += $2 } + END { for (i = 1; i <= n; i++) printf "%s=%d ", k[i], sum[k[i]] + 0 }' + } + delta() { + awk -v a="$1" -v b="$2" 'BEGIN { + n = split(a, x, " "); split(b, y, " ") + for (i = 1; i <= n; i++) { + split(x[i], p, "="); split(y[i], q, "=") + if (q[2] - p[2] != 0) printf "%s=%d ", p[1], q[2] - p[2] + } + }' + } + ''; +} diff --git a/nix/microvms/guest/mkVm.nix b/nix/microvms/guest/mkVm.nix new file mode 100644 index 00000000..8df5ea87 --- /dev/null +++ b/nix/microvms/guest/mkVm.nix @@ -0,0 +1,203 @@ +# nix/microvms/guest/mkVm.nix +# +# Parameterised microVM guest (uds-rdma-proxy pattern). Builds a NixOS guest that imports the +# microvm.nix module, boots the SAME kernelPackages the homa.ko was built against (co-pinning +# avoids "vermagic" drift), mounts the host /nix/store over 9p so the .ko store path is +# visible in the guest, and exports it as $HOMA_KO. Evaluates to the QEMU runner package +# (config.microvm.declaredRunner). +# +# autoTest=true runs the smoke test (smoke.nix) at boot as a systemd oneshot, with its output +# on the serial console, then powers the VM off: smoke-test.nix only has to capture and grep the +# console (xtcp2 pattern). autoTest=false gives an interactive VM with root autologin. +# +# sanitizer = "kasan" | "kcsan" boots that debug kernel (debug-kernels.nix: heavy, a full +# kernel build), with homa.ko built against it, so Homa is instrumented too. +# +# kernelPackages selects the guest kernel (one VM per kernel/config.nix matrix entry, xtcp2-style). +# +# consolePorts = { serial; virtio; } makes the lifecycle guest (lifecycle.nix, pcp pattern): +# ttyS0 on a TCP port and logged to ./serial.log from the first byte, and a virtio console +# (hvc0, a root shell) on a second port for fast host-driven commands. +# +# pairNode = "server" | "client" makes the guest one half of the two-VM pair test: a second +# NIC (eth1) on a QEMU link (pairLink) to the other guest, a static IP, cp_node's node<N> +# hostnames, and pair.nix's script in place of the smoke test. + +{ pkgs +, lib +, microvm +, nixpkgs +, kernelPackages # guest kernel (with a sanitizer, the base of the debug one) +, buildSystem +, constants +, buildModule # build/module.nix: homa.ko for a kernelPackages +, homaUtils # user-space tools (demo_server1/demo_client1 for the smoke test) +, sanitizer ? null # null | "kasan" | "kcsan" +, autoTest ? false +, pairNode ? null # null | "server" | "client" +, pairPort ? null # host port of the pair link (unique per pair) +, pairExperiments ? null # sweep list for the client (pair.nix); null = normal pair test +, pairLink ? "dgram" # QEMU backend for eth1: "dgram" (UDP) | "socket" (TCP stream) +, consolePorts ? null # { serial; virtio; }: lifecycle guest consoles on host TCP ports +}: + +let + debugKernel = import ./debug-kernels.nix { inherit pkgs lib; }; + guestKernelPackages = + if sanitizer != null then debugKernel sanitizer kernelPackages else kernelPackages; + isKasan = sanitizer == "kasan"; + + # Co-pinned module: homa.ko is built against the kernel the guest boots, so the debug + # VM's homa.ko is KASAN-instrumented too (kbuild takes the flags from the tree's config). + homaKo = buildModule { kernelPackages = guestKernelPackages; enableDebug = sanitizer != null; }; + homaKoPath = "${homaKo}/lib/modules/${guestKernelPackages.kernel.modDirVersion}/homa.ko"; + + smoke = import ./smoke.nix { + inherit pkgs homaKoPath homaUtils sanitizer; + inherit (constants) oopsRegex echoRpcs sanitizerSlowdown; + }; + + pair = constants.pair; + isPair = pairNode != null; + node = pair.nodes.${pairNode}; + pairScripts = import ./pair.nix { + inherit pkgs lib homaKoPath homaUtils pair; + inherit (constants) oopsRegex echoRpcs; + experiments = pairExperiments; + }; + qdisc = import ./qdisc.nix { inherit pkgs; }; + testScript = if isPair then pairScripts.${pairNode} else smoke; + testExe = lib.getExe testScript; + isLifecycle = consolePorts != null; + + guest = nixpkgs.lib.nixosSystem { + system = buildSystem; + modules = [ + microvm.nixosModules.microvm + ({ pkgs, lib, ... }: { + boot.kernelPackages = guestKernelPackages; + boot.kernelParams = [ + "console=ttyS0,115200" + "systemd.show_status=true" + # Zero freed memory, so a use-after-free in Homa reads NULLs and oopses at once + # instead of usually reading stale but valid pointers (STATUS.md F8). + "init_on_free=1" + ] ++ lib.optionals isPair [ + # Stable names: the user NIC is eth0 and the pair link eth1. + "net.ifnames=0" + ] ++ lib.optionals isKasan [ "kmemleak=on" "kasan_multi_shot" ] + # A sanitizer kernel's udev can take minutes to announce devices; past systemd's + # 90 s wait, the getty on hvc0 never starts and the lifecycle has no shell. + ++ lib.optionals (sanitizer != null) [ "systemd.default_device_timeout_sec=600" ]; + boot.initrd.availableKernelModules = [ "9p" "9pnet" "9pnet_virtio" "virtio_pci" ]; + + microvm = { + hypervisor = "qemu"; + vcpu = constants.vcpu; + mem = if sanitizer != null then constants.memDebug else constants.mem; + + # Host /nix/store visible in the guest, so $HOMA_KO resolves. + shares = [{ + source = "/nix/store"; + mountPoint = "/nix/store"; + tag = "nix-store"; + proto = "9p"; + }]; + + # The "pc" machine (as in xtcp2): on qemu's default "microvm" machine the guest's + # 8250 UART never registers as ttyS0, so the console goes silent after early boot. + qemu.machine = "pc"; + + # Minimal NAT interface (needed only so the store mount/init works). + interfaces = [{ + type = "user"; + id = "eth0"; + mac = "52:54:00:00:00:01"; + }]; + + # Lifecycle consoles. ttyS0 stays the only kernel console, so hvc0 carries just + # the shell (no printk lines in command output) and serial.log has everything, + # including oopses after the shell is gone. Both are servers with wait=off, and + # serial.log is relative to qemu's working directory. + qemu.serialConsole = !isLifecycle; + qemu.extraArgs = lib.optionals isLifecycle [ + "-chardev" + "socket,id=ser0,host=127.0.0.1,port=${toString consolePorts.serial},server=on,wait=off,logfile=serial.log" + "-serial" + "chardev:ser0" + "-device" + "virtio-serial-pci" + "-chardev" + "socket,id=con0,host=127.0.0.1,port=${toString consolePorts.virtio},server=on,wait=off" + "-device" + "virtconsole,chardev=con0" + ] + # Pair link (eth1), point to point. "dgram": UDP between pairPort (node0) and + # pairPort + dgramPortOffset (node1). "socket": a TCP stream that node0 listens on + # and node1 connects to; it delays packets by milliseconds (STATUS.md F2), so it is + # kept only for comparison. + ++ lib.optionals isPair [ + "-netdev" + (if pairLink == "dgram" then + let + offset = node: if pairNode == node then pair.dgramPortOffset else 0; + local = pairPort + offset "client"; + remote = pairPort + offset "server"; + in + "dgram,id=homapair,local.type=inet,local.host=127.0.0.1,local.port=${toString local}" + + ",remote.type=inet,remote.host=127.0.0.1,remote.port=${toString remote}" + else + "socket,id=homapair,${node.role}=127.0.0.1:${toString pairPort}") + "-device" + "virtio-net-pci,netdev=homapair,mac=${node.mac}" + ]; + }; + + networking.interfaces = lib.mkIf isPair { + eth1 = { + useDHCP = false; + ipv4.addresses = [{ address = node.ip; inherit (pair) prefixLength; }]; + }; + }; + networking.hosts = lib.mkIf isPair (lib.mapAttrs' + (_: n: lib.nameValuePair n.ip [ n.hostname ]) + pair.nodes); + # The NixOS firewall would drop Homa (IP protocol 146) and the coordination ports. + networking.firewall.enable = !isPair; + + # Interactive use: passwordless root autologin on the console. + services.getty.autologinUser = "root"; + users.users.root.password = ""; + + services.udev.extraRules = lib.mkIf (sanitizer == "kcsan") + (import ./kcsan.nix { inherit pkgs; }); + + networking.hostName = if isPair then node.hostname else constants.hostname; + + environment.systemPackages = [ pkgs.kmod pkgs.iproute2 pkgs.coreutils homaUtils testScript ] + ++ lib.optionals isPair [ pairScripts.nodeStats qdisc ]; + environment.variables.HOMA_KO = homaKoPath; + + systemd.services.homa-test = lib.mkIf autoTest { + wantedBy = [ "multi-user.target" ]; + after = [ "multi-user.target" ]; + serviceConfig = { + Type = "oneshot"; + StandardOutput = "journal+console"; + StandardError = "journal+console"; + ExecStart = testExe; + # Pass or fail, end the run so the host sees qemu exit. + ExecStopPost = "${pkgs.systemd}/bin/systemctl poweroff --no-block"; + }; + }; + + # Trim NixOS bloat for a fast microVM. + documentation.enable = false; + documentation.nixos.enable = false; + nix.enable = false; + system.stateVersion = constants.stateVersion; + }) + ]; + }; +in +guest.config.microvm.declaredRunner diff --git a/nix/microvms/guest/pair.nix b/nix/microvms/guest/pair.nix new file mode 100644 index 00000000..e6b9cbe6 --- /dev/null +++ b/nix/microvms/guest/pair.nix @@ -0,0 +1,230 @@ +# nix/microvms/guest/pair.nix +# +# In-guest scripts for the two-VM pair test (PLAN-pairs.md). Two guests share a private L2 +# link (eth1, a QEMU "dgram" netdev). Each runs one of these as a systemd oneshot, prints +# phase markers on the serial console, then powers off. The host (pair-test.nix) greps both +# consoles. +# +# server (node0): insmod -> demo_server1 + cp_node servers (Homa, TCP) -> open the "ready" +# port -> wait for the client's report on the "done" port -> stop servers -> rmmod -> +# dmesg gate -> HOMA_PAIR_SERVER_PASS | HOMA_PAIR_SERVER_FAIL:<phase> +# client (node1): insmod -> ping node0 -> wait for the ready port -> echo 100 B..1 MB across +# VMs -> cp_node Homa load -> cp_node TCP load (baseline) -> rmmod -> dmesg gate -> +# report to node0 -> HOMA_PAIR_PASS | HOMA_PAIR_FAIL:<phase> +# +# The ready/done ports are plain TCP (socat), separate from Homa, so a Homa failure still +# lets the server shut down cleanly instead of waiting for the host timeout. +# +# experiments != null turns the client into a sweep (STATUS.md F2): instead of the echo and +# load phases, it runs cp_node once per entry, with the entry's net.homa sysctls applied on +# BOTH nodes. To do that the server also runs a shell on the private link's control port +# (test VMs only). Each run prints one HOMA_EXP line with the RTTs and the deltas of the +# selected /proc/net/homa_metrics counters on each node. + +{ pkgs +, lib +, homaKoPath +, homaUtils +, pair # constants.pair +, oopsRegex # constants.oopsRegex +, echoRpcs # constants.echoRpcs +, experiments ? null # null, or a list of { name, proto, workload, gbps, sysctls } +}: + +let + server = pair.nodes.server; + tools = with pkgs; [ kmod coreutils gnugrep gawk procps iproute2 iputils socat homaUtils ]; + isExp = experiments != null; + + metrics = import ./metrics.nix { inherit pkgs lib; }; + stats = lib.getExe metrics.nodeStats; + + # Shared helpers. $PFX is the marker prefix (HOMA_PAIR or HOMA_PAIR_SERVER). + common = '' + ok() { echo "''${PFX}_$1_OK"; } + dmesg_gate() { + if dmesg | grep -qE '${oopsRegex}'; then + dmesg | tail -n 40 + return 1 + fi + } + echo "''${PFX}_START host=$(cat /proc/sys/kernel/hostname) kernel=$(uname -r)" + ''; + + # The cp_node client command line, for one load run against node0. Homa uses its own + # server ports. + cpClient = { proto, workload ? pair.workload, gbps ? pair.gbps }: '' + timeout ${toString pair.loadSeconds} cp_node client --protocol ${proto} \ + --workload ${workload} --gbps ${gbps} --first-server 0 --server-nodes 1 --id 1 \ + ${lib.optionalString (proto == "homa") "--first-port ${toString pair.cpNodeHomaPort}"} \ + 2>&1 || true''; + + # The result of one cp_node client run: prints the last stats line, fails unless RPCs + # completed. + # $1 = protocol (homa|tcp), $2 = cp_node's output + cpNode = '' + cp_node_check() { + local proto=$1 out=$2 line kops + line=$(echo "$out" | grep ' clients: ' | tail -n 1) + if [ -z "$line" ]; then + echo "$out" | tail -n 20 + return 1 + fi + echo "HOMA_PAIR_CPNODE_$proto $line" + kops=$(echo "$line" | awk '{ for (i = 1; i <= NF; i++) if ($i == "clients:") print $(i + 1) }') + awk -v k="$kops" 'BEGIN { exit !(k > 0) }' + } + ''; + # Client phases, normal mode: echo, then cp_node over Homa and over TCP. + normalPhases = '' + # Echo 100 B .. 1 MB: multi-packet messages and grants over virtio-net. + data_grant() { + grep -E '^(packets_sent_DATA|packets_rcvd_GRANT) ' /proc/net/homa_metrics \ + | awk '{ s += $2 } END { print s + 0 }' + } + before=$(data_grant) + responses=$(timeout 30 demo_client1 "$peer" 4000 | grep -c '^Received response' || true) + after=$(data_grant) + echo "HOMA_PAIR_ECHO responses=$responses (want ${toString echoRpcs}) data+grant packets=$((after - before))" + [ "$responses" = ${toString echoRpcs} ] || fail ECHO + ok ECHO + + cp_node_check homa "$(${cpClient { proto = "homa"; }})" || fail CPNODE_HOMA + ok CPNODE_HOMA + cp_node_check tcp "$(${cpClient { proto = "tcp"; }})" || fail CPNODE_TCP + ok CPNODE_TCP + ''; + + # Client phases, sweep mode: one cp_node run per experiment. + expSysctls = lib.unique (lib.concatMap (e: builtins.attrNames e.sysctls) experiments); + setFor = e: lib.concatStringsSep " " + (lib.mapAttrsToList (k: v: "net.homa.${k}=${toString v}") e.sysctls); + runOne = e: '' + both "sysctl -q -w $defaults ${setFor e}" + c0=$(${stats} | snap) + s0=$(on_peer ${stats} | snap) + out=$(${cpClient { inherit (e) proto workload gbps; }}) + c1=$(${stats} | snap) + s1=$(on_peer ${stats} | snap) + line=$(echo "$out" | grep ' clients: ' | tail -n 1 | sed -E 's/^.*clients: //') + echo "HOMA_EXP name=${e.name} ''${line:-NO_STATS} | client: $(delta "$c0" "$c1")| server: $(delta "$s0" "$s1")" + ''; + sweepPhases = '' + # Run a command on the server through its control shell. + on_peer() { echo "$1" | timeout 30 socat -t 20 - "TCP:$peer:${toString pair.ctlPort}"; } + # Run on both nodes; fail unless the server confirms it ran the command. + both() { + eval "$1" + on_peer "$1 && echo CTL_OK" | grep -q '^CTL_OK$' || fail CTL + } + ${metrics.snapDelta} + # Each run starts from the defaults, then applies its own sysctls. + defaults="" + for k in ${lib.concatStringsSep " " expSysctls}; do + defaults="$defaults net.homa.$k=$(sysctl -n "net.homa.$k")" + done + echo "HOMA_EXP_DEFAULTS$defaults" + + ${lib.concatMapStrings runOne experiments} + both "sysctl -q -w $defaults" + ok SWEEP + ''; +in +{ + inherit (metrics) nodeStats; + + server = pkgs.writeShellApplication { + name = "homa-pair-server"; + runtimeInputs = tools; + text = '' + PFX=HOMA_PAIR_SERVER + ${common} + fail() { + echo "HOMA_PAIR_SERVER_FAIL:$1" + dmesg | tail -n 40 + exit 1 + } + + insmod ${homaKoPath} || fail INSMOD + ok INSMOD + + demo_server1 > /tmp/demo_server1.log 2>&1 & + cp_node server --protocol homa --first-port ${toString pair.cpNodeHomaPort} \ + > /tmp/cp_node_homa.log 2>&1 & + cp_node server --protocol tcp > /tmp/cp_node_tcp.log 2>&1 & + sleep 1 + for p in demo_server1 cp_node; do + pgrep -x "$p" > /dev/null || fail "SERVERS_$p" + done + ok SERVERS + + ${lib.optionalString isExp '' + # Sweep mode: a shell on the private link so the client can set sysctls and read + # metrics on this node. + socat TCP-LISTEN:${toString pair.ctlPort},fork,reuseaddr SYSTEM:${pkgs.bash}/bin/bash & + ''} + # Tell the client we are ready (it polls this port), then block until it reports. + socat TCP-LISTEN:${toString pair.readyPort},fork,reuseaddr /dev/null & + echo HOMA_PAIR_SERVER_READY + report=$(timeout ${toString pair.clientTimeout} \ + socat -u TCP-LISTEN:${toString pair.donePort},reuseaddr STDOUT || true) + echo "HOMA_PAIR_SERVER_CLIENT_REPORT ''${report:-<none>}" + + pkill -x demo_server1 || true + pkill -x cp_node || true + pkill -x socat || true + sleep 1 + + grep -q '^homa ' /proc/modules || fail LSMOD + rmmod homa || fail RMMOD + ok RMMOD + dmesg_gate || fail DMESG + ok DMESG + + echo HOMA_PAIR_SERVER_PASS + ''; + }; + + client = pkgs.writeShellApplication { + name = "homa-pair-client"; + runtimeInputs = tools; + text = '' + PFX=HOMA_PAIR + ${common} + ${cpNode} + peer=${server.ip} + + # Every exit reports to the server so it can shut down without waiting. + report() { echo "$1" | timeout 10 socat -u STDIN "TCP:$peer:${toString pair.donePort}" || true; } + fail() { + echo "HOMA_PAIR_FAIL:$1" + dmesg | tail -n 40 + report "FAIL:$1" + exit 1 + } + + insmod ${homaKoPath} || fail INSMOD + ok INSMOD + + for _ in $(seq 60); do ping -c 1 -W 1 "$peer" > /dev/null 2>&1 && break; done + ping -c 3 -W 2 "$peer" > /dev/null || fail PING + ok PING + + for _ in $(seq ${toString pair.serverTimeout}); do + timeout 2 socat -u /dev/null "TCP:$peer:${toString pair.readyPort}" 2> /dev/null && break + sleep 1 + done + timeout 2 socat -u /dev/null "TCP:$peer:${toString pair.readyPort}" || fail PEER_READY + ok PEER_READY + + ${if isExp then sweepPhases else normalPhases} + rmmod homa || fail RMMOD + ok RMMOD + dmesg_gate || fail DMESG + ok DMESG + + report PASS + echo HOMA_PAIR_PASS + ''; + }; +} diff --git a/nix/microvms/guest/qdisc.nix b/nix/microvms/guest/qdisc.nix new file mode 100644 index 00000000..3882de10 --- /dev/null +++ b/nix/microvms/guest/qdisc.nix @@ -0,0 +1,40 @@ +# nix/microvms/guest/qdisc.nix +# +# homa-qdisc on|off|show (guest command): attach Homa's queuing discipline to every tx +# queue of eth1, as cloudlab/bin/config's config_qdisc does (mq at the root, homa on each +# child), or as the root qdisc when eth1 has a single queue (mq needs several). The homa +# qdisc takes no options, so stock tc can attach it. "show" prints the qdiscs and the +# speed eth1 reports (virtio-net: "Unknown!", i.e. SPEED_UNKNOWN; round-4 candidate 5). + +{ pkgs }: + +pkgs.writeShellApplication { + name = "homa-qdisc"; + runtimeInputs = with pkgs; [ coreutils gnugrep iproute2 ethtool ]; + text = '' + dev=eth1 + queues=$(find /sys/class/net/$dev/queues -maxdepth 1 -name 'tx-*' | wc -l) + case "''${1:-show}" in + on) + if [ "$queues" -gt 1 ]; then + tc qdisc replace dev $dev root handle 1: mq + for i in $(seq "$queues"); do + tc qdisc replace dev $dev parent "1:$(printf %x "$i")" homa + done + else + tc qdisc replace dev $dev root homa + fi + ;; + off) + tc qdisc del dev $dev root 2> /dev/null || true + ;; + show) ;; + *) + echo "usage: homa-qdisc on|off|show" >&2 + exit 2 + ;; + esac + echo "tx queues $queues, $(ethtool $dev | grep -E 'Speed:' | sed 's/^[[:space:]]*//')" + tc qdisc show dev $dev + ''; +} diff --git a/nix/microvms/guest/smoke.nix b/nix/microvms/guest/smoke.nix new file mode 100644 index 00000000..7f65b998 --- /dev/null +++ b/nix/microvms/guest/smoke.nix @@ -0,0 +1,144 @@ +# nix/microvms/guest/smoke.nix +# +# The in-guest smoke test (xtcp2 pattern): a shellcheck-clean script that the guest runs as a +# systemd oneshot with its output on the serial console. Each phase prints HOMA_SMOKE_<phase>_OK, +# and the run ends with exactly one of HOMA_SMOKE_PASS or HOMA_SMOKE_FAIL:<phase>; the host +# (smoke-test.nix) greps the captured console log for them. +# +# insmod -> lsmod -> (KCSAN kernel: KCSAN is on, watching Homa) -> loopback echo RPCs (demo_server1 <- demo_client1, 100 B .. 1 MB: +# exercises sendmsg and the recvmsg path) -> the same echo across two netns over veth with +# GRO and the RPS flow table on (exercises homa_gro_complete -> homa_set_softirq_cpu) -> +# (sanitizer kernels: kmemleak scan + KASAN/KCSAN/UBSAN gate) -> rmmod + +{ pkgs +, oopsRegex # constants.oopsRegex +, echoRpcs # constants.echoRpcs +, homaKoPath +, homaUtils +, sanitizer ? null # null | "kasan" | "kcsan" (debug-kernels.nix) +, sanitizerSlowdown # constants.sanitizerSlowdown +}: + +let + inherit (pkgs) lib; + # Seconds an echo client may take; sanitizer kernels are several times slower. + clientTimeout = 30 * (if sanitizer != null then sanitizerSlowdown else 1); +in + +pkgs.writeShellApplication { + name = "homa-smoke"; + runtimeInputs = with pkgs; [ kmod coreutils gnugrep gawk util-linux iproute2 ethtool homaUtils ]; + text = '' + fail() { + echo "HOMA_SMOKE_FAIL:$1" + dmesg | tail -n 40 + exit 1 + } + ok() { echo "HOMA_SMOKE_$1_OK"; } + + echo "HOMA_SMOKE_START kernel=$(uname -r)" + client_timeout=${toString clientTimeout} + + insmod ${homaKoPath} || fail INSMOD + ok INSMOD + + grep -q '^homa ' /proc/modules || fail LSMOD + ok LSMOD + + ${lib.optionalString (sanitizer == "kcsan") '' + # kcsan.nix's udev rule turns KCSAN on once homa.ko is loaded, watching only Homa's + # functions (debugfs lists them, one per line, after "whitelisted functions:"). + kcsan=/sys/kernel/debug/kcsan + for _ in $(seq 40); do + grep -q '^enabled: 1' "$kcsan" && break + sleep 0.5 + done + watched=$(grep -c '^ [a-z_]' "$kcsan" || true) + echo "HOMA_SMOKE_KCSAN watching=$watched functions" + grep -q '^enabled: 1' "$kcsan" || fail KCSAN + [ "$watched" -gt 0 ] || fail KCSAN + ok KCSAN + ''} + demo_server1 > /tmp/demo_server1.log 2>&1 & + server=$! + sleep 1 + responses=$(timeout "$client_timeout" demo_client1 127.0.0.1 4000 | grep -c '^Received response' || true) + kill "$server" 2>/dev/null || true + wait "$server" 2>/dev/null || true + echo "HOMA_SMOKE_ECHO responses=$responses (want ${toString echoRpcs})" + [ "$responses" = ${toString echoRpcs} ] || fail ECHO + ok ECHO + + # GRO + RPS: loopback skips GRO, so repeat the echo between two network namespaces over + # a veth pair with GRO on (veth then receives through NAPI). veth only hands a packet to + # GRO if the sending device has TSO off (veth_skb_is_eligible_for_gro), so turn it off. + # With the RPS socket flow table allocated, homa_gro_complete -> homa_set_softirq_cpu + # writes table entries; on 7.1+ that is the rps_tag_ptr code. A kprobe counts the calls + # to prove it ran. + echo 32768 > /proc/sys/net/core/rps_sock_flow_entries || fail GRO_SYSCTL + ip netns add homa-srv + ip netns add homa-cli + ip link add homa-v0 netns homa-srv type veth peer name homa-v1 netns homa-cli + ip -n homa-srv addr add 10.77.0.1/24 dev homa-v0 + ip -n homa-cli addr add 10.77.0.2/24 dev homa-v1 + for ns_dev in homa-srv:homa-v0 homa-cli:homa-v1; do + ns=''${ns_dev%%:*} + dev=''${ns_dev#*:} + ip -n "$ns" link set lo up + ip -n "$ns" link set "$dev" up + ip netns exec "$ns" ethtool -K "$dev" gro on tso off + done + + tracing=/sys/kernel/tracing + [ -e "$tracing/kprobe_events" ] || mount -t tracefs nodev "$tracing" + for probe in homa_gro_receive homa_set_softirq_cpu; do + echo "p:p_$probe $probe" >> "$tracing/kprobe_events" || fail GRO_KPROBE + echo 1 > "$tracing/events/kprobes/p_$probe/enable" + done + + ip netns exec homa-srv demo_server1 > /tmp/demo_server1_gro.log 2>&1 & + server=$! + sleep 1 + responses=$(timeout "$client_timeout" ip netns exec homa-cli demo_client1 10.77.0.1 4000 \ + | grep -c '^Received response' || true) + kill "$server" 2>/dev/null || true + wait "$server" 2>/dev/null || true + + gro_hits=$(awk '$1 == "p_homa_gro_receive" { print $2 }' "$tracing/kprobe_profile") + hits=$(awk '$1 == "p_homa_set_softirq_cpu" { print $2 }' "$tracing/kprobe_profile") + for probe in homa_gro_receive homa_set_softirq_cpu; do + echo 0 > "$tracing/events/kprobes/p_$probe/enable" + echo "-:p_$probe" >> "$tracing/kprobe_events" + done + ip netns del homa-srv + ip netns del homa-cli + + echo "HOMA_SMOKE_GRO responses=$responses (want ${toString echoRpcs}) homa_gro_receive calls=$gro_hits" \ + "homa_set_softirq_cpu calls=$hits" + [ "$responses" = ${toString echoRpcs} ] || fail GRO + [ "''${gro_hits:-0}" -gt 0 ] || fail GRO_NOT_USED + [ "''${hits:-0}" -gt 0 ] || fail GRO_RPS + ok GRO + + ${lib.optionalString (sanitizer != null) '' + echo scan > /sys/kernel/debug/kmemleak 2>/dev/null || true + sleep 5 + # Any report so far, before rmmod: KASAN, KCSAN, UBSAN, lockdep ("WARNING: ... lock + # ..."), or a leak the kmemleak scan found. + if dmesg | grep -qE '${oopsRegex}|unreferenced object'; then + fail SANITIZER + fi + ok SANITIZER + ''} + + rmmod homa || fail RMMOD + ok RMMOD + + if dmesg | grep -qE '${oopsRegex}'; then + fail DMESG + fi + ok DMESG + + echo HOMA_SMOKE_PASS + ''; +} diff --git a/nix/microvms/host/README.md b/nix/microvms/host/README.md new file mode 100644 index 00000000..ffd7299c --- /dev/null +++ b/nix/microvms/host/README.md @@ -0,0 +1,21 @@ +# nix/microvms/host: the commands that run the VM tests + +Each file here makes one kind of `nix run` target. It starts the guest's QEMU runner +([`../guest`](../guest)), watches it, and prints the verdict. + +| File | Targets | How | +|---|---|---| +| `smoke-test.nix` | `homa-microvm-test*` | bash: boots the guest under `timeout`, greps the serial log for `HOMA_SMOKE_PASS` | +| `pair-test.nix` | `homa-pair-test*`, `homa-pair-exp*` | bash: boots node0, waits for its link port, boots node1, greps both logs | +| `lifecycle.nix` | `homa-lifecycle*` | Tcl: [`../tcl/bin/lifecycle.tcl`](../tcl/bin/lifecycle.tcl) with this file's configuration | +| `pair-lifecycle.nix` | `homa-pair-lifecycle*` | Tcl: [`../tcl/bin/pair-lifecycle.tcl`](../tcl/bin/pair-lifecycle.tcl) | +| `matrix.nix` | `homa-vm-matrix` | Tcl: [`../tcl/bin/vm-matrix.tcl`](../tcl/bin/vm-matrix.tcl) runs every test above at once | +| `tcl.nix` | | `mkTclApp`, which packages a Tcl driver as a command, and the `tcl-tests` check | + +## Configuration goes in as JSON + +The Tcl drivers hold no settings of their own. Each `.nix` file here builds an attribute +set (QEMU runner paths, console ports, timeouts, the netem phases from +[`../constants.nix`](../constants.nix)), and `mkTclApp` writes it out with +`builtins.toJSON` and passes the file to the driver. So a timeout or a port is changed in +one place, in Nix, and the driver reads it. diff --git a/nix/microvms/host/lifecycle.nix b/nix/microvms/host/lifecycle.nix new file mode 100644 index 00000000..d68947e1 --- /dev/null +++ b/nix/microvms/host/lifecycle.nix @@ -0,0 +1,41 @@ +# nix/microvms/host/lifecycle.nix +# +# Lifecycle test (pcp/xdp2 pattern): the host boots a guest with two consoles (mkVm.nix +# consolePorts) and drives it phase by phase, with a timeout and a time for each. The +# driver is ../tcl/bin/lifecycle.tcl (Tcl/Expect, see ../tcl/README.md); this file only gives it +# its configuration: +# +# 1 start qemu running, console ports listening +# 2 serial first kernel line in serial.log (ttyS0, 115200 baud, logged from the start) +# 2b virtio a root shell answering on hvc0 (the fast virtio console) +# 3 smoke homa-smoke (smoke.nix) over hvc0: insmod, echo, GRO/RPS, rmmod +# 4 cycles insmod / netns add+del / rmmod, moduleCycles times (pernet init and exit) +# 5 dmesg no BUG/Oops/WARNING/KASAN/UBSAN in the guest's dmesg, read over hvc0 +# 6 shutdown poweroff over hvc0 +# 7 exit qemu exits +# 8 serial-log no oops anywhere in serial.log, including after the shell has gone +# (late RCU callbacks: STATUS.md F8) +# +# The phase table is printed and written to summary.md; the last line is the VERDICT. Logs +# are in $HOMA_VM_LOGDIR (default: a fresh mktemp -d), which is also qemu's working +# directory. + +{ constants +, tclApps # tcl.nix +, runner # config.microvm.declaredRunner for a guest built with consolePorts +, consolePorts # { serial; virtio; } +, label ? "" +}: + +let + lc = constants.lifecycle; +in +tclApps.mkTclApp { + name = "homa-lifecycle${label}"; + driver = "lifecycle.tcl"; + config = { + inherit label; + inherit (lc) timeouts moduleCycles; + vm = consolePorts // { runner = "${runner}"; }; + }; +} diff --git a/nix/microvms/host/matrix.nix b/nix/microvms/host/matrix.nix new file mode 100644 index 00000000..7db5186f --- /dev/null +++ b/nix/microvms/host/matrix.nix @@ -0,0 +1,28 @@ +# nix/microvms/host/matrix.nix +# +# One command for every VM test (PLAN-pairs.md phase 3): runs each per-kernel smoke test, +# lifecycle and pair lifecycle, and each (server, client) pair test, in parallel, then +# prints a summary table +# (also written to $logdir/summary.md): +# +# | Test | Kernel(s) | Verdict | Time (s) | Echo | GRO calls | Homa Kops/s | Homa P50 / P99 (us) | TCP ... | +# +# Each test keeps its own console logs and output under $HOMA_VM_LOGDIR/<test>/. The tests +# share nothing on the host (each pair has its own link port), so they can all run at once; +# HOMA_VM_JOBS caps how many run together (default: all). Exits non-zero if any test fails. +# The driver is ../tcl/bin/vm-matrix.tcl (see ../tcl/README.md); this file gives it the tests. + +{ lib +, tclApps # tcl.nix +, echoRpcs # constants.echoRpcs +, tests # list of { name, kernels, exe }: exe is one test's package (default.nix) +}: + +tclApps.mkTclApp { + name = "homa-vm-matrix"; + driver = "vm-matrix.tcl"; + config = { + inherit echoRpcs; + tests = map (t: { inherit (t) name kernels; exe = lib.getExe t.exe; }) tests; + }; +} diff --git a/nix/microvms/host/pair-lifecycle.nix b/nix/microvms/host/pair-lifecycle.nix new file mode 100644 index 00000000..ff38a6e8 --- /dev/null +++ b/nix/microvms/host/pair-lifecycle.nix @@ -0,0 +1,79 @@ +# nix/microvms/host/pair-lifecycle.nix +# +# Pair lifecycle test (PLAN-pairs.md phase 4): the two-VM pair (node0 server, node1 client, +# on a private eth1 link) driven from the host over each guest's hvc0, with every phase +# timed. Neither guest runs a test of its own. The driver is ../tcl/bin/pair-lifecycle.tcl +# (Tcl/Expect, see ../tcl/README.md); this file only gives it its configuration. +# +# 1-3 start, serial, virtio each node +# 4 insmod each node +# 5 link node1 pings node0 over eth1 +# 6 servers demo_server1 and cp_node Homa/TCP servers on node0 +# 7 traffic baseline from node1: echo (demo_client1, 100 B .. 1 MB, six RPCs), +# cp_node Homa, cp_node TCP; counter deltas on both nodes +# 8.i netem <name> tc netem on eth1 of both nodes (constants.lifecycle.netem), +# traffic <name> then the same traffic, which must still complete. Each +# node's measured netem drop rate must match the configured +# loss (no tail drops: every qdisc has limit >= 100000), and +# with loss, Homa must have sent RESENDs +# 9 netem clear, traffic recovered +# 10 qdisc homa Homa's qdisc on eth1 of both nodes (qdisc.nix), traffic +# through it (TCP must pass the qdisc). virtio-net reports +# no link speed (round-4 candidate 5) +# 11 qdisc paced net.homa.link_mbps lowered so the pacer must defer and +# send packets; traffic +# 12 qdisc off link_mbps restored, default qdisc back +# 13 stop servers +# 14 rmmod each node +# 15-18 dmesg, shutdown, exit, serial-log each node +# +# The phase table is printed and written to summary.md; the last line is the VERDICT. Logs +# are in $HOMA_VM_LOGDIR (default: a fresh mktemp -d): node0/ and node1/ (each the working +# directory of its qemu, with serial.log), and traffic/<name>/ per traffic run. + +{ lib +, constants +, tclApps # tcl.nix +, serverRunner # declaredRunner: pairNode = "server", consolePorts = ports.node0 +, clientRunner # declaredRunner: pairNode = "client", consolePorts = ports.node1 +, ports # { node0 = { serial; virtio; }; node1 = { serial; virtio; }; } +, metrics # metrics.nix: expMetrics are the counters each traffic phase reports +, label ? "" +, slowdown ? 1 # every phase timeout is multiplied by this (sanitizer kernels are slow) +}: + +let + lc = constants.lifecycle; + pair = constants.pair; + netem = lc.netem; + netemArgs = p: lib.optionalString (p.delay != null) "delay ${p.delay} " + + "loss ${p.loss}% limit ${toString netem.limit}"; +in +# Shallower queues would let netem tail-drop under load and count it as link loss. +assert netem.limit >= 100000; +tclApps.mkTclApp { + name = "homa-pair-lifecycle${label}"; + driver = "pair-lifecycle.tcl"; + config = { + inherit label; + # Booting slows down more than the rest: a sanitizer guest's udev can take many minutes + # to announce hvc0, so its boot gets the same budget as the debug smoke test's. + timeouts = lib.mapAttrs (_: t: t * slowdown) lc.timeouts + // lib.optionalAttrs (slowdown > 1) { virtio = constants.bootTimeoutDebug; }; + inherit (lc.qdisc) pacedMbps; + inherit (constants) echoRpcs; + peer = pair.nodes.server.ip; + metrics = metrics.expMetrics; + cpNode = { + inherit (pair) workload gbps; + seconds = pair.loadSeconds; + homaPort = pair.cpNodeHomaPort; + }; + netem = { + inherit (netem) limit; + phases = map (p: { inherit (p) name loss; args = netemArgs p; }) netem.phases; + }; + node0 = ports.node0 // { runner = "${serverRunner}"; }; + node1 = ports.node1 // { runner = "${clientRunner}"; }; + }; +} diff --git a/nix/microvms/host/pair-test.nix b/nix/microvms/host/pair-test.nix new file mode 100644 index 00000000..ebf0f9ef --- /dev/null +++ b/nix/microvms/host/pair-test.nix @@ -0,0 +1,100 @@ +# nix/microvms/host/pair-test.nix +# +# Orchestrator for the two-VM pair test (PLAN-pairs.md, uds-rdma-proxy pattern, console +# markers as in smoke-test.nix). Both guests drive themselves (pair.nix) and power off when done, +# so the host only has to: +# 1. start node0 (server), whose QEMU binds its end of the link (UDP for "dgram", a TCP +# listen for "socket"); +# 2. wait until that port is open (checked with ss; with "socket" a probe connection +# would take the link's only accept); +# 3. start node1 (client); +# 4. wait for both qemu processes to exit, then grep both consoles: +# node1: HOMA_PAIR_PASS node0: HOMA_PAIR_SERVER_PASS +# Logs go to $HOMA_VM_LOGDIR (default: a fresh mktemp -d), printed at the start. + +{ pkgs +, constants +, serverRunner # declaredRunner for the pairNode = "server" guest +, clientRunner # declaredRunner for the pairNode = "client" guest +, label ? "" # "", "-6_12-7_3" (server kernel first), ... +, port # host port of the QEMU socket link; unique per pair so pairs can run together +, link ? "dgram" # "dgram" (node0 binds UDP) | "socket" (node0 listens on TCP) +}: + +let + pair = constants.pair; + # How to see that node0's QEMU has its end of the link open. + ssFlags = if link == "dgram" then "-Hlun" else "-Hltn"; +in +pkgs.writeShellApplication { + name = "homa-pair-test${label}"; + runtimeInputs = with pkgs; [ coreutils gnugrep iproute2 procps ]; + text = '' + logdir="''${HOMA_VM_LOGDIR:-$(mktemp -d)}" + mkdir -p "$logdir" + slog="$logdir/node0${label}.log" + clog="$logdir/node1${label}.log" + port=${toString port} + echo "console logs: $slog $clog" + # Is anything bound to the link port yet (node0's qemu)? + link_open() { ss ${ssFlags} "sport = :$port" | grep -q .; } + + if link_open; then + echo "port $port is already in use (another pair test running?)" + echo "VERDICT: FAIL" + exit 1 + fi + + timeout --kill-after=10s ${toString pair.runTimeout}s \ + ${serverRunner}/bin/microvm-run > "$slog" 2>&1 & + spid=$! + + for _ in $(seq 60); do + link_open && break + kill -0 "$spid" 2>/dev/null || break + sleep 0.5 + done + if ! link_open; then + echo "node0 never listened on 127.0.0.1:$port" + tail -n 30 "$slog" + kill "$spid" 2>/dev/null || true + echo "VERDICT: FAIL" + exit 1 + fi + + cstatus=0 + timeout --kill-after=10s ${toString pair.runTimeout}s \ + ${clientRunner}/bin/microvm-run > "$clog" 2>&1 || cstatus=$? + [ "$cstatus" = 124 ] && echo "node1 timed out" + + # The server powers off after the client's report; give it a moment, then stop it. + for _ in $(seq 60); do + kill -0 "$spid" 2>/dev/null || break + sleep 1 + done + kill "$spid" 2>/dev/null || true + wait "$spid" 2>/dev/null || true + + # Lines look like "[ 13.44] homa-pair-client[522]: HOMA_PAIR_ECHO_OK"; -a because the + # logs also hold terminal escape sequences. + show() { grep -aE '\]: HOMA_(PAIR_|EXP)' "$1" | sed -E 's/^.*\]: //' | tr -d '\r' || true; } + echo "--- node0 (server)" + show "$slog" + echo "--- node1 (client)" + show "$clog" + + verdict=PASS + grep -aqE '\]: HOMA_PAIR_SERVER_PASS' "$slog" || verdict=FAIL + grep -aqE '\]: HOMA_PAIR_PASS' "$clog" || verdict=FAIL + if [ "$verdict" = FAIL ]; then + for f in "$slog" "$clog"; do + if ! grep -aqE '\]: HOMA_PAIR_(SERVER_)?(PASS|FAIL)' "$f"; then + echo "--- $(basename "$f") never reported; last console lines:" + tail -n 30 "$f" + fi + done + fi + echo "VERDICT: $verdict" + [ "$verdict" = PASS ] + ''; +} diff --git a/nix/microvms/host/smoke-test.nix b/nix/microvms/host/smoke-test.nix new file mode 100644 index 00000000..ab4a0d40 --- /dev/null +++ b/nix/microvms/host/smoke-test.nix @@ -0,0 +1,52 @@ +# nix/microvms/host/smoke-test.nix +# +# The microVM orchestrator (xtcp2 pattern). Produces a writeShellApplication that boots a +# guest built with autoTest=true, captures its serial console, and returns a verdict. The +# guest runs the smoke test itself (smoke.nix) and powers off, so the host only waits for +# qemu to exit and greps the console log: +# HOMA_SMOKE_PASS -> VERDICT: PASS +# HOMA_SMOKE_FAIL:<phase> -> VERDICT: FAIL (phase reported) +# neither before timeout -> VERDICT: FAIL (boot or hang) +# +# The log is kept in $HOMA_VM_LOGDIR (default: a fresh mktemp -d), printed at the start. + +{ pkgs +, constants +, runner # config.microvm.declaredRunner for an autoTest guest +, label ? "" # "", "-debug", "-7_3", ... +, withSanitizers ? false +}: + +let + timeout = if withSanitizers then constants.bootTimeoutDebug else constants.bootTimeout; +in +pkgs.writeShellApplication { + name = "homa-microvm-test${label}"; + runtimeInputs = [ pkgs.coreutils pkgs.gnugrep ]; + text = '' + logdir="''${HOMA_VM_LOGDIR:-$(mktemp -d)}" + mkdir -p "$logdir" + log="$logdir/console${label}.log" + echo "microVM console log: $log" + + status=0 + timeout --kill-after=10s ${toString timeout}s ${runner}/bin/microvm-run > "$log" 2>&1 || status=$? + [ "$status" = 124 ] && echo "microVM timed out after ${toString timeout}s" + + # Console lines look like "[ 13.44] homa-smoke[522]: HOMA_SMOKE_PASS"; -a because the + # log also holds terminal escape sequences. + marker='homa-smoke\[[0-9]+\]: HOMA_SMOKE_' + grep -aE "$marker" "$log" | tr -d '\r' || true + + if grep -aqE "''${marker}PASS" "$log"; then + echo "VERDICT: PASS" + exit 0 + fi + if ! grep -aqE "''${marker}FAIL" "$log"; then + echo "smoke test never reported; last console lines:" + tail -n 30 "$log" + fi + echo "VERDICT: FAIL" + exit 1 + ''; +} diff --git a/nix/microvms/host/tcl.nix b/nix/microvms/host/tcl.nix new file mode 100644 index 00000000..141acd52 --- /dev/null +++ b/nix/microvms/host/tcl.nix @@ -0,0 +1,162 @@ +# nix/microvms/host/tcl.nix +# +# Packages the Tcl/Expect drivers in ../tcl (see ../tcl/README.md): +# +# mkTclApp { name, driver, config? } bin/<name>: runs ../tcl/bin/<driver> with config, an +# attribute set written out as JSON, if it is given +# test homa-tcl-test: the tcltest suite, or part of it +# lint homa-tcl-lint: nagelfar, tclint, the oops regex +# tests the check: lint, the suite under Tcl 8.6 (with +# Expect) and the pure tables under Tcl 9.0 +# +# Expect is built for Tcl 8.6 in nixpkgs, so the drivers run on 8.6; the pure code +# (parse.tcl, protocol.tcl, report.tcl) is kept Tcl 9 clean by running its tests there too. + +{ pkgs, lib, oopsRegex }: + +let + tcl = pkgs.tcl; + tcllib = pkgs.tclPackages.tcllib; + + # The package and drivers only: editing the tests or README does not change the apps. + src = lib.fileset.toSource { + root = ../tcl; + fileset = lib.fileset.unions [ + ../tcl/pkgIndex.tcl + ../tcl/parse.tcl + ../tcl/protocol.tcl + ../tcl/report.tcl + ../tcl/console.tcl + ../tcl/lc.tcl + ../tcl/bin + ]; + }; + + testSrc = lib.fileset.toSource { + root = ../tcl; + fileset = lib.fileset.unions [ + (lib.fileset.fileFilter (f: f.hasExt "tcl" || f.hasExt "test") ../tcl) + ../tcl/tclint.toml + ]; + }; + + # Where the drivers find their packages: ours, Expect and tcllib (json). + autoPath = [ "${src}" "${pkgs.expect}/lib" "${tcllib}/lib" ]; + + # nixpkgs' nagelfar has no syntax database; its source tarball does. + nagelfarDb = pkgs.runCommand "nagelfar-db" { } '' + mkdir -p $out + tar -xzf ${pkgs.nagelfar.src} --strip-components=1 -C $out \ + --wildcards '*/syntaxdb86.tcl' '*/packagedb/tcltestdb.tcl' + ''; + # The test files that need neither Expect nor a guest, which also run on Tcl 9.0. + pureTests = "parse.test protocol.test report.test style.test"; + + # Both apps work in a writable copy of testSrc (the linters and tests write files), which + # the shell's EXIT trap removes. + inCopy = '' + work=$(mktemp -d) + trap 'rm -rf "$work"' EXIT + cp -r ${testSrc}/. "$work" + chmod -R u+w "$work" + cd "$work" + export LANG=C.UTF-8 + ''; + + # nix run .#homa-tcl-lint: nagelfar, tclint, and the oops regex against constants.nix. + lint = pkgs.writeShellApplication { + name = "homa-tcl-lint"; + runtimeInputs = [ tcl pkgs.coreutils pkgs.gnugrep pkgs.nagelfar pkgs.tclint ]; + text = '' + ${inCopy} + echo "== nagelfar" + # homa.db describes our own commands and classes, generated from the loaded code. + TCLLIBPATH="${pkgs.expect}/lib" tclsh tests/lintdb.tcl > homa.db + nagelfar -H -s ${nagelfarDb}/syntaxdb86.tcl -s ${nagelfarDb}/packagedb/tcltestdb.tcl -s homa.db \ + ./*.tcl bin/*.tcl tests/*.tcl tests/*.test | tee nagelfar.out + if grep -E '^[^:]+: (Line +)?[0-9]+: [EW] ' nagelfar.out; then exit 1; fi + + echo "== tclint" + tclint . + + echo "== parse.tcl's oops regex is constants.oopsRegex" + tclsh <<'EOF' + source parse.tcl + if {$homa::lc::parse::oopsRe ne {${oopsRegex}}} { + puts "parse.tcl: oopsRe is $homa::lc::parse::oopsRe, constants.nix has {${oopsRegex}}" + exit 1 + } + EOF + ''; + }; + + # nix run .#homa-tcl-test -- [--tcl9] [tcltest options]: the suite on Tcl 8.6 with Expect, + # or with --tcl9 on Tcl 9.0, where only the pure files can run (they are the default + # there). The options go to tests/all.tcl, e.g. -file wait.test, -match 'waitFor-*'. + test = pkgs.writeShellApplication { + name = "homa-tcl-test"; + runtimeInputs = [ pkgs.bash pkgs.coreutils ]; + text = '' + tclsh=${tcl}/bin/tclsh + export TCLLIBPATH=${pkgs.expect}/lib + if [[ ''${1-} == --tcl9 ]]; then + shift + tclsh=${pkgs.tcl-9_0}/bin/tclsh + unset TCLLIBPATH + if [[ " $* " != *" -file "* ]]; then + set -- -file "${pureTests}" "$@" + fi + fi + ${inCopy} + rc=0 + "$tclsh" tests/all.tcl "$@" || rc=$? + exit "$rc" + ''; + }; +in +{ + inherit lint test; + + # Without a config, the driver gets only the command line's arguments. + mkTclApp = { name, driver, config ? null }: + let + configFile = pkgs.writeText "${name}.json" (builtins.toJSON config); + configArg = lib.optionalString (config != null) configFile; + in + pkgs.writeTextFile { + inherit name; + executable = true; + destination = "/bin/${name}"; + text = '' + #!${tcl}/bin/tclsh + # ${name}: runs ${driver} (nix/microvms/tcl/bin)${lib.optionalString (config != null) " with ${configFile}"}. + set env(PATH) ${lib.makeBinPath [ pkgs.coreutils ]}:$env(PATH) + set auto_path [list ${lib.concatStringsSep " " autoPath} {*}$auto_path] + package require homa::lc + set argv [list ${configArg} {*}$argv] + homa::lc::utf8 stdout + source ${src}/bin/${driver} + ''; + checkPhase = '' + ${tcl}/bin/tclsh <<'EOF' + set auto_path [list ${lib.concatStringsSep " " autoPath} {*}$auto_path] + package require homa::lc + package require json + if {![info complete [read [open ${src}/bin/${driver}]]]} { + error "${driver}: incomplete script" + } + EOF + ''; + meta.mainProgram = name; + }; + + # The check: everything the two apps do, with their defaults. + tests = pkgs.runCommand "homa-tcl-tests" { nativeBuildInputs = [ lint test ]; } '' + homa-tcl-lint + echo "== tests, Tcl 8.6 with Expect" + homa-tcl-test + echo "== pure tests, Tcl 9.0" + homa-tcl-test --tcl9 + touch $out + ''; +} diff --git a/nix/microvms/ports.nix b/nix/microvms/ports.nix new file mode 100644 index 00000000..8d41253b --- /dev/null +++ b/nix/microvms/ports.nix @@ -0,0 +1,69 @@ +# nix/microvms/ports.nix +# +# Every host port the VM tests bind on 127.0.0.1, from one table, so that all of them can +# run at once (homa-vm-matrix). Evaluation fails if two tests would share a port. +# +# console <name> { serial; virtio; }: a lifecycle guest's consoles (TCP), two ports from +# lifecycle.portBase on +# pairConsoles <name> +# { node0 = { serial; virtio; }; node1 = ...; }: a pair lifecycle's +# consoles (TCP), four ports from lifecycle.pairConsoleBase on +# link <name> a pair's eth1 link: one port from pair.linkPortBase on. A "dgram" +# link is UDP on link and link + pair.dgramPortOffset, a "socket" link +# TCP on link. +# +# <name> is "default", a vmMatrix series or a debugNames entry (the sanitizer VMs) for the +# consoles; for links, any name in linkNames. Ports are handed out in list order, so adding +# a kernel only appends ports. + +{ lib, constants, vmSeries, debugNames, linkNames }: + +let + lc = constants.lifecycle; + pair = constants.pair; + + consoleNames = [ "default" ] ++ vmSeries ++ debugNames; + indexOf = names: lib.listToAttrs (lib.imap0 (i: n: lib.nameValuePair n i) names); + consoleIndex = indexOf consoleNames; + linkIndex = indexOf (map (l: l.name) linkNames); + + console = name: + let base = lc.portBase + 2 * consoleIndex.${name}; in + { serial = base; virtio = base + 1; }; + + pairConsoles = name: + let base = lc.pairConsoleBase + 4 * consoleIndex.${name}; in + { + node0 = { serial = base; virtio = base + 1; }; + node1 = { serial = base + 2; virtio = base + 3; }; + }; + + link = name: pair.linkPortBase + linkIndex.${name}; + + # Every port as "<proto>/<port>" with its owner: TCP and UDP ports are separate, so a UDP + # link may share a number with a TCP console. + bound = + lib.concatMap + (n: map (p: { key = "tcp/${toString p}"; owner = "console ${n}"; }) + (lib.attrValues (console n) + ++ lib.concatMap lib.attrValues (lib.attrValues (pairConsoles n)))) + consoleNames + ++ lib.concatMap + (l: + let p = link l.name; in + if l.link == "dgram" then [ + { key = "udp/${toString p}"; owner = "link ${l.name}"; } + { key = "udp/${toString (p + pair.dgramPortOffset)}"; owner = "link ${l.name}"; } + ] else [ + { key = "tcp/${toString p}"; owner = "link ${l.name}"; } + ]) + linkNames; + + clashes = lib.filterAttrs (_: owners: lib.length owners > 1) + (lib.groupBy' (acc: b: acc ++ [ b.owner ]) [ ] (b: b.key) bound); +in +assert lib.assertMsg (clashes == { }) + "nix/microvms/ports.nix: ports bound twice: ${builtins.toJSON clashes}"; +{ + inherit console pairConsoles link; +} diff --git a/nix/microvms/tcl/README.md b/nix/microvms/tcl/README.md new file mode 100644 index 00000000..2e3c55dc --- /dev/null +++ b/nix/microvms/tcl/README.md @@ -0,0 +1,161 @@ +# nix/microvms/tcl: the lifecycle drivers, in Tcl and Expect + +The lifecycle tests boot QEMU guests and drive them from the host, phase by phase, over +each guest's consoles. This directory is the code that does the driving: a small Tcl +package, `homa::lc`, and one driver script per test. Nix (`../host/tcl.nix`) packages each +driver as a command and passes it its configuration as a JSON file. That way, every port, +timeout and netem setting still comes from `../constants.nix`. + +| Command | Driver | Nix wrapper | +|----------------------------------|--------------------------|------------------------------| +| `homa-lifecycle[-<series>]` | `bin/lifecycle.tcl` | `../host/lifecycle.nix` | +| `homa-pair-lifecycle[-<series>]` | `bin/pair-lifecycle.tcl` | `../host/pair-lifecycle.nix` | +| `homa-vm-matrix` | `bin/vm-matrix.tcl` | `../host/matrix.nix` | +| `homa-vm-compare` | `bin/vm-compare.tcl` | `../default.nix` (no config) | + +```sh +nix run .#homa-lifecycle-7_3 # one guest: boot, smoke, module cycles, dmesg, poweroff +nix run .#homa-pair-lifecycle-6_12 # two guests: traffic, netem loss, Homa's qdisc +HOMA_VM_LOGDIR=/tmp/lc nix run .#homa-lifecycle # keep the logs somewhere known +HOMA_VM_JOBS=4 nix run .#homa-vm-matrix # every VM test, at most 4 at once +nix build .#checks.x86_64-linux.tcl-tests # the tests and lint below +nix run .#homa-tcl-test -- -file wait.test # some of the tests (see Tests) +nix run .#homa-tcl-lint # just the lint +nix run .#homa-vm-compare -- /tmp/before /tmp/after # two matrix runs, phase by phase +``` + +`nix develop` has `tclsh` with Expect and tcllib, `nagelfar` and `tclint`, for working on +the drivers by hand. + +### Comparing two runs + +After changing a driver, run the matrix with the logs kept, and compare them with a run +from before the change: + +```sh +HOMA_VM_LOGDIR=/tmp/after nix run .#homa-vm-matrix +nix run .#homa-vm-compare -- /tmp/before /tmp/after +``` + +For each test (`<test>.out`), `homa-vm-compare` compares what two good runs share +(`parse shape`): each phase's id, name and verdict, the `HOMA_*` lines with their numbers +masked, and the `VERDICT` lines. Times, details and counters are ignored. It prints +`<test>: same (N lines)`, or `<test>: differs` followed by the changes (`-` for a line only +in the old run, `+` for one only in the new run, from `report diff`). It exits 0 if every +test is the same, 1 if not, and 2 on a usage error. + +## Files + +| File | Package | What it is | +|---|---|---| +| `protocol.tcl` | `homa::lc::protocol` | The console's end-marker protocol (below): `protocol wrap` builds the line to send, `protocol line` reads one line back. Pure; only `console.tcl` uses it. | +| `parse.tcl` | `homa::lc::parse` | Pure functions that read guest output: counters and their deltas, cp_node lines, netem stats (`parse netemCheck`), oopses, smoke verdicts. An ensemble: `parse field $line P99`. | +| `report.tcl` | `homa::lc::report` | Pure functions that format results: verdicts, phase lines, `summary.md` rows, counter deltas, a pair traffic run's detail (`report trafficDetail`) and `homa-vm-matrix`'s rows. | +| `console.tcl` | `homa::lc::console` | `Console`, a TclOO class. It runs one shell command at a time over a guest console with Expect, and returns `{rc out}`. | +| `lc.tcl` | `homa::lc` | `Table`, the phase table: it times each phase, prints its line and writes `summary.md` with the VERDICT. `Node`, one guest: its QEMU process, its consoles, and the phases every test runs on each guest; `Node phase` runs a phase that needs the guest's shell. `Batch` runs whole tests as child processes, up to a job limit, through the event loop. `waitFor` (below), `readFile` and `writeFile`. | +| `pkgIndex.tcl` | | Tells `package require` where each package lives. | +| `bin/*.tcl` | | The drivers. Each reads its JSON config, then runs its phases; `vm-compare.tcl` takes two log directories instead. | +| `tests/` | | The tcltest suite and the lint database generator (below). | + +## The console protocol + +Each guest runs a root shell on hvc0, and QEMU exposes that console as a host TCP port. +`Console run` sends one command line followed by an end marker: + +``` +homa-smoke 2>&1; echo "HOMA_LC_""END 7 rc=$?" +``` + +It then reads lines until `HOMA_LC_END 7 rc=<n>` arrives. The quotes split the marker in +the command as sent, so a console that echoes the command back cannot fake the marker. A +command that times out leaves its own marker to arrive later. The marker's number tells the +next command to skip it. + +`run` returns rc 124 on a timeout and 125 once the connection has closed, the same codes +`timeout(1)` uses. Expect does the buffering, so a line split across TCP reads, a CRLF, or +a line longer than Expect's buffer are all handled in one place. + +## The phase table contract + +Each phase prints one line, and the run ends with one VERDICT line. `homa-vm-matrix` and +humans both read these, so the format is fixed (`report phaseLine`): + +``` + phase 3 smoke: PASS (5210 ms): HOMA_SMOKE_PASS +VERDICT: FAIL (2 phases) +``` + +A phase script runs in the caller's scope and gives `{rc detail}`, as its value or with +`return`. An rc of 0 is a PASS, `skip` is a SKIP, and any other integer is a FAIL. Anything +else fails only that phase, and the run goes on: an error (its stack is appended to +`phase-errors.log` in the log directory), a `break` or `continue`, or a result without a +valid rc. Once a guest's shell is gone, the phases that need it (`Node phase`) SKIP. + +Waiting is done by `waitFor ms interval condition`: it checks an expression in the caller's +scope every `interval` ms, handling pending events (QEMU's output) in between, and returns +1 once it is true or 0 at the deadline. `Node` waits for its console ports, the first kernel +line and QEMU's exit with it. + +## Tests + +Every test is a row in a table, with a description, a category (positive, negative, +boundary or corner), its input, and the expected result or error (`testTable` in +`tests/helpers.tcl`): + +```tcl +testTable combine { + {"the first failure is kept" positive {0 1} ok 1} + ... +} {report combine {*}$input} +``` + +| File | Covers | +|---|---| +| `tests/protocol.test` | `protocol wrap` and `protocol line`: markers, stale markers, CRs, bad ids | +| `tests/parse.test` | every `parse` function, `shape` (for `homa-vm-compare`) included | +| `tests/report.test` | every `report` function, `trafficDetail`, `matrixRow` and `diff` included | +| `tests/wait.test` | `waitFor`: at once, before and at the deadline, bad arguments, nested waits | +| `tests/console.test` | `Console`, against a real bash behind a local socket: timeouts, eof, stale markers, CRLF, split writes, 20000-line outputs | +| `tests/table.test` | `Table`: verdicts, the phase lines, `summary.md`, errors, `return`, `break` and `continue` inside phases, `phase-errors.log` | +| `tests/node.test` | `Node`'s phases end to end, against `tests/fakeqemu.tcl`: a fake QEMU that writes a serial log and serves bash on the console port, booting cleanly, with a kernel BUG, with a shell that never answers, never listening, or not starting at all; and `Node phase` | +| `tests/files.test` | `readFile` and `writeFile`: UTF-8 whatever the locale, missing and empty files, bytes that are not UTF-8, appends | +| `tests/batch.test` | `Batch`: exit codes, signals, a command that cannot start, each test's own directory, the job limit, bad limits and duplicate names; `withEnv`, and `HOMA_VM_LOGDIR` left as it was | +| `tests/style.test` | the sources themselves: every proc, method and constructor has `Arguments:` (if it takes any), `Results:` and `Side effects:`; no `eval`; no `namespace export *` | + +Run them, or part of them, with `homa-tcl-test`. Its options go to tcltest, and each row is +a test named `<table>-<n>`: + +```sh +nix run .#homa-tcl-test # all of them, Tcl 8.6 +nix run .#homa-tcl-test -- -file wait.test # one file +nix run .#homa-tcl-test -- -file "parse.test report.test" # several +nix run .#homa-tcl-test -- -match 'trafficDetail-*' # one table +nix run .#homa-tcl-test -- -match waitFor-3 # one row +nix run .#homa-tcl-test -- -verbose bpe -match 'delta-*' # each test's body, pass and error +nix run .#homa-tcl-test -- --tcl9 # the pure files on Tcl 9.0 +nix run .#homa-tcl-lint # the linters below +``` + +The app runs a copy of the tests as the flake sees them (a new file needs `git add -N`). +The `tcl-tests` check runs the lint, the whole suite on Tcl 8.6 and the pure files on Tcl +9.0; the lint is: + +- **nagelfar** checks syntax and variable use. `tests/lintdb.tcl` generates its database + for our own commands from the loaded code (ensemble maps, `info args`, class + definitions), so the lint follows the code. +- **tclint** checks style. + +It also checks that `parse.tcl`'s oops regex (`oopsRe`) is the same as +`constants.oopsRegex`, which the guests' own dmesg checks use. + +Expect in nixpkgs is built for Tcl 8.6, so the drivers run on 8.6. Running the pure tables +on Tcl 9.0 as well keeps `protocol`, `parse` and `report` ready for 9. + +## Why Tcl here + +The bash these drivers replace had to rebuild Expect by hand: `/dev/tcp`, `read -t` +loops, partial-line buffers and numbered markers. In Tcl, Expect does that work. The event +loop (`chan event`, `waitFor`) watches QEMU's output while a command runs, and TclOO keeps +each guest's state in one object instead of a set of bash arrays. The scripts that run +inside the guests (`homa-smoke`, `homa-node-stats`, `homa-qdisc`) stay bash, since the +guests have no Tcl, and are shellchecked as before. diff --git a/nix/microvms/tcl/bin/lifecycle.tcl b/nix/microvms/tcl/bin/lifecycle.tcl new file mode 100644 index 00000000..7fb2f5c5 --- /dev/null +++ b/nix/microvms/tcl/bin/lifecycle.tcl @@ -0,0 +1,93 @@ +# lifecycle.tcl -- +# +# The single-VM lifecycle test (homa-lifecycle): boots one guest and +# drives it over its consoles, phase by phase, each with a timeout and a +# time: +# +# 1 start qemu running, console ports listening +# 2 serial first kernel line in serial.log (ttyS0) +# 2b virtio a root shell answering on hvc0 +# 3 smoke homa-smoke: insmod, echo, GRO/RPS, rmmod +# 4 cycles insmod / netns add+del / rmmod, moduleCycles times +# 5 dmesg no BUG/Oops/WARNING/KASAN/UBSAN in dmesg +# 6 shutdown poweroff over hvc0 +# 7 exit qemu exits +# 8 serial-log no oops anywhere in serial.log +# +# lifecycle.tcl config.json +# +# config.json (written by host/lifecycle.nix) holds the label, the +# timeouts, moduleCycles and the guest's runner and console ports. Logs +# go to $HOMA_VM_LOGDIR (default: a new temporary directory), which is +# also QEMU's working directory. Exits 1 if any phase fails. + +package require homa::lc +package require json +namespace import ::homa::lc::parse + +# main -- +# +# Runs the phases. +# +# Arguments: +# configFile the path of config.json. +# +# Results: +# The exit code: 0 if every phase passed, otherwise 1. +# +# Side effects: +# Changes to the log directory, boots and stops the guest, sends the +# phases' commands to it, writes the phase logs and summary.md, and +# prints each phase and the verdict. + +proc main {configFile} { + set config [json::json2dict [homa::lc::readFile $configFile]] + set logdir [homa::lc::logdir] + cd $logdir + puts "lifecycle[dict get $config label]: logs in $logdir" + + set table [homa::lc::Table new] + set vm [homa::lc::Node new $table . [dict get $config vm] [dict get $config timeouts]] + try { + $vm start 1 + $vm serial 2 + $vm virtio 2b + $vm phase 3 smoke { + set result [$vm run smoke.log [dict get $config timeouts smoke] {homa-smoke 2>&1}] + set text [dict get $result out] + foreach line [lsearch -all -inline [split $text \n] *HOMA_SMOKE_*] { + puts " $line" + } + list [dict get $result rc] [parse smokeDetail $text [dict get $result rc]] + } + # One line (a multi-line command would bring the shell's continuation + # prompts), in a subshell, so a failure ends the loop but not the + # shell. + set count [dict get $config moduleCycles] + set steps { + {insmod "$HOMA_KO"} + {ip netns add homa-lc} + {ip netns exec homa-lc ip link set lo up} + {ip netns del homa-lc} + {rmmod homa} + {echo cycle $i ok} + } + $vm phase 4 cycles { + set loop [format {(set -e; for i in $(seq %d); do %s; done) 2>&1} \ + $count [join $steps "; "]] + set result [$vm run cycles.log [dict get $config timeouts cycles] $loop] + set done [parse countLines {^cycle .* ok$} [dict get $result out]] + list [dict get $result rc] "$done of $count" + } + $vm dmesg 5 + $vm shutdown 6 + $vm waitExit 7 + $vm serialLog 8 + return [$table verdict] + } finally { + $vm destroy + $table destroy + } +} + +exit [main {*}$argv] diff --git a/nix/microvms/tcl/bin/pair-lifecycle.tcl b/nix/microvms/tcl/bin/pair-lifecycle.tcl new file mode 100644 index 00000000..509dfc6f --- /dev/null +++ b/nix/microvms/tcl/bin/pair-lifecycle.tcl @@ -0,0 +1,509 @@ +# pair-lifecycle.tcl -- +# +# The pair lifecycle test (homa-pair-lifecycle): two guests, node0 the +# server and node1 the client, on a private eth1 link, driven from the +# host over each one's hvc0 with every phase timed. Neither guest runs a +# test of its own. +# +# 1-3 start, serial, virtio each node +# 4 insmod each node +# 5 link node1 pings node0 over eth1 +# 6 servers demo_server1 and cp_node servers on node0 +# 7 traffic baseline from node1: echo (six RPCs, 100 B .. 1 MB), +# cp_node Homa, cp_node TCP; counter deltas +# 8.i netem <name> tc netem on eth1 of both nodes, then the +# traffic <name> same traffic. Each node's measured drop +# rate must match the configured loss, and +# with loss, Homa must have sent RESENDs +# 9 netem clear, traffic recovered +# 10 qdisc homa Homa's qdisc on eth1 of both nodes, then +# traffic through it (TCP must pass it too) +# 11 qdisc paced net.homa.link_mbps lowered so the pacer +# must defer packets; traffic +# 12 qdisc off link_mbps restored, default qdisc back +# 13 stop servers +# 14 rmmod each node +# 15-18 dmesg, shutdown, exit, serial-log each node +# +# pair-lifecycle.tcl config.json +# +# config.json is written by host/pair-lifecycle.nix. Logs go to +# $HOMA_VM_LOGDIR (default: a new temporary directory): node0/ and node1/ +# (each its QEMU's working directory, with serial.log), and +# traffic/<name>/ per traffic run. Exits 1 if any phase fails. + +package require homa::lc +package require json +namespace import ::homa::lc::parse ::homa::lc::report + +# Pair -- +# +# One pair lifecycle run: the two guests, the phase table, and the +# phases that span both guests. +# +# set pair [Pair new $configFile] +# exit [$pair run] + +oo::class create Pair { + # config the parsed config.json. + # table the run's homa::lc::Table. + # nodes node name -> homa::lc::Node, for node0 and node1. + # timeout seconds for an ordinary command. + variable config table nodes timeout + + # constructor -- + # + # Reads configFile, moves to the log directory and creates the nodes + # (without starting them). + # + # Arguments: + # configFile the path of config.json. + # + # Results: + # None. + # + # Side effects: + # Changes the current directory to the log directory and prints where + # the logs are. + + constructor {configFile} { + set config [json::json2dict [homa::lc::readFile $configFile]] + set timeout [dict get $config timeouts command] + set logdir [homa::lc::logdir] + cd $logdir + puts "pair lifecycle[dict get $config label]: logs in $logdir" + set table [homa::lc::Table new] + set nodes [dict create] + foreach key {node0 node1} { + dict set nodes $key [homa::lc::Node new $table $key [dict get $config $key] \ + [dict get $config timeouts] " $key"] + } + } + + # destructor -- + # + # Stops both guests. + + destructor { + dict for {key node} $nodes { + $node destroy + } + $table destroy + } + + # run -- + # + # Runs every phase. + # + # Results: + # The exit code: 0 if every phase passed, otherwise 1. + # + # Side effects: + # Boots, drives and stops both guests, records each phase in the + # table, writes the logs and summary.md, and prints each phase, the + # HOMA_PAIR_* lines and the verdict. + + method run {} { + foreach step {start serial virtio} id {1 2 3} { + dict for {key guest} $nodes { + $guest $step $id + } + } + + my EachNode node 4 insmod { + set result [$node run insmod.log $timeout {insmod "$HOMA_KO" 2>&1}] + list [dict get $result rc] [parse firstLine [dict get $result out]] + } + + set peer [dict get $config peer] + set client [my Node node1] + set server [my Node node0] + $client phase 5 link { + set result [$client run ping.log $timeout "for i in \$(seq 20); do ping -c 1\ + -W 1 $peer > /dev/null && break; done; ping -q -c 3 -W 2 $peer"] + set summary [lsearch -all -inline -regexp [split [dict get $result out] \n] \ + {packet loss|rtt}] + list [dict get $result rc] [join $summary " "] + } + + # Each server in a subshell, so the console's shell has no jobs to + # report. + set homaPort [dict get $config cpNode homaPort] + $server phase 6 servers { + set daemons [list \ + "demo_server1 > /tmp/demo_server1.log" \ + "cp_node server --protocol homa --first-port $homaPort\ + > /tmp/cp_node_homa.log" \ + "cp_node server --protocol tcp > /tmp/cp_node_tcp.log"] + set start [join [lmap cmd $daemons {string cat "($cmd 2>&1 < /dev/null &)"}] "; "] + set result [$server run servers.log $timeout "$start; sleep 1; echo\ + demo_server1=\$(pgrep -cx demo_server1) cp_node=\$(pgrep -cx cp_node)"] + set running [parse lastLine [dict get $result out]] + list [expr {$running eq "demo_server1=1 cp_node=2" ? [dict get $result rc] : 1}] \ + $running + } + + my Traffic 7 baseline + set i 0 + foreach impairment [dict get $config netem phases] { + lassign [lmap key {name args loss} {dict get $impairment $key}] name netemArgs loss + incr i + my Netem 8.$i $name $netemArgs + my Traffic 8.$i $name $loss [expr {$loss == 0 ? "" : "packets_sent_RESEND"}] + } + my OnBoth 9 "netem clear" netem-clear.log \ + "tc qdisc del dev eth1 root && ! tc qdisc show dev eth1 | grep -q netem" \ + "default qdisc on eth1" + my Traffic 9 recovered + + # The pacer works from the link speed. virtio-net reports none + # (SPEED_UNKNOWN: round-4 candidate 5), so phase 11 sets + # net.homa.link_mbps, which the qdisc then uses, low enough that the + # pacer has to defer packets. + my OnBoth 10 "qdisc homa" qdisc-on.log [join { + {sysctl -n net.homa.link_mbps > /tmp/link_mbps} + {homa-qdisc on > /tmp/qdisc.out && head -n 1 /tmp/qdisc.out\ + && grep '^qdisc homa' /tmp/qdisc.out} + } "; "] + my Traffic 10 homa-qdisc "" qdisc_tcp_packets + my OnBoth 11 "qdisc paced" qdisc-paced.log \ + "sysctl -w net.homa.link_mbps=[dict get $config pacedMbps]" + my Traffic 11 homa-qdisc-paced "" pacer_homa_packets + my OnBoth 12 "qdisc off" qdisc-off.log [join { + {sysctl -w net.homa.link_mbps=$(cat /tmp/link_mbps)} + {homa-qdisc off} + {! tc qdisc show dev eth1 | grep -q '^qdisc homa'} + } " && "] + + $server phase 13 "stop servers" { + set result [$server run servers.log $timeout [join { + {pkill -x demo_server1} {pkill -x cp_node} {sleep 1} + {! pgrep -x 'demo_server1|cp_node'} + } "; "] a] + list [dict get $result rc] "" + } + + my EachNode node 14 rmmod { + set result [$node run rmmod.log $timeout {rmmod homa 2>&1}] + list [dict get $result rc] [parse firstLine [dict get $result out]] + } + foreach step {dmesg shutdown waitExit serialLog} id {15 16 17 18} { + dict for {key guest} $nodes { + $guest $step $id + } + } + return [$table verdict] + } + + # ---- Phases on both nodes -------------------------------------------------- + + # Traffic -- + # + # One traffic run from node1 to node0 (see Measure), judged and + # described by report trafficDetail. + # + # Arguments: + # id, name the phase; it is named "traffic <name>". + # loss netem's configured loss in percent, or "" without + # netem. With netem, each node's measured drop rate is + # checked against it. + # must a counter that must have moved (summed over both + # nodes), or "": packets_sent_RESEND under loss, the + # qdisc and pacer counters with Homa's qdisc. + # + # Results: + # The phase's rc, or skip. + # + # Side effects: + # Records the phase; runs the traffic on both guests; writes + # traffic/<name>/ (logs and deltas); prints the counter deltas, and + # for the baseline, the HOMA_PAIR_* lines. + + method Traffic {id name {loss {}} {must {}}} { + my BothPhase $id "traffic $name" { + set dir [file join [pwd] traffic $name] + file mkdir $dir + set measured [my Measure $dir $loss] + set deltaLines [lmap {key counters} [dict get $measured deltas] { + string cat "$key: " [report deltaLine $counters] + }] + homa::lc::writeFile [file join $dir deltas] [join $deltaLines \n] + set outcome [report trafficDetail $measured [dict get $config echoRpcs] $must] + + if {$name eq "baseline"} { + # The lines homa-vm-matrix puts in its table. + puts "HOMA_PAIR_ECHO responses=[dict get $measured echoes]\ + (want [dict get $config echoRpcs])" + puts "HOMA_PAIR_CPNODE_homa [dict get $measured homa]" + puts "HOMA_PAIR_CPNODE_tcp [dict get $measured tcp]" + } + foreach line $deltaLines { + puts " $line" + } + set rc [expr {[llength [dict get $outcome failed]] ? 1 : [dict get $measured rc]}] + list $rc [dict get $outcome detail] + } + } + + # Netem -- + # + # Replaces eth1's root qdisc with netem on both nodes, so its counters + # start from zero, and checks that the limit took. The detail is + # node1's netem settings. + # + # Arguments: + # id, name the phase; it is named "netem <name>". + # netemArgs the arguments to tc's netem qdisc. + # + # Results: + # The phase's rc, or skip. + # + # Side effects: + # Replaces eth1's root qdisc on both guests, writes netem-<name>.log + # in each node's directory, and records the phase. + + method Netem {id name netemArgs} { + set limit [dict get $config netem limit] + my BothPhase $id "netem $name" { + set rc 0 + set outs [dict create] + dict for {key node} $nodes { + set result [$node run netem-$name.log $timeout [join [list \ + {tc qdisc del dev eth1 root 2> /dev/null} \ + "tc qdisc add dev eth1 root netem $netemArgs && tc qdisc show dev eth1"] "; "]] + dict set outs $key [dict get $result out] + if {[dict get $result rc] || ![parse netemLimitOk [dict get $result out] $limit]} { + set rc 1 + } + } + list $rc [parse netemDetail [dict get $outs node1]] + } + } + + # OnBoth -- + # + # Runs one command on both nodes as a single phase. The detail is + # given, or else node1's output on one line. + # + # Arguments: + # id, name the phase. + # log the log file's name in each node's directory. + # cmd the command. + # detail the phase's detail, or "" for node1's output. + # + # Results: + # The phase's rc, or skip. + # + # Side effects: + # Runs cmd on both guests, writes log on each, and records the phase. + + method OnBoth {id name log cmd {detail {}}} { + my BothPhase $id $name { + set rc 0 + set outs [dict create] + dict for {key node} $nodes { + set result [$node run $log $timeout $cmd] + dict set outs $key [dict get $result out] + set rc [report combine $rc [dict get $result rc]] + } + if {$detail eq ""} { + set detail [regsub -all { +} [string map {\n " "} [dict get $outs node1]] " "] + } + list $rc $detail + } + } + + # ---- Internals --------------------------------------------------------------- + + # Measure -- + # + # Runs the traffic from node1 to node0 (echo, then cp_node over Homa + # and over TCP) between two counter snapshots of each node, and with + # netem, reads each node's drops. Logs go to dir. + # + # Arguments: + # dir the run's log directory. + # loss netem's configured loss in percent, or "" without netem. + # + # Results: + # The measured dictionary report trafficDetail takes, plus rc: the + # snapshots' combined exit status. + # + # Side effects: + # Runs the traffic and counter reads on the guests and writes their + # logs to dir. + + method Measure {dir loss} { + set client [my Node node1] + lassign [my Snapshot $dir stats0] status0 before + + set echoTimeout [dict get $config timeouts echo] + set echo "timeout [expr {$echoTimeout - 10}] demo_client1 [dict get $config peer] 4000\ + > /tmp/echo.out 2>&1 < /dev/null; grep -c '^Received response' /tmp/echo.out" + set echoCount [parse lastLine [dict get \ + [$client run [file join $dir echo.log] $echoTimeout $echo] out]] + foreach proto {homa tcp} { + set stats($proto) [parse lastLine [dict get \ + [$client run [file join $dir $proto.log] $timeout [my CpNode $proto]] out]] + } + + lassign [my Snapshot $dir stats1] status1 after + set moved [dict map {key counters} $after { + parse delta [dict get $before $key] $counters + }] + set drops [dict create] + if {$loss ne ""} { + dict for {key node} $nodes { + dict set drops $key [parse netemCheck $key [dict get [$node run \ + [file join $dir $key-tc] $timeout "tc -s qdisc show dev eth1"] out] $loss] + } + } + return [dict create rc [report combine $status0 $status1] echoes $echoCount \ + homa $stats(homa) tcp $stats(tcp) deltas $moved checks $drops] + } + + # Snapshot -- + # + # Reads Homa's counters on each node, logging them to + # dir/<node>-<name>. + # + # Arguments: + # dir the log directory. + # name the snapshot's name, e.g. stats0. + # + # Results: + # A list: the combined exit status, and node name -> counters. + # + # Side effects: + # Runs homa-node-stats on both guests and writes dir/<node>-<name>. + + method Snapshot {dir name} { + set status 0 + set counters [dict create] + dict for {key node} $nodes { + set result [$node run [file join $dir $key-$name] $timeout homa-node-stats] + set status [report combine $status [dict get $result rc]] + dict set counters $key [parse snap [dict get $result out] [dict get $config metrics]] + } + return [list $status $counters] + } + + # CpNode -- + # + # Builds the command for one cp_node client run over proto, which + # prints cp_node's last stats line. Output goes to a file, grepped + # afterwards: under the console shell's job control, timeout's SIGTERM + # also reaches the rest of a pipeline, losing its output. + # + # Arguments: + # proto homa or tcp. + # + # Results: + # The command. + # + # Side effects: + # None. + + method CpNode {proto} { + lassign [lmap key {workload gbps seconds homaPort} {dict get $config cpNode $key}] \ + workload gbps seconds homaPort + set options [list --protocol $proto --workload $workload --gbps $gbps \ + --first-server 0 --server-nodes 1 --id 1] + if {$proto eq "homa"} { + lappend options --first-port $homaPort + } + set out /tmp/cp_node_$proto.out + return "timeout $seconds cp_node client [join $options] > $out 2>&1 < /dev/null;\ + grep ' clients: ' $out | tail -n 1" + } + + # Node -- + # + # Looks up one of the nodes. + # + # Arguments: + # name node0 or node1. + # + # Results: + # The homa::lc::Node called name. + # + # Side effects: + # None. + + method Node {name} { + return [dict get $nodes $name] + } + + # BothPhase -- + # + # A phase that needs both nodes' shells: runs script as Table phase + # does, in the caller's scope, or SKIPs it naming which shell is gone. + # + # Arguments: + # id, name the phase. + # script the phase script; it returns {rc detail}. + # + # Results: + # The phase's rc, or skip. + # + # Side effects: + # Runs script and records the phase (or the SKIP) in the table. + + method BothPhase {id name script} { + set up [lmap key {node0 node1} {[my Node $key] shell}] + if {$up ne "1 1"} { + $table skip $id $name [format "no shell on hvc0 (node0 %d, node1 %d)" {*}$up] + return skip + } + tailcall $table phase $id $name $script + } + + # EachNode -- + # + # Runs a phase on each node, as "<name> <node>", like foreach: script + # runs in the caller's scope with varName set to the homa::lc::Node, + # and a node without a shell SKIPs it (see homa::lc::Node phase). + # + # Arguments: + # varName the caller's variable set to each node. + # id, name the phase; on each node it is named "<name> <node>". + # script the phase script; it returns {rc detail}. + # + # Results: + # None. + # + # Side effects: + # Sets varName in the caller, runs script, and records a phase (or a + # SKIP) per node in the table. + + method EachNode {varName id name script} { + upvar 1 $varName node + dict for {key node} $nodes { + uplevel 1 [list $node phase $id "$name $key" $script] + } + } +} + +# main -- +# +# Runs the pair lifecycle. +# +# Arguments: +# configFile the path of config.json. +# +# Results: +# The exit code: 0 if every phase passed, otherwise 1. +# +# Side effects: +# Those of Pair run; both guests are stopped at the end. + +proc main {configFile} { + set pair [Pair new $configFile] + try { + return [$pair run] + } finally { + $pair destroy + } +} + +exit [main {*}$argv] diff --git a/nix/microvms/tcl/bin/vm-compare.tcl b/nix/microvms/tcl/bin/vm-compare.tcl new file mode 100644 index 00000000..7d4b757f --- /dev/null +++ b/nix/microvms/tcl/bin/vm-compare.tcl @@ -0,0 +1,94 @@ +# vm-compare.tcl -- +# +# homa-vm-compare: compares two homa-vm-matrix runs, test by test, by +# what two good runs share: each phase's id, name and verdict, the +# HOMA_* lines with their numbers masked, and the VERDICT lines (see +# "parse shape"). Times, details and counters are ignored. Use it after +# changing a driver, against the logs of a run from before the change: +# +# vm-compare.tcl old-logdir new-logdir +# +# Each log directory is a matrix run's $HOMA_VM_LOGDIR, holding one +# <test>.out per test. Prints each test's changes, "-" for a line only +# in old and "+" for one only in new. Exits 0 if every test is the same, +# 1 if any differs or is missing from one side, 2 on a usage error. + +package require homa::lc +namespace import ::homa::lc::parse ::homa::lc::report + +# compare -- +# +# Compares the runs. +# +# Arguments: +# old the old run's log directory. +# new the new run's log directory. +# +# Results: +# The exit code: 0 if every test is the same, 1 if not, 2 if a log +# directory is missing or holds no <test>.out files. +# +# Side effects: +# Reads the <test>.out files and prints each test's result and the +# verdict. + +proc compare {old new} { + set dirs [list $old $new] + foreach dir $dirs { + if {![file isdirectory $dir]} { + puts stderr "homa-vm-compare: $dir: not a directory" + return 2 + } + } + set names {} + foreach dir $dirs { + foreach path [glob -nocomplain -directory $dir *.out] { + lappend names [file rootname [file tail $path]] + } + } + set names [lsort -unique $names] + if {![llength $names]} { + puts stderr "homa-vm-compare: no <test>.out files in [join $dirs { or }]" + return 2 + } + + set differ 0 + foreach name $names { + set shapes {} + foreach dir $dirs { + set path [file join $dir $name.out] + if {[file exists $path]} { + lappend shapes [parse shape [homa::lc::readFile $path]] + } + } + if {[llength $shapes] < 2} { + set side [expr {[file exists [file join $old $name.out]] ? "old" : "new"}] + puts "$name: only in $side" + incr differ + continue + } + set changes [report diff {*}$shapes] + if {![llength $changes]} { + puts "$name: same ([llength [lindex $shapes 1]] lines)" + continue + } + puts "$name: differs" + foreach change $changes { + puts " $change" + } + incr differ + } + set count [llength $names] + if {$differ} { + puts "VERDICT: FAIL ($differ of $count differ)" + return 1 + } + puts "VERDICT: PASS ($count of $count the same)" + return 0 +} + +if {[llength $argv] != 2} { + puts stderr "usage: homa-vm-compare old-logdir new-logdir" + exit 2 +} +exit [compare {*}$argv] diff --git a/nix/microvms/tcl/bin/vm-matrix.tcl b/nix/microvms/tcl/bin/vm-matrix.tcl new file mode 100644 index 00000000..a022a7f0 --- /dev/null +++ b/nix/microvms/tcl/bin/vm-matrix.tcl @@ -0,0 +1,83 @@ +# vm-matrix.tcl -- +# +# homa-vm-matrix: runs every VM test (each kernel's smoke test, +# lifecycle and pair lifecycle, and each server/client pair) in +# parallel, then prints a summary table, also written to summary.md: +# +# | Test | Kernel(s) | Verdict | Time (s) | Echo | GRO calls | ... +# +# vm-matrix.tcl config.json +# +# config.json (written by host/matrix.nix) lists the tests, each +# {name kernels exe}, and gives echoRpcs (how many echo RPCs each test +# sends). Each test runs in its own directory, $HOMA_VM_LOGDIR/<name>/ +# (the tests share nothing on the host), with its output in <name>.out. +# HOMA_VM_JOBS caps how many run at once (default: all). Exits 1 if any +# test fails. + +package require homa::lc +package require json +namespace import ::homa::lc::report + +# main -- +# +# Runs the tests. +# +# Arguments: +# configFile the path of config.json. +# +# Results: +# The exit code: 0 if every test passed, otherwise 1. +# +# Side effects: +# Runs each test as a process in its own directory under the log +# directory, writes summary.md there, and prints the table and verdict. + +proc main {configFile} { + global env + set config [json::json2dict [homa::lc::readFile $configFile]] + set tests [dict get $config tests] + set count [llength $tests] + set logdir [homa::lc::logdir] + set jobs $count + if {[info exists env(HOMA_VM_JOBS)] && $env(HOMA_VM_JOBS) ne ""} { + set jobs $env(HOMA_VM_JOBS) + } + puts "VM matrix: $count tests, up to $jobs at once; logs in $logdir" + + set start [clock seconds] + set batch [homa::lc::Batch new $logdir $jobs] + try { + set results [$batch run $tests] + } finally { + $batch destroy + } + + set rows [list \ + "| Test | Kernel(s) | Verdict | Time (s) | Echo | GRO calls | Homa Kops/s\ + | Homa P50 / P99 (us) | TCP Kops/s | TCP P50 / P99 (us) |" \ + "|---|---|---|---|---|---|---|---|---|---|"] + set failed 0 + foreach test $tests { + set name [dict get $test name] + lassign [dict get $results $name] - secs + set outcome [report matrixRow $name [dict get $test kernels] $secs \ + [dict get $config echoRpcs] [homa::lc::readFile [file join $logdir $name.out]]] + if {[dict get $outcome verdict] ne "PASS"} { + incr failed + } + lappend rows [dict get $outcome row] + } + set summary [file join $logdir summary.md] + homa::lc::writeFile $summary [join $rows \n] + puts "\n[join $rows \n]\n" + puts "summary: $summary ([expr {[clock seconds] - $start}] s)" + if {$failed} { + puts "VERDICT: FAIL ($failed of $count)" + return 1 + } + puts "VERDICT: PASS ($count of $count)" + return 0 +} + +exit [main {*}$argv] diff --git a/nix/microvms/tcl/console.tcl b/nix/microvms/tcl/console.tcl new file mode 100644 index 00000000..5ce7a9ea --- /dev/null +++ b/nix/microvms/tcl/console.tcl @@ -0,0 +1,180 @@ +# console.tcl -- +# +# homa::lc::Console runs shell commands over a guest's console: a TCP +# connection to QEMU's virtio console (hvc0), where the guest runs a root +# shell. Expect does the reading; this class adds the end-marker protocol +# (protocol.tcl), so each command's exit status comes back with its output. + +package require Tcl 8.6- +package require Expect +package require homa::lc::protocol + +namespace eval ::homa::lc { + namespace export Console +} + +# homa::lc::Console -- +# +# One guest console. +# +# set console [homa::lc::Console new [socket 127.0.0.1 $port]] +# set result [$console run "uname -r" 30] +# dict get $result rc ;# the command's exit status, 124, or 125 +# dict get $result out ;# its output, without the final newline +# +# A command that times out leaves its output and end marker to arrive +# later; the numbered markers let the next command skip them. Once the +# connection closes, every command returns 125 at once. + +oo::class create ::homa::lc::Console { + # spawnId Expect's id for the connection, or "" once it has closed. + # lastId the number of the last command sent. + variable spawnId lastId + + # constructor -- + # + # Takes over chan, a connected socket (or any readable and writable + # channel); it is closed when the console closes. + # + # Arguments: + # chan the channel. + # matchMax bytes Expect buffers before handing a partial line over + # (see run). Tests lower it to exercise long lines. + # + # Results: + # None. + # + # Side effects: + # Expect spawns on chan, and log_user is turned off, so Expect no + # longer copies what it reads to stdout (console output goes to the + # phase logs instead). + + constructor {chan {matchMax 65536}} { + # Methods run in the object's namespace; find protocol. + namespace path ::homa::lc + # Expect copies everything it reads to stdout unless told not to; + # console output goes to the phase logs instead. + log_user 0 + set lastId 0 + spawn -noecho -open $chan + set spawnId $spawn_id + match_max -i $spawnId $matchMax + } + + destructor { + my Close + } + + # alive -- + # + # Whether the connection is open. + # + # Results: + # 1 while the connection is open, otherwise 0. + # + # Side effects: + # None. + + method alive {} { + return [expr {$spawnId ne ""}] + } + + # send -- + # + # Writes one line to the guest without waiting for anything, e.g. + # "poweroff". + # + # Arguments: + # line the line, without its newline. + # + # Results: + # 1 if it was written, 0 if the console has closed. + # + # Side effects: + # The line is sent to the guest. A failed write closes the console. + + method send {line} { + if {$spawnId eq ""} { + return 0 + } + try { + exp_send -i $spawnId -- "$line\n" + } on error {} { + my Close + return 0 + } + return 1 + } + + # run -- + # + # Runs one shell command and waits for its end marker. + # + # Arguments: + # cmd the command, as the guest's shell will read it. + # timeout seconds to wait for the whole command. + # + # Results: + # A dictionary: + # rc the command's exit status; 124 if it did not finish in time; + # 125 if the console closed (the guest powered off or crashed). + # out what the command printed, one line per line, with carriage + # returns and every end marker removed. + # + # Side effects: + # The wrapped command is sent to the guest, and output up to its end + # marker is consumed. End of file closes the console. + + method run {cmd timeout} { + set id [incr lastId] + if {![my send [protocol wrap $cmd $id]]} { + return [dict create rc 125 out {}] + } + set status 124 + set lines {} + set partial {} + # One line per match, so the buffer never holds more than a line + # (and Expect's match_max limit applies to single lines). A longer + # line fills the buffer; full_buffer keeps what it has so far. The + # whole match is used, not a subexpression: Expect sets an empty + # subexpression to the rest of the buffer. + expect -i $spawnId -timeout $timeout -re {[^\n]*\n} { + set line $partial[string range $expect_out(0,string) 0 end-1] + set partial {} + lassign [protocol line $id $line] keep text done rc + if {$keep} { + lappend lines $text + } + if {!$done} { + exp_continue -continue_timer + } + set status $rc + } full_buffer { + append partial $expect_out(buffer) + exp_continue -continue_timer + } eof { + my Close + set status 125 + } + return [dict create rc $status out [join $lines \n]] + } + + # Close -- + # + # Closes the connection (and the channel Expect took over), once. + # + # Results: + # None. + # + # Side effects: + # The connection is closed; later commands return 125 at once. + + method Close {} { + if {$spawnId ne ""} { + catch {close -i $spawnId} + set spawnId {} + } + } +} + +package provide homa::lc::console 1.0 diff --git a/nix/microvms/tcl/lc.tcl b/nix/microvms/tcl/lc.tcl new file mode 100644 index 00000000..5c13b498 --- /dev/null +++ b/nix/microvms/tcl/lc.tcl @@ -0,0 +1,1019 @@ +# lc.tcl -- +# +# The lifecycle drivers' building blocks: +# +# homa::lc::Table the phase table: times and records each phase, prints +# its line, and writes summary.md with the VERDICT. +# homa::lc::Node one guest: its QEMU process and consoles, and the +# phases every lifecycle test runs on each guest (start, +# serial, virtio, dmesg, shutdown, exit, serial-log). +# homa::lc::Batch runs whole test commands in parallel, each in its own +# directory (homa-vm-matrix). +# +# A failed phase does not stop a run, but phases that need a guest's +# shell are SKIPped once it is gone. + +package require Tcl 8.6- +package require homa::lc::parse +package require homa::lc::report +package require homa::lc::console + +namespace eval ::homa::lc { + namespace export Table Node Batch logdir waitFor readFile writeFile utf8 withEnv +} + +# homa::lc::waitFor -- +# +# Waits until condition is true or ms milliseconds have passed, checking +# it every interval milliseconds and handling the events that came in +# meanwhile (QEMU's output, for one), which a plain "after" would not. +# There is no flag to share: a wait started by an event handler during +# another one just runs to its own deadline, and the outer one then +# goes on to its own. +# +# Arguments: +# ms how long to wait, at least 0. +# interval how often to check, at least 1. +# condition an expression, evaluated in the caller's scope, as +# while does. It is checked once more at the deadline. +# +# Results: +# 1 if the condition became true, 0 if the time ran out. +# +# Side effects: +# Handles pending events (update) while it waits. + +proc ::homa::lc::waitFor {ms interval condition} { + if {![string is entier -strict $ms] || $ms < 0} { + return -code error "bad wait \"$ms\"" + } + if {![string is entier -strict $interval] || $interval < 1} { + return -code error "bad interval \"$interval\"" + } + set deadline [expr {[clock milliseconds] + $ms}] + while {![uplevel 1 [list expr $condition]]} { + set left [expr {$deadline - [clock milliseconds]}] + if {$left <= 0} { + return 0 + } + after [expr {min($interval, $left)}] + update + } + return 1 +} + +# homa::lc::logdir -- +# +# Returns the run's log directory: $HOMA_VM_LOGDIR, created if need be, +# or a new temporary directory. +# +# Results: +# The log directory's path. +# +# Side effects: +# Creates $HOMA_VM_LOGDIR if missing, or else a new temporary directory. + +proc ::homa::lc::logdir {} { + global env + if {[info exists env(HOMA_VM_LOGDIR)] && $env(HOMA_VM_LOGDIR) ne ""} { + file mkdir $env(HOMA_VM_LOGDIR) + return [file normalize $env(HOMA_VM_LOGDIR)] + } + return [exec mktemp -d] +} + +# homa::lc::readFile -- +# +# Returns a file's contents, read as UTF-8, or an empty string if it does +# not exist (yet). Bytes that are not UTF-8 (a serial log can hold line +# noise) are kept as single characters rather than raising an error. +# +# Arguments: +# path the file. +# +# Results: +# The file's contents, or an empty string if it does not exist. +# +# Side effects: +# None. + +proc ::homa::lc::readFile {path} { + try { + set chan [open $path r] + } trap {POSIX ENOENT} {} { + return {} + } + try { + utf8 $chan + return [read $chan] + } finally { + close $chan + } +} + +# homa::lc::writeFile -- +# +# Writes text, as UTF-8, plus a final newline unless it is empty, to a +# file. +# +# Arguments: +# path the file. +# text what to write. +# access w to replace the file, a to append to it. +# +# Results: +# None. +# +# Side effects: +# Creates, replaces or appends to the file. + +proc ::homa::lc::writeFile {path text {access w}} { + set chan [open $path $access] + try { + utf8 $chan + puts -nonewline $chan [expr {$text eq "" ? "" : "$text\n"}] + } finally { + close $chan + } +} + +# homa::lc::utf8 -- +# +# Makes a channel read and write UTF-8 whatever the locale, and, on +# Tcl 9, replace bad bytes rather than fail on them (8.6 always does). +# +# Arguments: +# chan the channel. +# +# Results: +# None. +# +# Side effects: +# Sets the channel's encoding (and, on Tcl 9, its profile). + +proc ::homa::lc::utf8 {chan} { + chan configure $chan -encoding utf-8 + if {[package vsatisfies [package provide Tcl] 9-]} { + chan configure $chan -profile replace + } +} + +# homa::lc::withEnv -- +# +# Runs script in the caller's scope with env(name) set to value, then +# puts env(name) back as it was, set or unset, even if script fails. +# Tcl 8.6's "open |" has no option for a child's environment, so it is +# set around the open. +# +# Arguments: +# name the environment variable's name. +# value its value while script runs. +# script the script to run. +# +# Results: +# The script's result. +# +# Side effects: +# Sets env(name) while script runs, and whatever script does. + +proc ::homa::lc::withEnv {name value script} { + global env + set had [info exists env($name)] + if {$had} { + set old $env($name) + } + set env($name) $value + try { + return [uplevel 1 $script] + } finally { + if {$had} { + set env($name) $old + } else { + unset -nocomplain env($name) + } + } +} + +# homa::lc::Batch -- +# +# Runs test commands in parallel, at most limit at a time, each in its +# own directory under logdir with HOMA_VM_LOGDIR set to it, and its +# output in logdir/<name>.out. +# +# set batch [homa::lc::Batch new $logdir 4] +# set results [$batch run {{name a exe /bin/a} {name b exe /bin/b}}] +# dict get $results a ;# {rc secs} +# +# rc is the command's exit status, or the signal's name (e.g. SIGKILL) +# if one ended it. + +oo::class create ::homa::lc::Batch { + # logdir where each test's directory and .out file go. + # limit how many tests may run at once. + # waiting the tests not started yet. + # running how many are running. + # results test name -> {rc secs}, as each finishes. + variable logdir limit waiting running results + + # constructor -- + # + # Makes a batch that runs its tests under dir, jobs at a time. + # + # Arguments: + # dir the log directory for the tests' directories and .out files. + # jobs how many tests may run at once, at least 1. + # + # Results: + # None. + # + # Side effects: + # Sets the object's namespace path to ::homa::lc. + + constructor {dir jobs} { + # Methods run in the object's namespace; find withEnv. + namespace path ::homa::lc + if {![string is entier -strict $jobs] || $jobs < 1} { + return -code error "bad job limit \"$jobs\"" + } + set logdir $dir + set limit $jobs + lassign {{} 0 {}} waiting running results + } + + # run -- + # + # Runs tests, a list of dictionaries each with a name and an exe (a + # command without arguments), and waits for all of them. + # + # Arguments: + # tests a list of dictionaries, each with a name and an exe. + # + # Results: + # A dictionary: each test's name -> {rc secs}. + # + # Side effects: + # Runs the tests; enters the event loop (vwait) until all are done. + + method run {tests} { + set names [lmap test $tests {dict get $test name}] + if {[llength [lsort -unique $names]] != [llength $names]} { + return -code error "test names are not unique: $names" + } + set waiting $tests + set results [dict create] + my Fill + while {[dict size $results] < [llength $tests]} { + vwait [my varname results] + } + return $results + } + + # Fill -- + # + # Starts waiting tests, up to the limit. + # + # Results: + # None. + # + # Side effects: + # Starts processes and removes them from the waiting list. + + method Fill {} { + while {[llength $waiting] && $running < $limit} { + set waiting [lassign $waiting test] + my Start [dict get $test name] [dict get $test exe] + } + } + + # Start -- + # + # Starts one test. microvm.nix puts the QMP socket (<hostname>.sock) + # in the current directory, and every pair's server is node0: hence a + # directory each. + # + # Arguments: + # name the test's name. + # exe the command to run. + # + # Results: + # None. + # + # Side effects: + # Creates the test's directory and .out file and starts its process, + # or records an error result if it cannot start. + + method Start {name exe} { + set dir [file join $logdir $name] + file mkdir $dir + set out [open [file join $logdir $name.out] w] + set here [pwd] + cd $dir + try { + set chan [withEnv HOMA_VM_LOGDIR $dir {open |[list $exe < /dev/null 2>@1] r}] + } on error {message} { + close $out + dict set results $name [list "error: $message" 0] + return + } finally { + cd $here + } + chan configure $chan -blocking 0 -translation binary + chan configure $out -translation binary + chan event $chan readable [namespace code [list my Copy $name $chan $out [clock seconds]]] + incr running + } + + # Copy -- + # + # Copies a test's output to its .out file. At eof, records its exit + # status and time, and starts the next waiting test. + # + # Arguments: + # name the test's name. + # chan the pipe from the test's process. + # out the .out file's channel. + # started when it started, from clock seconds. + # + # Results: + # None. + # + # Side effects: + # Writes to the .out file; at eof closes both channels, prints a done + # line, records the result and starts more tests. + + method Copy {name chan out started} { + puts -nonewline $out [read $chan] + if {![eof $chan]} { + return + } + close $out + chan configure $chan -blocking 1 + set rc 0 + try { + close $chan + } trap CHILDSTATUS {- options} { + set rc [lindex [dict get $options -errorcode] 2] + } trap CHILDKILLED {- options} { + set rc [lindex [dict get $options -errorcode] 2] + } + puts " done: $name (exit $rc)" + incr running -1 + dict set results $name [list $rc [expr {[clock seconds] - $started}]] + my Fill + } +} + +# homa::lc::Table -- +# +# The phase table of one run. +# +# set table [homa::lc::Table new] +# $table phase 3 smoke { +# ... +# list $rc $detail +# } +# exit [$table verdict] + +oo::class create ::homa::lc::Table { + # rows summary.md's rows, one per phase. + # failed how many phases failed. + variable rows failed + + # constructor -- + # + # Makes an empty phase table. + # + # Results: + # None. + # + # Side effects: + # Sets the object's namespace path to ::homa::lc. + + constructor {} { + # Methods run in the object's namespace; find report and writeFile. + namespace path ::homa::lc + set rows {} + set failed 0 + } + + # record -- + # + # Records one phase and prints its line. + # + # Arguments: + # id the phase's number, e.g. 2b or 8.3. + # name its name. + # rc 0 for PASS, skip for SKIP, anything else for FAIL. + # start when it started, from clock milliseconds. + # detail a short note for the table. + # + # Results: + # None. + # + # Side effects: + # Adds a row, counts a FAIL, and prints the phase line. + + method record {id name rc start {detail {}}} { + set result [report verdict $rc] + if {$result eq "FAIL"} { + incr failed + } + set ms [expr {[clock milliseconds] - $start}] + lappend rows [report tableRow $id $name $result $ms $detail] + puts [report phaseLine $id $name $result $ms $detail] + } + + # skip -- + # + # Records a phase that could not run. + # + # Arguments: + # id the phase's number. + # name its name. + # detail why it could not run. + # + # Results: + # None. + # + # Side effects: + # As record: adds a SKIP row and prints its line. + + method skip {id name detail} { + my record $id $name skip [clock milliseconds] $detail + } + + # phase -- + # + # Times script, run in the caller's scope, and records its result: a + # list {rc detail}, either the script's value or what it returns. + # Anything else fails the phase rather than the run: an error (its + # stack goes to phase-errors.log), a break or continue, or a result + # without a valid rc. + # + # Arguments: + # id the phase's number. + # name its name. + # script the phase's script. + # + # Results: + # The phase's rc. + # + # Side effects: + # Whatever script does; records the phase (see record), and appends + # an error's stack to phase-errors.log. + + method phase {id name script} { + set start [clock milliseconds] + try { + uplevel 1 $script + } on ok {result} - on return {result} { + set outcome [my Outcome $result] + } on break {} { + set outcome {1 {error: break in phase script}} + } on continue {} { + set outcome {1 {error: continue in phase script}} + } on error {message options} { + writeFile phase-errors.log "phase $id $name: $message\n[dict get $options -errorinfo]" a + set outcome [list 1 "error: $message"] + } + lassign $outcome rc detail + my record $id $name $rc $start $detail + return $rc + } + + # Outcome -- + # + # Returns a phase script's result as {rc detail}, or {1 "error: ..."} + # if it is not a list with a valid rc. + # + # Arguments: + # result the phase script's result. + # + # Results: + # {rc detail}, or {1 "error: ..."}. + # + # Side effects: + # None. + + method Outcome {result} { + try { + lassign $result rc detail + report verdict $rc + return [list $rc $detail] + } on error {message} { + return [list 1 "error: $message"] + } + } + + # verdict -- + # + # Writes summary.md (in the current directory), prints it and the + # VERDICT line, and returns the exit code for the run. + # + # Results: + # 0 if no phase failed, else 1. + # + # Side effects: + # Writes summary.md and prints the table and the VERDICT line. + + method verdict {} { + set table [join [list \ + "| Phase | Name | Result | Time (ms) | Detail |" \ + "|---|---|---|---|---|" {*}$rows] \n] + writeFile summary.md $table + puts "\n$table\n" + if {$failed} { + puts "VERDICT: FAIL ($failed phases)" + return 1 + } + puts "VERDICT: PASS" + return 0 + } +} + +# homa::lc::Node -- +# +# One guest, built with consolePorts (mkVm.nix): ttyS0 on one host TCP +# port, logged by QEMU to serial.log from the first byte, and a root +# shell on hvc0 (the virtio console) on another. +# +# set node [homa::lc::Node new $table node0 $vm $timeouts " node0"] +# $node start 1 +# ... +# $node phase 4 insmod { +# set result [$node run insmod.log 30 {insmod "$HOMA_KO"}] +# list [dict get $result rc] "" +# } +# +# dir is the guest's log directory, which is also QEMU's working +# directory. vm is a dictionary: runner (its microvm runner), serial and +# virtio (the console ports). timeouts gives seconds per phase: start, +# serial, virtio, command and exit. + +oo::class create ::homa::lc::Node { + # table the run's homa::lc::Table. + # suffix appended to this node's phase names, e.g. " node0". + # dir, timeouts from the constructor. + # runner, serialPort, virtioPort from vm. + # qemu the QEMU pipeline, "" once it has exited. + # qemuLog the channel its output is copied to. + # console the hvc0 homa::lc::Console, "" until connected. + # shell 1 once the shell on hvc0 has answered. + # poweroffSent 1 once "poweroff" has been sent. + variable table suffix dir runner serialPort virtioPort timeouts \ + qemu qemuLog console shell poweroffSent + + # constructor -- + # + # Makes a node; QEMU is not started until start. + # + # Arguments: + # phaseTable the run's homa::lc::Table. + # logDir the guest's log directory. + # vm a dictionary: runner, serial and virtio. + # phaseTimeouts seconds per phase: start, serial, virtio, command, exit. + # nameSuffix appended to this node's phase names. + # + # Results: + # None. + # + # Side effects: + # Sets the object's namespace path to ::homa::lc. + + constructor {phaseTable logDir vm phaseTimeouts {nameSuffix {}}} { + namespace path ::homa::lc + set table $phaseTable + set dir $logDir + set timeouts $phaseTimeouts + set suffix $nameSuffix + lassign [lmap key {runner serial virtio} {dict get $vm $key}] \ + runner serialPort virtioPort + lassign {{} {} {} 0 0} qemu qemuLog console shell poweroffSent + } + + # destructor -- + # + # Kills QEMU if it is still running, and closes the console. + + destructor { + if {$console ne ""} { + $console destroy + } + if {$qemu ne ""} { + catch {exec kill {*}[pid $qemu]} + my Reap + } + } + + # dir -- + # + # Returns the node's log directory. + # + # Results: + # The node's log directory. + # + # Side effects: + # None. + + method dir {} { + return $dir + } + + # shell -- + # + # Returns 1 while the node's shell answers on hvc0. + # + # Results: + # 1 if the shell answers, else 0. + # + # Side effects: + # None. + + method shell {} { + return [expr {$shell && [$console alive]}] + } + + # need -- + # + # Returns 1 if the shell answers; otherwise SKIPs the phase and + # returns 0. + # + # Arguments: + # id the phase's number. + # name its name. + # + # Results: + # 1 if the shell answers, else 0. + # + # Side effects: + # Records a SKIP for the phase if the shell is gone. + + method need {id name} { + if {[my shell]} { + return 1 + } + $table skip $id $name "no shell on hvc0" + return 0 + } + + # phase -- + # + # A phase that needs this node's shell: runs script as Table phase + # does, in the caller's scope, or SKIPs it if the shell is gone. + # + # Arguments: + # id the phase's number. + # name its name. + # script the phase's script. + # + # Results: + # The phase's rc, or skip. + # + # Side effects: + # As Table phase, or records a SKIP. + + method phase {id name script} { + if {![my need $id $name]} { + return skip + } + tailcall $table phase $id $name $script + } + + # run -- + # + # Runs a command in the node's shell (see homa::lc::Console run) and + # writes its output to log, in the node's directory. + # + # Arguments: + # log the log file's name. + # timeout seconds. + # cmd the command. + # access w to replace the log, a to append to it. + # + # Results: + # The Console's result, {rc out}. + # + # Side effects: + # Runs cmd in the guest; writes or appends to the log file. + + method run {log timeout cmd {access w}} { + set result [$console run $cmd $timeout] + writeFile [file join $dir $log] [dict get $result out] $access + return $result + } + + # ---- Phases ---------------------------------------------------------------- + + # start -- + # + # Starts QEMU in the node's directory and waits for its console ports. + # + # Arguments: + # id the phase's number. + # + # Results: + # The phase's rc (see Table phase). + # + # Side effects: + # Records the phase; starts QEMU, its output copied to qemu.log. + + method start {id} { + $table phase $id start$suffix { + file mkdir $dir + set here [pwd] + cd $dir + try { + set qemu [open |[list $runner/bin/microvm-run < /dev/null 2>@1] r] + } finally { + cd $here + } + set qemuLog [open [file join $dir qemu.log] w] + chan configure $qemu -blocking 0 -translation binary + chan configure $qemuLog -translation binary + chan event $qemu readable [namespace code {my Drain}] + set qemuPid [pid $qemu] + set listening 0 + waitFor [my Ms start] 100 {$qemu eq "" || [set listening [my Listening]]} + list [expr {!$listening}] "qemu pid $qemuPid, consoles on $serialPort/$virtioPort" + } + } + + # serial -- + # + # Waits for the first kernel line in serial.log. NixOS boots "quiet", + # so that is any kernel message, not the banner. + # + # Arguments: + # id the phase's number. + # + # Results: + # The phase's rc (see Table phase). + # + # Side effects: + # Records the phase. + + method serial {id} { + $table phase $id serial$suffix { + set first {} + waitFor [my Ms serial] 200 {$qemu eq "" || [set first [my FirstKernelLine]] ne ""} + list [expr {$first eq ""}] "first kernel line at $first" + } + } + + # virtio -- + # + # Connects to hvc0 and probes until the autologin shell answers. The + # probe also makes it a plain shell: no echo, no (coloured) prompt, no + # pager. Probes sent before the shell started are answered late, and + # the console's numbered markers skip those answers. It then waits + # for the boot and udev to finish: a slow (sanitizer) guest answers on hvc0 + # minutes before udev hands eth1 to networkd for its address. + # + # Arguments: + # id the phase's number. + # + # Results: + # The phase's rc (see Table phase). + # + # Side effects: + # Records the phase; opens the hvc0 Console, sets shell, and writes + # virtio.log and uptime.log. + + method virtio {id} { + $table phase $id virtio$suffix { + set up {} + if {![catch {socket 127.0.0.1 $virtioPort} chan]} { + set console [Console new $chan] + # Each probe waits up to 3 s for its answer. + waitFor [my Ms virtio] 1 { + $qemu eq "" || ![$console alive] || [set shell [my Probe]] + } + } + if {$shell} { + set out [dict get [my run uptime.log [dict get $timeouts virtio] \ + {systemctl is-system-running --wait > /dev/null; udevadm settle --timeout=900;\ + echo $(uname -r) $(cut -d' ' -f1 /proc/uptime)}] out] + lassign [parse lastLine $out] kernel uptime + set up "kernel $kernel, guest uptime $uptime s" + } + list [expr {!$shell}] $up + } + } + + # dmesg -- + # + # Fails if the guest's dmesg reports a kernel bug or warning. + # + # Arguments: + # id the phase's number. + # + # Results: + # The phase's rc (see Table phase), or skip. + # + # Side effects: + # Records the phase; writes dmesg.log and prints any oops lines. + + method dmesg {id} { + my phase $id dmesg$suffix { + set result [my run dmesg.log [dict get $timeouts command] dmesg] + set out [dict get $result out] + my Report [lrange [parse oops $out] 0 4] + set rc [expr {[parse oopsCount $out] ? 1 : [dict get $result rc]}] + list $rc "[llength [parse lines $out]] lines" + } + } + + # shutdown -- + # + # Sends poweroff over hvc0, without waiting. + # + # Arguments: + # id the phase's number. + # + # Results: + # The phase's rc (see Table phase), or skip. + # + # Side effects: + # Records the phase; the guest powers off; sets poweroffSent. + + method shutdown {id} { + my phase $id shutdown$suffix { + set poweroffSent [$console send poweroff] + list [expr {!$poweroffSent}] "poweroff sent over hvc0" + } + } + + # waitExit -- + # + # Waits for QEMU to exit, which it must not do before the poweroff. + # + # Arguments: + # id the phase's number. + # + # Results: + # The phase's rc (see Table phase). + # + # Side effects: + # Records the phase. + + method waitExit {id} { + $table phase $id exit$suffix { + waitFor [my Ms exit] 200 {$qemu eq ""} + if {$qemu ne ""} { + list 1 "still running after [dict get $timeouts exit] s" + } elseif {!$poweroffSent} { + list 1 "qemu exited before poweroff (guest crash?)" + } else { + list 0 "after poweroff" + } + } + } + + # serialLog -- + # + # Fails if serial.log reports a kernel bug or warning anywhere, + # including after the shell has gone (late RCU callbacks: STATUS.md + # F8). + # + # Arguments: + # id the phase's number. + # + # Results: + # The phase's rc (see Table phase). + # + # Side effects: + # Records the phase and prints any oops lines. + + method serialLog {id} { + $table phase $id serial-log$suffix { + set text [readFile [file join $dir serial.log]] + my Report [lrange [parse oops $text RIP:] 0 5] + list [expr {[parse oopsCount $text] > 0}] "[llength [parse lines $text]] lines" + } + } + + # ---- Internals --------------------------------------------------------------- + + # Ms -- + # + # Returns timeouts(key), which is in seconds, in milliseconds. + # + # Arguments: + # key a phase name in timeouts. + # + # Results: + # The timeout in milliseconds. + # + # Side effects: + # None. + + method Ms {key} { + return [expr {[dict get $timeouts $key] * 1000}] + } + + # Listening -- + # + # Returns 1 if the virtio console port accepts a connection. + # + # Results: + # 1 if it accepts one, else 0. + # + # Side effects: + # Opens and closes a connection to the port. + + method Listening {} { + if {[catch {socket 127.0.0.1 $virtioPort} probe]} { + return 0 + } + close $probe + return 1 + } + + # FirstKernelLine -- + # + # Returns the timestamp of the first kernel line in serial.log so far, + # or an empty string. + # + # Results: + # The timestamp, or an empty string. + # + # Side effects: + # None. + + method FirstKernelLine {} { + return [parse firstKernelLine [readFile [file join $dir serial.log]]] + } + + # Probe -- + # + # Sends the plain-shell settings over hvc0 and returns 1 if the shell + # answered them within 3 s. + # + # Results: + # 1 if the shell answered, else 0. + # + # Side effects: + # Changes the guest shell's settings; appends to virtio.log. + + method Probe {} { + set plain [join { + {stty -echo} PS1= PROMPT_COMMAND= {export TERM=dumb PAGER=cat SYSTEMD_PAGER=} + } "; "] + return [expr {[dict get [my run virtio.log 3 $plain a] rc] == 0}] + } + + # Report -- + # + # Prints lines of guest output under the phase line, indented. + # + # Arguments: + # lines the lines to print. + # + # Results: + # None. + # + # Side effects: + # Prints to stdout. + + method Report {lines} { + foreach line $lines { + puts " $line" + } + } + + # Drain -- + # + # Copies QEMU's output to qemu.log. At end of file QEMU has exited. + # + # Results: + # None. + # + # Side effects: + # Writes to qemu.log; at end of file reaps QEMU (see Reap). + + method Drain {} { + puts -nonewline $qemuLog [read $qemu] + if {[eof $qemu]} { + my Reap + } + } + + # Reap -- + # + # Reaps QEMU and closes its log. + # + # Results: + # None. + # + # Side effects: + # Closes the QEMU pipe and qemu.log, and sets qemu to "". + + method Reap {} { + chan configure $qemu -blocking 1 + catch {close $qemu} + close $qemuLog + set qemu {} + } +} + +package provide homa::lc 1.0 diff --git a/nix/microvms/tcl/parse.tcl b/nix/microvms/tcl/parse.tcl new file mode 100644 index 00000000..2bc53b96 --- /dev/null +++ b/nix/microvms/tcl/parse.tcl @@ -0,0 +1,520 @@ +# parse.tcl -- +# +# Pure functions used by the lifecycle drivers to read guest output: +# dmesg, tc, Homa's counters, cp_node's stats. report.tcl formats it. +# Nothing here touches a console, a file or the clock, so tests/parse.test +# covers all of it with tables. +# +# Every command is a subcommand of the ensemble homa::lc::parse, e.g. +# "parse field $line P99" after "namespace import homa::lc::parse". + +package require Tcl 8.6- + +namespace eval ::homa::lc { + namespace export parse +} +namespace eval ::homa::lc::parse { + namespace export firstLine lastLine lines countLines firstKernelLine oops \ + smokeDetail snap delta get field positive trafficFailures netemSent \ + dropRate netemCheck netemLimitOk netemDetail oopsCount shape + namespace ensemble create + + # Kernel messages that fail the dmesg and serial-log phases. + variable oopsRe {BUG:|Oops|WARNING:|KASAN|UBSAN|general protection fault|list_add corruption} +} + +# ---- Text ------------------------------------------------------------------------ + +# homa::lc::parse::firstLine -- +# +# Returns the first line of text. +# +# Arguments: +# text any text. +# +# Results: +# The first line, or an empty string if text is empty. +# +# Side effects: +# None. + +proc ::homa::lc::parse::firstLine {text} { + return [lindex [split $text \n] 0] +} + +# homa::lc::parse::lastLine -- +# +# Returns the last non-empty line of text. +# +# Arguments: +# text any text. +# +# Results: +# The last non-empty line, or an empty string if there is none. +# +# Side effects: +# None. + +proc ::homa::lc::parse::lastLine {text} { + return [lindex [lsearch -all -inline -not -exact [split $text \n] {}] end] +} + +# homa::lc::parse::lines -- +# +# Splits text into its lines. Unlike split alone, trailing newlines do not +# start more (empty) lines. +# +# Arguments: +# text any text. +# +# Results: +# A list of the lines. +# +# Side effects: +# None. + +proc ::homa::lc::parse::lines {text} { + return [split [string trimright $text \n] \n] +} + +# homa::lc::parse::countLines -- +# +# Counts the lines of text that match a regular expression. +# +# Arguments: +# re the regular expression. +# text the text to search. +# +# Results: +# How many lines of text match re. +# +# Side effects: +# None. + +proc ::homa::lc::parse::countLines {re text} { + return [llength [lsearch -all -regexp [split $text \n] $re]] +} + +# ---- Guest output -------------------------------------------------------------- + +# homa::lc::parse::firstKernelLine -- +# +# Finds the first kernel message in a serial log. +# +# Arguments: +# text the serial log. +# +# Results: +# The message's timestamp, e.g. "[ 2.179107]", or an empty string if +# the kernel has printed nothing yet. +# +# Side effects: +# None. + +proc ::homa::lc::parse::firstKernelLine {text} { + if {[regexp -line {^\[ *[0-9]+\.[0-9]+\]} $text stamp]} { + return $stamp + } + return {} +} + +# homa::lc::parse::oops -- +# +# Returns the lines of text that report a kernel bug or warning. +# +# Arguments: +# text dmesg or serial console output. +# also a regular expression for more lines to return with them, in +# order (e.g. "RIP:", which names the faulting function). +# +# Results: +# A list of the matching lines, in order. +# +# Side effects: +# None. + +proc ::homa::lc::parse::oops {text {also {}}} { + variable oopsRe + set re [expr {$also eq "" ? $oopsRe : "$oopsRe|$also"}] + return [lsearch -all -inline -regexp [split $text \n] $re] +} + +# homa::lc::parse::oopsCount -- +# +# Counts the lines of text that report a kernel bug or warning. +# +# Arguments: +# text dmesg or serial console output. +# +# Results: +# How many lines of text report a kernel bug or warning. +# +# Side effects: +# None. + +proc ::homa::lc::parse::oopsCount {text} { + return [llength [oops $text]] +} + +# homa::lc::parse::smokeDetail -- +# +# Returns the detail for the smoke phase. +# +# Arguments: +# text homa-smoke's output. +# rc the console's result for the command. +# +# Results: +# homa-smoke's last verdict, or the last step it reached when it gave +# none. +# +# Side effects: +# None. + +proc ::homa::lc::parse::smokeDetail {text rc} { + set verdicts [regexp -all -inline {HOMA_SMOKE_(?:PASS|FAIL:[A-Z_]+)} $text] + if {[llength $verdicts]} { + return [lindex $verdicts end] + } + set steps [regexp -all -inline {HOMA_SMOKE_[A-Z_]+_OK} $text] + return "no verdict (rc $rc), last step: [lindex $steps end]" +} + +# ---- Counters ---------------------------------------------------------------------- + +# homa::lc::parse::snap -- +# +# Reads homa-node-stats output ("name value ..." lines) into a dictionary +# of the given counters, summing repeated names (Homa reports one line per +# core). Counters that do not appear are 0. +# +# Arguments: +# text homa-node-stats output. +# keys the counters to keep, in order. +# +# Results: +# A dictionary of counter name -> sum, in the order of keys. +# +# Side effects: +# None. + +proc ::homa::lc::parse::snap {text keys} { + set sums [dict create] + foreach key $keys { + dict set sums $key 0 + } + foreach line [split $text \n] { + # Not lassign: the rest of the line is free text, not a list. + if {[regexp {^(\S+)\s+(\S+)} $line -> name value] + && [dict exists $sums $name] && [string is entier -strict $value]} { + dict incr sums $name $value + } + } + return $sums +} + +# homa::lc::parse::delta -- +# +# Computes the counters that changed between two snaps, as after - before. +# A counter missing from either side counts as 0. +# +# Arguments: +# before the earlier snap (a dictionary of counters). +# after the later snap. +# +# Results: +# A dictionary of each changed counter -> its difference, in before's +# order, then any counters only in after. +# +# Side effects: +# None. + +proc ::homa::lc::parse::delta {before after} { + # In the first snap's order, then any new counters: a key seen twice + # gives the same difference, and dict set keeps its first place. + set moved [dict create] + foreach key [concat [dict keys $before] [dict keys $after]] { + set diff [expr {[get $after $key] - [get $before $key]}] + if {$diff != 0} { + dict set moved $key $diff + } + } + return $moved +} + +# homa::lc::parse::get -- +# +# Looks up one counter. +# +# Arguments: +# counters a dictionary of counters. +# key the counter's name. +# +# Results: +# Counter key from counters, or 0 if it is missing. +# +# Side effects: +# None. + +proc ::homa::lc::parse::get {counters key} { + if {[dict exists $counters $key]} { + return [dict get $counters $key] + } + return 0 +} + +# ---- cp_node ----------------------------------------------------------------------- + +# homa::lc::parse::field -- +# +# Finds the word after word in a cp_node stats line (e.g. "clients:" for +# Kops/s, "P50", "P99"). +# +# Arguments: +# line the stats line. +# word the word before the value. +# missing what to return if there is none; "-" by default. +# +# Results: +# The word after word, or missing if there is none. +# +# Side effects: +# None. + +proc ::homa::lc::parse::field {line word {missing -}} { + set words [regexp -all -inline {\S+} $line] + set i [lsearch -exact $words $word] + if {$i < 0 || $i + 1 >= [llength $words]} { + return $missing + } + return [lindex $words $i+1] +} + +# homa::lc::parse::positive -- +# +# Tells whether value is a number above zero. +# +# Arguments: +# value any string. +# +# Results: +# 1 if value is a number above zero, 0 otherwise (including "-"). +# +# Side effects: +# None. + +proc ::homa::lc::parse::positive {value} { + return [expr {[string is double -strict $value] && $value > 0}] +} + +# homa::lc::parse::trafficFailures -- +# +# Judges one traffic run: "echo" fails unless every echo RPC completed, +# "homa" or "tcp" unless that cp_node run timed any RPC (a P99 above 0; at +# 5 % loss TCP manages so few RPCs that its Kops/s rounds to 0). +# +# Arguments: +# want how many echo RPCs demo_client1 sends (constants.echoRpcs). +# echoes how many echo responses arrived. +# homa cp_node's last stats line over Homa. +# tcp cp_node's last stats line over TCP. +# +# Results: +# A list of what failed: any of echo, homa and tcp, in that order. An +# error if want is not an integer. +# +# Side effects: +# None. + +proc ::homa::lc::parse::trafficFailures {want echoes homa tcp} { + if {![string is entier -strict $want]} { + return -code error "bad echo count \"$want\"" + } + set failed {} + if {![string is entier -strict $echoes] || $echoes != $want} { + lappend failed echo + } + foreach proto {homa tcp} line [list $homa $tcp] { + if {![positive [field $line P99]]} { + lappend failed $proto + } + } + return $failed +} + +# ---- tc netem ------------------------------------------------------------------------ + +# homa::lc::parse::netemSent -- +# +# Reads the netem qdisc's counters from "tc -s qdisc show" output. +# +# Arguments: +# text "tc -s qdisc show" output. +# +# Results: +# {packets drops} from the netem qdisc, or an empty list if there is no +# netem qdisc with counters. +# +# Side effects: +# None. + +proc ::homa::lc::parse::netemSent {text} { + set re {^qdisc netem [^\n]*\n Sent \d+ bytes (\d+) pkt \(dropped (\d+),} + if {[regexp -line -- $re $text -> packets drops]} { + return [list $packets $drops] + } + return {} +} + +# homa::lc::parse::dropRate -- +# +# Checks a node's measured netem drop rate against the configured loss. +# With no loss configured there must be no drops. Otherwise the rate may +# be up to twice the loss plus half a point (it reads a little low, since +# tc counts sent packets per GSO segment but drops per skb), and, once +# there are 10000 packets, must be at least a quarter of the loss, so +# netem is really on (a slow kernel batches more segments per skb: 0.1 % +# has read as 0.04 %). Tail drops would push the rate up; netem's limit +# prevents them. +# +# Arguments: +# packets packets netem sent. +# drops packets netem dropped. +# loss the configured loss, in percent. +# +# Results: +# A dictionary: rate (percent, two decimals) and ok (1 or 0). +# +# Side effects: +# None. + +proc ::homa::lc::parse::dropRate {packets drops loss} { + foreach count [list $packets $drops] { + if {![string is entier -strict $count] || $count < 0} { + return -code error "not a count: \"$count\"" + } + } + if {![string is double -strict $loss]} { + return -code error "not a number: \"$loss\"" + } + set total [expr {$packets + $drops}] + set pct [expr {$total ? 100.0 * $drops / $total : 0.0}] + if {$loss == 0} { + set pass [expr {$drops == 0}] + } else { + set pass [expr {$pct <= 2 * $loss + 0.5 + && ($total < 10000 || $pct >= $loss / 4.0)}] + } + return [dict create rate [format %.2f $pct] ok $pass] +} + +# homa::lc::parse::netemCheck -- +# +# Checks one node's netem drops (see dropRate) from its "tc -s qdisc +# show" output. With no netem counters there is nothing to judge: the +# check passes, and the report shows the drops as unknown. +# +# Arguments: +# node the node's name, for the report. +# text its "tc -s qdisc show dev eth1" output. +# loss the configured loss, in percent. +# +# Results: +# A dictionary: ok (1 or 0) and report, e.g. "node1 4559/100969 +# (4.52%)" or "node1 ?/0 (0.00%)". +# +# Side effects: +# None. + +proc ::homa::lc::parse::netemCheck {node text loss} { + set sent [netemSent $text] + if {[llength $sent]} { + lassign $sent packets drops + set shown $drops + } else { + lassign {0 0 ?} packets drops shown + } + set measured [dropRate $packets $drops $loss] + return [dict create ok [dict get $measured ok] report [format "%s %s/%s (%s%%)" \ + $node $shown [expr {$packets + $drops}] [dict get $measured rate]]] +} + +# homa::lc::parse::netemLimitOk -- +# +# Checks that "tc qdisc show" output has a netem qdisc with exactly the +# given limit. +# +# Arguments: +# text "tc qdisc show" output. +# limit the expected limit, in packets. +# +# Results: +# 1 if it does, 0 otherwise. An error if limit is not an integer. +# +# Side effects: +# None. + +proc ::homa::lc::parse::netemLimitOk {text limit} { + if {![string is entier -strict $limit]} { + return -code error "not a count: \"$limit\"" + } + return [regexp -line -- "^qdisc netem .*limit ${limit}( |\$)" $text] +} + +# homa::lc::parse::netemDetail -- +# +# Reads the netem qdisc's settings from "tc qdisc show" output. +# +# Arguments: +# text "tc qdisc show" output. +# +# Results: +# The settings, without tc's "qdisc netem <handle>: root refcnt <n>" +# prefix, or an empty string if there is no netem qdisc. +# +# Side effects: +# None. + +proc ::homa::lc::parse::netemDetail {text} { + set re {^qdisc netem [0-9a-f]*: root refcnt [0-9]* ([^\n]*)} + if {[regexp -line -- $re $text -> settings]} { + return $settings + } + return {} +} + +# ---- A driver's output ---------------------------------------------------------- + +# homa::lc::parse::shape -- +# +# Reduces a driver's output (a <test>.out file of homa-vm-matrix) to the +# lines two good runs share, for homa-vm-compare: each phase's id, name +# and verdict, the HOMA_* lines with their numbers masked, and the +# VERDICT lines. Times, details, counters and the summary table vary +# from run to run and are dropped. +# +# Arguments: +# text the output. +# +# Results: +# A list of lines: "phase <id> <name>: <verdict>", "HOMA_..." with each +# number (decimals too) replaced by N, and "VERDICT: ..." as it is. +# +# Side effects: +# None. + +proc ::homa::lc::parse::shape {text} { + set lines {} + foreach line [split $text \n] { + if {[regexp {^\s*phase (\S+) (.*?): (PASS|FAIL|SKIP) \(\d+ ms\)} $line -> id name result]} { + lappend lines "phase $id $name: $result" + } elseif {[regexp {^VERDICT: } $line]} { + lappend lines $line + } elseif {![string match |* $line] && [regexp {HOMA_[A-Z0-9_]+.*} $line homa]} { + lappend lines [regsub -all {\d+(?:\.\d+)?} $homa N] + } + } + return $lines +} + +package provide homa::lc::parse 1.0 diff --git a/nix/microvms/tcl/pkgIndex.tcl b/nix/microvms/tcl/pkgIndex.tcl new file mode 100644 index 00000000..c27d8252 --- /dev/null +++ b/nix/microvms/tcl/pkgIndex.tcl @@ -0,0 +1,11 @@ +# pkgIndex.tcl -- +# +# homa::lc::parse, report and protocol are pure Tcl (8.6 or 9). +# homa::lc::console and homa::lc (the phase table and the guest nodes) +# need Expect, which is built for Tcl 8.6. + +package ifneeded homa::lc::parse 1.0 [list source -encoding utf-8 [file join $dir parse.tcl]] +package ifneeded homa::lc::report 1.0 [list source -encoding utf-8 [file join $dir report.tcl]] +package ifneeded homa::lc::protocol 1.0 [list source -encoding utf-8 [file join $dir protocol.tcl]] +package ifneeded homa::lc::console 1.0 [list source -encoding utf-8 [file join $dir console.tcl]] +package ifneeded homa::lc 1.0 [list source -encoding utf-8 [file join $dir lc.tcl]] diff --git a/nix/microvms/tcl/protocol.tcl b/nix/microvms/tcl/protocol.tcl new file mode 100644 index 00000000..9857ed68 --- /dev/null +++ b/nix/microvms/tcl/protocol.tcl @@ -0,0 +1,110 @@ +# protocol.tcl -- +# +# The end-marker protocol homa::lc::Console speaks to a guest's shell. +# Each command sent is followed by an echo of a numbered end marker that +# carries the command's exit status: +# +# uname -r; echo "HOMA_LC_""END 7 rc=$?" +# +# The quotes split the marker in the command itself, so a shell that +# echoes its input never prints a line that matches. A command that timed +# out prints its output and marker later, under an older number; the +# next command keeps that output but does not stop at its marker. +# +# Pure, like parse.tcl: tests/protocol.test covers it with tables, on +# Tcl 8.6 and 9. Only console.tcl uses it. + +package require Tcl 8.6- + +namespace eval ::homa::lc::protocol { + namespace export wrap line + namespace ensemble create + + # Any end marker, ours or a stale one; the first subexpression is its + # command number, the second its exit status. + variable markerRe {HOMA_LC_END (\d+) rc=(\d+)} +} + +# homa::lc::protocol::wrap -- +# +# Returns the line to send to the shell for one command. +# +# Arguments: +# cmd the shell command, on one line: the shell would print a +# continuation prompt into the output for each further line. +# id the command's number, a positive integer. +# +# Results: +# The command followed by its end marker's echo. +# +# Side effects: +# None. + +proc ::homa::lc::protocol::wrap {cmd id} { + checkId $id + if {[string trim $cmd] eq ""} { + return -code error "empty command" + } + if {[string first \n $cmd] >= 0} { + return -code error "command has more than one line" + } + return "$cmd; echo \"HOMA_LC_\"\"END $id rc=\$?\"" +} + +# homa::lc::protocol::line -- +# +# Reads one line the shell printed while command id was running. +# +# Arguments: +# id the running command's number. +# line the line, without its newline; a tty's carriage return is +# allowed. +# +# Results: +# A list {keep text done rc}: +# keep 1 if text is output to keep. A line that held only an end +# marker is dropped; any other line is kept, blank or not. +# text the line without the carriage return or any end marker, +# leaving what came before it on the line ("printf abc" prints no +# newline before the marker). +# done 1 if the line carries command id's marker. +# rc the command's exit status if done, else "". +# +# Side effects: +# None. + +proc ::homa::lc::protocol::line {id line} { + variable markerRe + checkId $id + set line [string trimright $line \r] + set rc {} + foreach {- markerId status} [regexp -all -inline -- $markerRe $line] { + if {$markerId eq $id} { + set rc $status + } + } + set text [regsub -all -- $markerRe $line {}] + set keep [expr {$text ne "" || $text eq $line}] + return [list $keep $text [expr {$rc ne ""}] $rc] +} + +# homa::lc::protocol::checkId -- +# +# Raises an error unless id is a positive integer. +# +# Arguments: +# id a command's number. +# +# Results: +# None. +# +# Side effects: +# None. + +proc ::homa::lc::protocol::checkId {id} { + if {![string is entier -strict $id] || $id < 1} { + return -code error "bad command id \"$id\"" + } +} + +package provide homa::lc::protocol 1.0 diff --git a/nix/microvms/tcl/report.tcl b/nix/microvms/tcl/report.tcl new file mode 100644 index 00000000..9795e148 --- /dev/null +++ b/nix/microvms/tcl/report.tcl @@ -0,0 +1,310 @@ +# report.tcl -- +# +# Pure functions that format what the drivers print and write: phase +# verdicts, progress lines and summary.md rows, counter deltas, and +# homa-vm-matrix's table rows. tests/report.test covers them with tables. +# +# Every command is a subcommand of the ensemble homa::lc::report, e.g. +# "report phaseLine ..." after "namespace import homa::lc::report". + +package require Tcl 8.6- +package require homa::lc::parse + +namespace eval ::homa::lc { + namespace export report +} +namespace eval ::homa::lc::report { + namespace export verdict combine phaseLine tableRow deltaLine trafficDetail matrixRow diff + namespace ensemble create + # Find parse (for field). + namespace path ::homa::lc + + # Shown in a table cell when a value is missing. + variable dash — +} + +# ---- Phase results ----------------------------------------------------------- + +# homa::lc::report::verdict -- +# +# Maps a phase's result to PASS, SKIP or FAIL. +# +# Arguments: +# rc 0, any other exit status, or "skip". +# +# Results: +# PASS, SKIP or FAIL. An error if rc is neither an integer nor "skip". +# +# Side effects: +# None. + +proc ::homa::lc::report::verdict {rc} { + if {$rc eq "skip"} { + return SKIP + } + if {![string is entier -strict $rc]} { + return -code error "bad rc \"$rc\"" + } + return [expr {$rc == 0 ? "PASS" : "FAIL"}] +} + +# homa::lc::report::combine -- +# +# Combines the results of several commands in one phase. Folded over the +# commands, this gives the last failure, or 0. +# +# Arguments: +# rc the result so far. +# next the next command's exit status. +# +# Results: +# next if it failed (is not 0), else rc. An error if either is not an +# integer. +# +# Side effects: +# None. + +proc ::homa::lc::report::combine {rc next} { + foreach value [list $rc $next] { + if {![string is entier -strict $value]} { + return -code error "bad rc \"$value\"" + } + } + return [expr {$next == 0 ? $rc : $next}] +} + +# homa::lc::report::phaseLine -- +# +# Returns the progress line printed when a phase ends. +# +# Arguments: +# phase the phase's number. +# name the phase's name. +# verdict PASS, SKIP or FAIL. +# ms how long it took, in milliseconds. +# detail what to add after the verdict, or "". +# +# Results: +# The line, e.g. " phase 3 smoke: PASS (120 ms): ...". +# +# Side effects: +# None. + +proc ::homa::lc::report::phaseLine {phase name verdict ms detail} { + set line " phase $phase $name: $verdict ($ms ms)" + if {$detail ne ""} { + append line ": $detail" + } + return $line +} + +# homa::lc::report::tableRow -- +# +# Returns a phase's row in summary.md (a Markdown table). +# +# Arguments: +# phase the phase's number. +# name the phase's name. +# verdict PASS, SKIP or FAIL. +# ms how long it took, in milliseconds. +# detail the phase's detail; any "|" in it is escaped. +# +# Results: +# The Markdown table row. +# +# Side effects: +# None. + +proc ::homa::lc::report::tableRow {phase name verdict ms detail} { + set detail [string map {| \\|} $detail] + return "| $phase | $name | $verdict | $ms | $detail |" +} + +# ---- Counters -------------------------------------------------------------------- + +# homa::lc::report::deltaLine -- +# +# Formats counters for the deltas log. +# +# Arguments: +# counters a dictionary of counter name -> value. +# +# Results: +# The counters as space-separated "name=value" pairs. +# +# Side effects: +# None. + +proc ::homa::lc::report::deltaLine {counters} { + return [join [lmap {key value} $counters {string cat $key = $value}]] +} + +# ---- Traffic runs ------------------------------------------------------------------ + +# homa::lc::report::trafficDetail -- +# +# Judges and describes one pair traffic run (echo, cp_node over Homa and +# TCP, counter deltas, netem drops). +# +# Arguments: +# measured a dictionary: echoes (responses that arrived), homa and +# tcp (cp_node's last stats lines), deltas (node name -> +# counter deltas) and checks (node name -> parse +# netemCheck's result; empty without netem). +# want how many echo RPCs were sent (constants.echoRpcs). +# must a counter that must have gone up, summed over the nodes, +# or "". +# +# Results: +# A dictionary: failed (what failed, from parse trafficFailures, then +# drops-<node> and no-<counter>) and detail (the phase's detail). +# +# Side effects: +# None. + +proc ::homa::lc::report::trafficDetail {measured want must} { + lassign [lmap key {echoes homa tcp deltas checks} {dict get $measured $key}] \ + echoes homa tcp deltas checks + set failures [parse trafficFailures $want $echoes $homa $tcp] + set text [format "echo %s/%s; Homa %s Kops P50 %s P99 %s us; TCP %s Kops P99 %s us" \ + $echoes $want [parse field $homa clients:] [parse field $homa P50] \ + [parse field $homa P99] [parse field $tcp clients:] [parse field $tcp P99]] + if {[dict size $checks]} { + append text "; netem drops [join [lmap {node check} $checks {dict get $check report}]]" + dict for {node check} $checks { + if {![dict get $check ok]} { + lappend failures drops-$node + } + } + } + append text [format "; RESENDs %d, resent %d, rpc_timeouts %d, tcp_retrans %d" \ + [sum $deltas packets_sent_RESEND] [sum $deltas resent_packets] \ + [sum $deltas rpc_timeouts] [sum $deltas tcp_retrans]] + if {$must ne ""} { + set count [sum $deltas $must] + append text "; $must $count" + if {$count <= 0} { + lappend failures no-$must + } + } + if {[llength $failures]} { + append text "; failed: [join $failures]" + } + return [dict create failed $failures detail $text] +} + +# homa::lc::report::sum -- +# +# Sums one counter over every node's deltas. +# +# Arguments: +# deltas a dictionary of node name -> counters. +# key the counter's name. +# +# Results: +# Counter key summed over the nodes; 0 if there are none. +# +# Side effects: +# None. + +proc ::homa::lc::report::sum {deltas key} { + return [tcl::mathop::+ {*}[lmap {node counters} $deltas {parse get $counters $key}]] +} + +# ---- homa-vm-matrix -------------------------------------------------------------------- + +# homa::lc::report::matrixRow -- +# +# Reads one test's output for homa-vm-matrix's table. +# +# Arguments: +# name the test (its package name). +# kernels the kernel(s) it ran. +# secs how long it took. +# want how many echo RPCs a test sends (constants.echoRpcs). +# out its output. A test passed if a line starts "VERDICT: PASS"; echo, +# GRO and cp_node figures come from its HOMA_* lines, each optional. +# +# Results: +# A dictionary: verdict (PASS or FAIL) and row (the Markdown table row). +# +# Side effects: +# None. + +proc ::homa::lc::report::matrixRow {name kernels secs want out} { + variable dash + set result [expr {[regexp -line {^VERDICT: PASS} $out] ? "PASS" : "FAIL"}] + set echoes 0 + regexp {HOMA_(?:SMOKE|PAIR)_ECHO responses=(\d+)} $out -> echoes + set gro $dash + regexp {homa_gro_receive calls=(\d+)} $out -> gro + set cells [list $name $kernels $result $secs $echoes/$want $gro] + foreach proto {homa tcp} { + set line [lindex [regexp -all -inline -line "HOMA_PAIR_CPNODE_$proto .*" $out] end] + lappend cells [parse field $line clients: $dash] \ + "[parse field $line P50 $dash] / [parse field $line P99 $dash]" + } + return [dict create verdict $result row "| [join $cells { | }] |"] +} + +# ---- homa-vm-compare ----------------------------------------------------------------- + +# homa::lc::report::diff -- +# +# Lists the changes between two lists of lines, by their longest common +# subsequence (the lists are a run's phase lines: a hundred or so). +# +# Arguments: +# old the lines before. +# new the lines after. +# +# Results: +# The changes in order, each "- <line>" (only in old) or "+ <line>" +# (only in new), removals before additions where a line was replaced; +# an empty list if the lists are equal. +# +# Side effects: +# None. + +proc ::homa::lc::report::diff {old new} { + set n [llength $old] + set m [llength $new] + # common(i,j): the length of the longest common subsequence of + # old[i..] and new[j..]. + for {set i $n} {$i >= 0} {incr i -1} { + for {set j $m} {$j >= 0} {incr j -1} { + if {$i == $n || $j == $m} { + set common($i,$j) 0 + } elseif {[lindex $old $i] eq [lindex $new $j]} { + set common($i,$j) [expr {$common([expr {$i + 1}],[expr {$j + 1}]) + 1}] + } else { + set common($i,$j) [expr {max($common([expr {$i + 1}],$j), + $common($i,[expr {$j + 1}]))}] + } + } + } + set changes {} + set i 0 + set j 0 + while {$i < $n && $j < $m} { + if {[lindex $old $i] eq [lindex $new $j]} { + incr i + incr j + } elseif {$common([expr {$i + 1}],$j) >= $common($i,[expr {$j + 1}])} { + lappend changes "- [lindex $old $i]" + incr i + } else { + lappend changes "+ [lindex $new $j]" + incr j + } + } + foreach line [lrange $old $i end] { + lappend changes "- $line" + } + foreach line [lrange $new $j end] { + lappend changes "+ $line" + } + return $changes +} + +package provide homa::lc::report 1.0 diff --git a/nix/microvms/tcl/tclint.toml b/nix/microvms/tcl/tclint.toml new file mode 100644 index 00000000..4cc3126d --- /dev/null +++ b/nix/microvms/tcl/tclint.toml @@ -0,0 +1,8 @@ +# tclint settings for this package (nix/microvms/host/tcl.nix runs it in the tcl-tests check). +# +# pkgIndex.tcl is left to nagelfar: tclint cannot parse "package ifneeded" scripts yet and +# says so for every line. +exclude = ["pkgIndex.tcl"] + +# The tests too, not just *.tcl. +extensions = ["tcl", "test"] diff --git a/nix/microvms/tcl/tests/all.tcl b/nix/microvms/tcl/tests/all.tcl new file mode 100644 index 00000000..7e9b0965 --- /dev/null +++ b/nix/microvms/tcl/tests/all.tcl @@ -0,0 +1,25 @@ +# all.tcl -- +# +# Runs every *.test file in this directory, in this interpreter. +# +# tclsh all.tcl ?tcltest options? +# +# For example "-file parse.test" runs just the pure tables, which is what +# the Tcl 9 run does (Expect is built for Tcl 8.6 only). Exits non-zero +# if any test fails. + +package require tcltest 2 +encoding system utf-8 + +tcltest::configure -testdir [file dirname [file normalize [info script]]] \ + -singleproc 1 {*}$argv + +# runAllTests returns normally even when tests fail; count failures from its +# report instead. +set failed 0 +proc tcltest::cleanupTestsHook {} { + variable numTests + incr ::failed $numTests(Failed) +} +tcltest::runAllTests +exit [expr {$failed > 0}] diff --git a/nix/microvms/tcl/tests/batch.test b/nix/microvms/tcl/tests/batch.test new file mode 100644 index 00000000..56aaefd6 --- /dev/null +++ b/nix/microvms/tcl/tests/batch.test @@ -0,0 +1,181 @@ +# batch.test -- +# +# Table-driven tests for homa::lc::Batch, which homa-vm-matrix uses to +# run every VM test at once. Small shell scripts stand in for the tests. +# Every row is {description category input outcome expected}; see +# testTable in helpers.tcl. The input is {limit tests}, each test +# {name script}, and the body returns each test's rc and output. + +source [file join [file dirname [info script]] helpers.tcl] +package require homa::lc + +# runBatch -- +# +# Writes each test's script to an executable file, runs them all through +# a Batch, and returns what each did. +# +# Arguments: +# limit the Batch's job limit. +# tests a list of {name script}: script is the body of a sh script, +# or "missing" for a test whose command does not exist. +# +# Results: +# A list, per test in order: {name rc out}, with the run's log +# directory replaced by LOGDIR. + +proc runBatch {limit tests} { + set logdir [tcltest::makeDirectory work] + try { + set specs [lmap test $tests { + lassign $test testName script + set path [file join $logdir $testName.sh] + if {$script ne "missing"} { + homa::lc::writeFile $path "#!/bin/sh\n$script" + file attributes $path -permissions 0755 + } + dict create name $testName exe $path + }] + set batch [homa::lc::Batch new $logdir $limit] + try { + lassign [quietly {$batch run $specs}] results + } finally { + $batch destroy + } + return [lmap test $tests { + set name [lindex $test 0] + set out [homa::lc::readFile [file join $logdir $name.out]] + string map [list $logdir LOGDIR] [list $name [lindex [dict get $results $name] 0] \ + [string trimright $out \n]] + }] + } finally { + tcltest::removeDirectory work + } +} + +# Each test holds ../lock while it runs: with more than one at a time, the +# second fails to take it and exits 9. +set holdLock {mkdir ../lock || exit 9; sleep 0.3; rmdir ../lock} + +testTable batch { + {"a passing test: rc 0 and its output" positive {1 {{a {echo hello}}}} ok + {{a 0 hello}}} + {"a failing test's exit status" negative {1 {{a {echo no; exit 3}}}} ok + {{a 3 no}}} + {"stderr goes to the same file" positive {1 {{a {echo out; echo err >&2}}}} ok + {{a 0 {out +err}}}} + {"each runs in its own directory, named by HOMA_VM_LOGDIR" positive + {2 {{a {pwd; echo "$HOMA_VM_LOGDIR"}} {b {pwd}}}} ok + {{a 0 {LOGDIR/a +LOGDIR/a}} {b 0 LOGDIR/b}}} + {"a test killed by a signal" negative {1 {{a {kill -KILL $$}}}} ok + {{a SIGKILL {}}}} + {"a test that cannot start" negative {1 {{a missing} {b {echo ran}}}} ok + {{a {error: couldn't execute "LOGDIR/a.sh": no such file or directory} {}} {b 0 ran}}} + {"an empty script passes" boundary {1 {{a {}}}} ok + {{a 0 {}}}} + {"no tests" boundary {1 {}} ok {}} + {"more tests than the limit all run" corner + {2 {{a {exit 1}} {b {exit 2}} {c {exit 3}} {d {exit 4}} {e {exit 5}}}} ok + {{a 1 {}} {b 2 {}} {c 3 {}} {d 4 {}} {e 5 {}}}} + {"a limit of 0 is an error" negative {0 {{a {exit 0}}}} error + {bad job limit "0"}} + {"a limit that is not a number" negative {many {{a {exit 0}}}} error + {bad job limit "many"}} + {"two tests with one name" negative {2 {{a {exit 0}} {a {exit 0}}}} error + {test names are not unique: a a}} +} { + runBatch {*}$input +} + +# The highest rc of three tests that each hold ../lock: 9 if any two ran +# at once. +testTable batchParallel { + {"a limit of 1 runs them one at a time" boundary 1 ok 0} + {"a limit of 2 runs two at once" positive 2 ok 9} + {"a limit above the number of tests runs them all at once" corner 10 ok 9} +} { + set rcs [lmap result [runBatch $input [list [list a $::holdLock] [list b $::holdLock] \ + [list c $::holdLock]]] {lindex $result 1}] + tcl::mathfunc::max {*}$rcs +} + +# envState -- +# +# Returns env(name) as a list of one value, or an empty list if unset, +# so "unset" and "set to empty" differ. + +proc envState {name} { + global env + if {[info exists env($name)]} { + return [list $env($name)] + } + return {} +} + +# setEnvState -- +# +# Sets env(name) from a state as envState returns it. + +proc setEnvState {name state} { + global env + if {[llength $state]} { + set env($name) [lindex $state 0] + } else { + unset -nocomplain env($name) + } +} + +# The input is {prior value script}: prior as envState gives it. The result is +# {code result after}: withEnv's catch code and result, and envState after. +testTable withEnv { + {"the script sees the value" positive {{} v {set ::env(HOMA_TEST_ENV)}} ok + {0 v {}}} + {"a prior value is restored" positive {old v {set ::env(HOMA_TEST_ENV)}} ok + {0 v old}} + {"an error in the script still restores the value" negative {old v {error boom}} ok + {1 boom old}} + {"an error with no prior value still unsets it" negative {{} v {error boom}} ok + {1 boom {}}} + {"an empty value is set, not unset" boundary {{} {} {info exists ::env(HOMA_TEST_ENV)}} ok + {0 1 {}}} + {"a prior empty value is restored as empty" boundary {{{}} v {set ::env(HOMA_TEST_ENV)}} ok + {0 v {{}}}} + {"nested calls restore in order" corner + {old v {list [homa::lc::withEnv HOMA_TEST_ENV inner {set ::env(HOMA_TEST_ENV)}]\ + $::env(HOMA_TEST_ENV)}} ok + {0 {inner v} old}} + {"the script runs in the caller's scope" corner {{} v {set here}} ok + {0 caller {}}} +} { + lassign $input prior value script + setEnvState HOMA_TEST_ENV $prior + set here caller + set code [catch {homa::lc::withEnv HOMA_TEST_ENV $value $script} result] + set after [envState HOMA_TEST_ENV] + unset -nocomplain ::env(HOMA_TEST_ENV) + list $code $result $after +} + +# The input is {prior script}: HOMA_VM_LOGDIR before the batch, as envState +# gives it, and the test's script ("missing" for one that cannot start). The +# result is HOMA_VM_LOGDIR after it, as envState gives it. +testTable batchEnv { + {"a caller's value is restored" positive {/keep {exit 0}} ok /keep} + {"unset before, unset after" positive {{} {exit 0}} ok {}} + {"an empty value is restored as empty" boundary {{{}} {exit 0}} ok {{}}} + {"restored even when the test cannot start" negative {/keep missing} ok /keep} + {"a value with spaces and a newline" corner {{"a b\nc"} {exit 0}} ok "{a b\nc}"} +} { + lassign $input prior script + set saved [envState HOMA_VM_LOGDIR] + setEnvState HOMA_VM_LOGDIR $prior + try { + runBatch 1 [list [list a $script]] + envState HOMA_VM_LOGDIR + } finally { + setEnvState HOMA_VM_LOGDIR $saved + } +} + +tcltest::cleanupTests diff --git a/nix/microvms/tcl/tests/console.test b/nix/microvms/tcl/tests/console.test new file mode 100644 index 00000000..b42ca808 --- /dev/null +++ b/nix/microvms/tcl/tests/console.test @@ -0,0 +1,182 @@ +# console.test -- +# +# Table-driven tests for homa::lc::Console. A real bash shell behind a +# local socket stands in for the guest's hvc0 shell, so these exercise +# Expect's reading as well as the marker protocol. Every row is +# {description category input outcome expected}; see testTable in +# helpers.tcl. + +source [file join [file dirname [info script]] helpers.tcl] +package require homa::lc::console + +# fakeGuest -- +# +# Starts a bash shell behind a listening socket and connects to it. +# +# Arguments: +# echo if 1, each line sent is echoed back before bash reads it, like +# a tty before "stty -echo". +# +# Results: +# The connected channel, for homa::lc::Console. + +proc fakeGuest {{echo 0}} { + set server [socket -server [list acceptGuest $echo] -myaddr 127.0.0.1 0] + set chan [socket 127.0.0.1 [lindex [chan configure $server -sockname] 2]] + vwait ::guestUp + close $server + return $chan +} + +# acceptGuest -- +# +# Accepts the connection from fakeGuest and joins it to a new bash, its +# stderr merged into stdout. When either side ends, both are closed. + +proc acceptGuest {echo sock addr port} { + set shell [open |[list bash --norc --noprofile 2>@1] r+] + foreach chan [list $sock $shell] { + chan configure $chan -blocking 0 -buffering none -translation binary + } + chan event $shell readable [list pump $shell $sock {} $sock $shell] + chan event $sock readable [list pump $sock $shell [expr {$echo ? $sock : ""}] $sock $shell] + set ::guestUp 1 +} + +# pump -- +# +# Copies what is ready from one channel to another, and to echo too if +# it is not empty. Closes both of the guest's channels at end of file. + +proc pump {from to echo sock shell} { + set data [read $from] + foreach chan [list $echo $to] { + if {$chan ne ""} { + puts -nonewline $chan $data + } + } + if {[eof $from]} { + foreach chan [list $sock $shell] { + catch {close $chan} + } + } +} + +# show -- +# +# Formats one Console result for a table, with newlines and carriage +# returns visible. + +proc show {result} { + set out [string map {\n \\n \r \\r} [dict get $result out]] + return "rc=[dict get $result rc] out=$out" +} + +# session -- +# +# Runs commands, a list of {command timeout}, in one fake guest and +# returns each result as formatted by show. + +proc session {commands {echo 0}} { + set console [homa::lc::Console new [fakeGuest $echo]] + try { + return [lmap step $commands { + show [$console run {*}$step] + }] + } finally { + $console destroy + } +} + +testTable consoleRun { + {"one line of output" positive + {{{echo hi} 5}} ok {{rc=0 out=hi}}} + {"no output" positive + {{true 5}} ok {{rc=0 out=}}} + {"several lines" positive + {{{printf 'a\nb\n'} 5}} ok {{rc=0 out=a\nb}}} + {"a non-zero exit status" positive + {{false 5}} ok {{rc=1 out=}}} + {"an exit status with stderr output" positive + {{{echo oops >&2; (exit 3)} 5}} ok {{rc=3 out=oops}}} + {"commands share one shell" positive + {{{x=1} 5} {{echo $x} 5}} ok {{rc=0 out=} {rc=0 out=1}}} + {"a command that outlasts its timeout" negative + {{{sleep 3; echo late} 1}} ok {{rc=124 out=}}} + {"the next command skips a timed-out command's marker" corner + {{{sleep 3; echo late} 1} {{echo next} 10}} + ok {{rc=124 out=} {rc=0 out=late\nnext}}} + {"the shell exits" negative + {{{exit 0} 5}} ok {{rc=125 out=}}} + {"commands after the console closed" negative + {{{exit 0} 5} {{echo hi} 5}} ok {{rc=125 out=} {rc=125 out=}}} + {"CRLF line endings, as a tty sends" corner + {{{printf 'a\r\nb\r\n'} 5}} ok {{rc=0 out=a\nb}}} + {"output with no final newline" corner + {{{printf abc} 5}} ok {{rc=0 out=abc}}} + {"a line split across writes" corner + {{{printf ab; sleep 0.5; echo cd} 5}} ok {{rc=0 out=abcd}}} + {"blank lines are kept" corner + {{{printf 'a\n\nb\n'} 5}} ok {{rc=0 out=a\n\nb}}} + {"an exit status above 125" boundary + {{{(exit 255)} 5}} ok {{rc=255 out=}}} +} {session $input} + +testTable consoleEcho { + {"the echoed command line is output, its marker is not matched" corner + {{{sleep 1; echo hi} 5}} + ok {{rc=0 out=sleep 1; echo hi; echo "HOMA_LC_""END 1 rc=$?"\nhi}}} +} {session $input 1} + +# volume -- +# +# Runs one command in a fake guest whose console buffers at most matchMax +# bytes, and summarises its output. + +proc volume {cmd matchMax} { + set console [homa::lc::Console new [fakeGuest] $matchMax] + try { + set result [$console run $cmd 30] + } finally { + $console destroy + } + set lines [parse lines [dict get $result out]] + return [format "rc=%s lines=%d chars=%d last=%.10s" [dict get $result rc] \ + [llength $lines] [string length [dict get $result out]] [lindex $lines end]] +} + +testTable consoleVolume { + {"20000 lines" boundary + {{seq 20000} 65536} ok {rc=0 lines=20000 chars=108893 last=20000}} + {"a line five times Expect's buffer" corner + {{head -c 5000 /dev/zero | tr '\0' x; echo} 1000} + ok {rc=0 lines=1 chars=5000 last=xxxxxxxxxx}} + {"a line exactly Expect's buffer" boundary + {{head -c 1000 /dev/zero | tr '\0' y; echo} 1000} + ok {rc=0 lines=1 chars=1000 last=yyyyyyyyyy}} +} {volume {*}$input} + +# sendThenRun -- +# +# Sends a line without waiting, then runs "echo $x" and sends again. +# Returns whether each send was written, and the command's result. + +proc sendThenRun {line} { + set console [homa::lc::Console new [fakeGuest]] + try { + set first [$console send $line] + set result [show [$console run {echo $x} 5]] + return "$first $result [$console send true] alive=[$console alive]" + } finally { + $console destroy + } +} + +testTable consoleSend { + {"a line sent without waiting still runs" positive + {x=5} ok {1 rc=0 out=5 1 alive=1}} + {"after the shell exits, sends fail" negative + {exit} ok {1 rc=125 out= 0 alive=0}} +} {sendThenRun $input} + +tcltest::cleanupTests diff --git a/nix/microvms/tcl/tests/fakeqemu.tcl b/nix/microvms/tcl/tests/fakeqemu.tcl new file mode 100644 index 00000000..612d8ecb --- /dev/null +++ b/nix/microvms/tcl/tests/fakeqemu.tcl @@ -0,0 +1,80 @@ +# fakeqemu.tcl -- +# +# Stands in for a guest's microvm-run in node.test: writes serial.log in +# the current directory, as QEMU does, and serves a bash shell on the +# virtio console port, as the guest's hvc0 autologin does. +# +# tclsh fakeqemu.tcl port mode bin +# +# port the virtio console port to listen on. +# mode ok, oops (the kernel logs a BUG, in serial.log and dmesg), +# silent (listens, but no shell ever answers) or deaf (never +# listens). +# bin put first on the shell's PATH: the test's poweroff and dmesg. +# +# The shell's poweroff ends this process, which is how QEMU exits. + +lassign $argv port mode bin + +set serial {[ 0.000000] Linux version 0.0-fake} +if {$mode eq "oops"} { + append serial "\n\[ 1.000000\] BUG: kernel NULL pointer dereference" + append serial "\n\[ 1.000001\] RIP: 0010:homa_fake+0x10/0x20" +} +set chan [open serial.log w] +puts $chan $serial +close $chan +puts "fake qemu: $mode on $port" +flush stdout + +set env(PATH) $bin:$env(PATH) +set env(FAKE_QEMU_PID) [pid] +set env(FAKE_MODE) $mode + +# serve -- +# +# Joins a console connection to a new bash (or, when silent, to +# nothing), as the guest's autologin shell on hvc0. + +proc serve {sock addr port} { + global mode + chan configure $sock -blocking 0 -buffering none -translation binary + if {$mode eq "silent"} { + chan event $sock readable [list drop $sock] + return + } + set shell [open |[list bash --norc --noprofile 2>@1] r+] + chan configure $shell -blocking 0 -buffering none -translation binary + chan event $shell readable [list relay $shell $sock] + chan event $sock readable [list relay $sock $shell] +} + +# relay -- +# +# Copies what is ready from one channel to the other. At end of file +# the session is over (a probe that connected and closed, or a shell +# that exited) and both are closed; like QEMU, the fake keeps serving. + +proc relay {from to} { + puts -nonewline $to [read $from] + if {[eof $from]} { + catch {close $from} + catch {close $to} + } +} + +# drop -- +# +# Reads and discards what a silent guest is sent. + +proc drop {sock} { + read $sock + if {[eof $sock]} { + close $sock + } +} + +if {$mode ne "deaf"} { + socket -server serve -myaddr 127.0.0.1 $port +} +vwait forever diff --git a/nix/microvms/tcl/tests/files.test b/nix/microvms/tcl/tests/files.test new file mode 100644 index 00000000..c99b4bf1 --- /dev/null +++ b/nix/microvms/tcl/tests/files.test @@ -0,0 +1,121 @@ +# files.test -- +# +# Table-driven tests for homa::lc::readFile and writeFile, the file helpers +# every driver reads its config and writes its logs with. Every row is +# {description category input outcome expected}; see testTable in +# helpers.tcl. + +source [file join [file dirname [info script]] helpers.tcl] +package require homa::lc + +# withFile -- +# +# Runs a script in the caller's scope, with varName set to the path of a +# file in a fresh directory, which the script may write. + +##nagelfar syntax withFile n c +proc withFile {varName script} { + upvar 1 $varName path + set path [file join [tcltest::makeDirectory files] f] + try { + return [uplevel 1 $script] + } finally { + tcltest::removeDirectory files + } +} + +# bytes -- +# +# A file's contents, as hex bytes. + +proc bytes {path} { + set chan [open $path rb] + try { + binary scan [read $chan] H* hex + return $hex + } finally { + close $chan + } +} + +# rawFile -- +# +# Writes bytes, given as hex, to a file. + +proc rawFile {path hex} { + set chan [open $path wb] + puts -nonewline $chan [binary format H* $hex] + close $chan +} + +# Each row's input is text, which writeFile writes and readFile reads back. +testTable filesRoundTrip { + {"ASCII comes back as written" positive "hello" ok "hello\n"} + {"non-ASCII (the matrix's kernel arrow) comes back as written" positive + "6_12 → 7_3" ok "6_12 → 7_3\n"} + {"lines keep their order and gain one final newline" positive "a\nb" ok "a\nb\n"} + {"empty text writes an empty file" boundary "" ok ""} + {"a trailing newline is kept, then one more added" corner "a\n" ok "a\n\n"} +} { + withFile f { + homa::lc::writeFile $f $input + homa::lc::readFile $f + } +} + +# Each row's input is text: the bytes writeFile writes, as hex. +testTable filesSpitBytes { + {"the arrow is written as three UTF-8 bytes" positive "→" ok e286920a} + {"an em dash, as in an empty matrix cell" positive "—" ok e280940a} + {"ASCII is one byte a character" positive "ok" ok 6f6b0a} + {"empty text writes no bytes" boundary "" ok ""} +} { + withFile f { + homa::lc::writeFile $f $input + bytes $f + } +} + +# Each row's input is a file's bytes, as hex, or "missing" for no file. +testTable filesSlurp { + {"UTF-8 JSON is decoded" positive 7b226b223a22e28692227d ok "{\"k\":\"→\"}"} + {"a file that does not exist yet reads as empty" boundary missing ok ""} + {"an empty file reads as empty" boundary "" ok ""} +} { + withFile f { + if {$input ne "missing"} { + rawFile $f $input + } + homa::lc::readFile $f + } +} + +# Each row's input is a file's bytes, as hex, that are not all UTF-8; the +# body returns how many characters readFile read (what each bad byte becomes +# differs between Tcl 8.6 and 9). +testTable filesSlurpBadBytes { + {"a lone byte that is not UTF-8 reads as one character" negative 61ff62 ok 3} + {"a UTF-8 sequence cut short does not fail the read" corner 61e28662 ok 4} + {"a file of nothing but bad bytes" corner ffff ok 2} +} { + withFile f { + rawFile $f $input + string length [homa::lc::readFile $f] + } +} + +# Each row's input is a list of texts appended in turn. +testTable filesAppend { + {"appends add after what is there" positive {a b} ok "a\nb\n"} + {"appending to a missing file creates it" boundary {a} ok "a\n"} + {"appending empty text adds nothing" corner {a {} b} ok "a\nb\n"} +} { + withFile f { + foreach text $input { + homa::lc::writeFile $f $text a + } + homa::lc::readFile $f + } +} + +tcltest::cleanupTests diff --git a/nix/microvms/tcl/tests/helpers.tcl b/nix/microvms/tcl/tests/helpers.tcl new file mode 100644 index 00000000..82bec7a2 --- /dev/null +++ b/nix/microvms/tcl/tests/helpers.tcl @@ -0,0 +1,119 @@ +# helpers.tcl -- +# +# Shared by every test file: loads tcltest and the package under test, and +# defines testTable, which turns a table of cases into tcltest tests. + +# Every test file sources this, and all.tcl runs them in one interpreter. +if {[package provide homa::lc::testhelpers] ne ""} { + return +} +package provide homa::lc::testhelpers 1.0 + +package require tcltest 2 + +set here [file dirname [file normalize [info script]]] +set auto_path [linsert $auto_path 0 [file dirname $here]] +package require homa::lc::parse +namespace import ::homa::lc::parse + +# testTable -- +# +# Runs one tcltest test per row of a table. Each row is a list +# +# {description category input outcome expected} +# +# description what the row checks, used as the test's description. +# category positive, negative, boundary or corner. +# input a single value, passed to the body as $input. +# outcome ok when the body must return expected, error when it +# must raise an error whose message is expected, errorGlob +# when the message must match expected as a glob pattern +# (for messages that differ between Tcl versions). +# expected the result, error message or pattern. +# +# Arguments: +# prefix test name prefix; rows are numbered prefix-1, prefix-2, ... +# cases the table. +# body script run for each row, with $input set to the row's +# input. Its result is compared with expected. +# +# Results: +# None. Fails the run with an error if a row is malformed. + +proc testTable {prefix cases body} { + set categories {positive negative boundary corner} + set outcomes {ok {ok exact} error {error exact} errorGlob {error glob}} + set n 0 + foreach row $cases { + incr n + if {[llength $row] != 5} { + return -code error "$prefix-$n: row has [llength $row] fields, want 5" + } + lassign $row description category value outcome expected + if {$category ni $categories} { + return -code error "$prefix-$n: unknown category \"$category\"" + } + if {![dict exists $outcomes $outcome]} { + return -code error "$prefix-$n: unknown outcome \"$outcome\"" + } + lassign [dict get $outcomes $outcome] codes match + tcltest::test $prefix-$n "$category: $description" \ + -body [list apply [list input $body] $value] \ + -returnCodes $codes -match $match -result $expected + } +} + +# Capture -- +# +# A channel transform that keeps what is written to a channel instead of +# passing it on, so tests can check what Table prints. + +oo::class create Capture { + variable text + + constructor {} { + set text "" + } + + method initialize {chan mode} { + return {initialize finalize write} + } + + method finalize {chan} { + my destroy + } + + method write {chan bytes} { + append text $bytes + return "" + } + + method text {} { + return $text + } +} + +# quietly -- +# +# Runs script in the caller's scope with stdout captured. The transform +# sees bytes after end-of-line translation, so lf is set meanwhile (a +# build sandbox's stdout may translate to crlf). +# +# Results: +# A list: the script's result, and what it printed. + +##nagelfar syntax quietly c +proc quietly {script} { + set translation [chan configure stdout -translation] + chan configure stdout -translation lf + set capture [Capture new] + chan push stdout $capture + try { + set result [uplevel 1 $script] + flush stdout + return [list $result [$capture text]] + } finally { + chan pop stdout + chan configure stdout -translation $translation + } +} diff --git a/nix/microvms/tcl/tests/lintdb.tcl b/nix/microvms/tcl/tests/lintdb.tcl new file mode 100644 index 00000000..843533fd --- /dev/null +++ b/nix/microvms/tcl/tests/lintdb.tcl @@ -0,0 +1,137 @@ +# lintdb.tcl -- +# +# Prints a nagelfar syntax database for this package, so nagelfar can +# check code that uses it, including the scripts passed to Table phase: +# +# tclsh lintdb.tcl > homa.db +# nagelfar -s syntaxdb86.tcl -s homa.db ... +# +# The ensemble and class entries come from the loaded code (ensemble +# maps, info args, info class definition), so they follow it. Only +# methods that take a script, and the Expect and tcllib commands, are +# written out here. + +set auto_path [linsert $auto_path 0 [file dirname [file dirname [file normalize [info script]]]]] +package require homa::lc + +# Commands from Expect and tcllib, and the variables Expect sets. +set fixed { + log_user x + match_max x* + spawn x* + exp_send x* + expect x* + exp_continue x? + json::json2dict x +} + +# Arguments that are scripts, run in the caller's scope: methods as +# "Class method", procedures by their name in homa::lc. +set scripts { + {Table phase} {x x c} + {Node phase} {x x c} + waitFor {x x e} + withEnv {x x c} +} + +# tokens -- +# +# Returns nagelfar's syntax tokens for an argument list, as from info +# args or a method definition: x for each required argument, x? for each +# one with a default, x* for args. + +proc tokens {arguments} { + if {![llength $arguments]} { + return 0 + } + return [lmap arg $arguments { + if {[lindex $arg 0] eq "args"} { + string cat x* + } elseif {[llength $arg] > 1} { + string cat x? + } else { + string cat x + } + }] +} + +# procArgs -- +# +# Returns a procedure's argument list as proc takes it: each argument, +# or {argument default}. + +proc procArgs {cmd} { + return [lmap arg [info args $cmd] { + if {[info default $cmd $arg value]} {list $arg $value} else {set arg} + }] +} + +# syntax -- +# +# Prints one ##nagelfar directive. A space in the command name (an +# object type and its method) is escaped; the rest are plain words. + +proc syntax {directive name args} { + puts "##nagelfar $directive [string map {" " "\\ "} $name] [join $args]" +} + +dict for {name spec} $fixed { + syntax syntax $name {*}$spec +} +puts "##nagelfar implicitvarcmd {spawn *} spawn_id" + +# The ensembles, under their full names and as imported (or, for protocol, +# as console.tcl finds it through namespace path). +foreach {ensemble names} { + ::homa::lc::parse {homa::lc::parse parse} + ::homa::lc::protocol {homa::lc::protocol protocol} + ::homa::lc::report {homa::lc::report report} +} { + set map [namespace ensemble configure $ensemble -map] + if {![llength $map]} { + foreach sub [namespace eval $ensemble {namespace export}] { + dict set map $sub ${ensemble}::$sub + } + } + foreach name $names { + syntax syntax $name s x* + syntax subcmd $name {*}[lsort [dict keys $map]] + foreach sub [lsort [dict keys $map]] { + syntax syntax "$name $sub" {*}[tokens [procArgs [dict get $map $sub]]] + } + } +} + +# The package's own procedures, under their full names and as found from +# the classes' methods (namespace path). +foreach cmd [lsort [info procs ::homa::lc::*]] { + if {[dict exists $scripts [namespace tail $cmd]]} { + set spec [dict get $scripts [namespace tail $cmd]] + } else { + set spec [tokens [procArgs $cmd]] + } + syntax syntax [string trimleft $cmd :] {*}$spec + syntax syntax [namespace tail $cmd] {*}$spec +} + +# The classes: each object is a command whose subcommands are its public +# methods, and Class new returns one. +foreach class {Table Node Batch Console} { + set type _obj,$class + set methods [lsort [concat [info class methods homa::lc::$class] destroy]] + syntax syntax $type s x* + syntax subcmd $type {*}$methods + foreach method $methods { + if {[dict exists $scripts "$class $method"]} { + set spec [dict get $scripts "$class $method"] + } elseif {$method eq "destroy"} { + set spec 0 + } else { + set spec [tokens [lindex [info class definition homa::lc::$class $method] 0]] + } + syntax syntax "$type $method" {*}$spec + } + set ctor [lindex [info class constructor homa::lc::$class] 0] + syntax syntax "homa::lc::$class new" {*}[tokens $ctor] + syntax return "homa::lc::$class new" $type +} diff --git a/nix/microvms/tcl/tests/node.test b/nix/microvms/tcl/tests/node.test new file mode 100644 index 00000000..53ca2e35 --- /dev/null +++ b/nix/microvms/tcl/tests/node.test @@ -0,0 +1,172 @@ +# node.test -- +# +# Table-driven tests for homa::lc::Node: every phase a lifecycle test +# runs on a guest, against fakeqemu.tcl in place of QEMU. Every row is +# {description category input outcome expected}; see testTable in +# helpers.tcl. + +source [file join [file dirname [info script]] helpers.tcl] +package require homa::lc + +set fakeQemu [file join [file dirname [file normalize [info script]]] fakeqemu.tcl] + +# freePort -- +# +# Returns a local TCP port that nothing is listening on. + +proc freePort {} { + set server [socket -server {apply {args {}}} -myaddr 127.0.0.1 0] + set port [lindex [chan configure $server -sockname] 2] + close $server + return $port +} + +# script -- +# +# Writes an executable sh script. + +proc script {path body} { + homa::lc::writeFile $path "#!/bin/sh\n$body" + file attributes $path -permissions 0755 +} + +# newGuest -- +# +# Makes a Node for a fake guest, in a new work directory. +# +# Arguments: +# mode fakeqemu.tcl's mode, or missing for a runner that does not +# exist. +# +# Results: +# {table node}: a new homa::lc::Table and the Node, which records its +# phases there. dropGuest destroys them. + +proc newGuest {mode} { + set root [tcltest::makeDirectory work] + set bin [file join $root runner bin] + file mkdir $bin + set port [freePort] + if {$mode ne "missing"} { + script [file join $bin microvm-run] \ + [list exec [info nameofexecutable] $::fakeQemu $port $mode $bin] + } + script [file join $bin poweroff] {kill "$FAKE_QEMU_PID"} + script [file join $bin dmesg] { +echo '[ 0.100000] homa: loaded' +[ "$FAKE_MODE" = oops ] && echo '[ 1.000000] WARNING: CPU: 0 PID: 1 at homa_fake' +exit 0} + set vm [dict create runner [file join $root runner] serial 1 virtio $port] + set phaseTable [homa::lc::Table new] + return [list $phaseTable [homa::lc::Node new $phaseTable [file join $root guest] $vm \ + {start 2 serial 2 virtio 3 command 3 exit 3}]] +} + +# dropGuest -- +# +# Destroys what newGuest made: the Node (killing its QEMU), the Table and +# the work directory. + +proc dropGuest {phaseTable guest} { + $guest destroy + $phaseTable destroy + tcltest::removeDirectory work +} + +# bootNode -- +# +# Runs every Node phase, as homa-lifecycle does, on a fake guest. +# +# Arguments: +# mode as for newGuest. +# +# Results: +# The phases' ids and verdicts, in order: {1 PASS 2 PASS ...}. + +proc bootNode {mode} { + lassign [newGuest $mode] phaseTable guest + try { + lassign [quietly { + $guest start 1 + $guest serial 2 + $guest virtio 2b + $guest dmesg 5 + $guest shutdown 6 + $guest waitExit 7 + $guest serialLog 8 + }] - printed + } finally { + dropGuest $phaseTable $guest + } + return [concat {*}[lmap {- id result} \ + [regexp -all -inline -line {^ phase (\S+) \S+: (\w+)} $printed] {list $id $result}]] +} + +# nodePhase -- +# +# Boots a fake guest, then runs script as Node phase 3 "probe". +# +# Arguments: +# mode as for newGuest. +# closeFirst 1 to end the guest's shell (and so close hvc0) first. +# script the phase script; it may set ran. +# +# Results: +# {rc verdict detail ran}: what Node phase returned, the phase's verdict +# and detail as printed, and 1 if script ran. + +proc nodePhase {mode closeFirst script} { + lassign [newGuest $mode] phaseTable guest + set ran 0 + try { + lassign [quietly { + $guest start 1 + $guest serial 2 + $guest virtio 2b + if {$closeFirst} { + $guest run exit.log 3 exit + } + set rc [$guest phase 3 probe $script] + }] - printed + } finally { + dropGuest $phaseTable $guest + } + regexp -line {^ phase 3 probe: (\w+) \(\d+ ms\)(?:: (.*))?$} $printed -> verdict detail + return [list $rc $verdict $detail $ran] +} + +testTable nodePhases { + {"a guest that boots, answers and powers off" positive ok ok + {1 PASS 2 PASS 2b PASS 5 PASS 6 PASS 7 PASS 8 PASS}} + {"a kernel BUG fails dmesg and serial-log only" negative oops ok + {1 PASS 2 PASS 2b PASS 5 FAIL 6 PASS 7 PASS 8 FAIL}} + {"no shell on hvc0: its phases skip, QEMU never exits" negative silent ok + {1 PASS 2 PASS 2b FAIL 5 SKIP 6 SKIP 7 FAIL 8 PASS}} + {"console port never listens" negative deaf ok + {1 FAIL 2 PASS 2b FAIL 5 SKIP 6 SKIP 7 FAIL 8 PASS}} + {"a runner that cannot start" corner missing ok + {1 FAIL 2 FAIL 2b FAIL 5 SKIP 6 SKIP 7 FAIL 8 PASS}} +} { + bootNode $input +} + +# The input is {mode closeFirst script}; the result {rc verdict detail ran}. +testTable nodePhase { + {"with a shell, the script runs in the caller" positive + {ok 0 {set ran 1; list 0 fine}} ok {0 PASS fine 1}} + {"its failure is the phase's" positive + {ok 0 {set ran 1; list 2 bad}} ok {2 FAIL bad 1}} + {"no shell on hvc0: SKIP, and the script never runs" negative + {silent 0 {set ran 1; list 0 fine}} ok {skip SKIP {no shell on hvc0} 0}} + {"the shell has exited: SKIP" corner + {ok 1 {set ran 1; list 0 fine}} ok {skip SKIP {no shell on hvc0} 0}} + {"a script that returns gives its rc" boundary + {ok 0 {set ran 1; return [list 0 early]}} ok {0 PASS early 1}} + {"a command run in the script reaches the guest" positive + {ok 0 {set ran 1; list [dict get [$guest run p.log 3 {(exit 3)}] rc] {}}} ok + {3 FAIL {} 1}} +} { + nodePhase {*}$input +} + +tcltest::cleanupTests diff --git a/nix/microvms/tcl/tests/parse.test b/nix/microvms/tcl/tests/parse.test new file mode 100644 index 00000000..be2656a9 --- /dev/null +++ b/nix/microvms/tcl/tests/parse.test @@ -0,0 +1,285 @@ +# parse.test -- +# +# Table-driven tests for homa::lc::parse, the pure functions behind the +# lifecycle drivers. Every row is {description category input outcome +# expected}; see testTable in helpers.tcl. + +source [file join [file dirname [info script]] helpers.tcl] + +# ---- Text ------------------------------------------------------------------ + +testTable firstLine { + {"first of several lines" positive "one\ntwo\n" ok one} + {"a single line without newline" boundary one ok one} + {"empty text" boundary {} ok {}} + {"a leading empty line counts" corner "\nsecond" ok {}} +} {parse firstLine $input} + +testTable lines { + {"one entry per line" positive "a\nb\n" ok {a b}} + {"no final newline" positive "a\nb" ok {a b}} + {"empty text has no lines" boundary {} ok {}} + {"a blank line in the middle is kept" corner "a\n\nb" ok {a {} b}} + {"only newlines" corner "\n\n" ok {}} +} {parse lines $input} + +testTable lastLine { + {"last of several lines" positive "one\ntwo\n" ok two} + {"trailing empty lines are skipped" corner "one\ntwo\n\n\n" ok two} + {"a single line" boundary one ok one} + {"empty text" boundary {} ok {}} + {"only newlines" corner "\n\n" ok {}} +} {parse lastLine $input} + +# ---- Guest output ---------------------------------------------------------- + +testTable firstKernelLine { + {"a printk timestamp" positive "\[ 2.179107\] Linux version" ok {[ 2.179107]}} + {"firmware noise before the kernel" corner "SeaBIOS\n\[ 0.000000\] x" ok {[ 0.000000]}} + {"no kernel line yet" negative "SeaBIOS (version 1.16)\n" ok {}} + {"empty serial log" boundary {} ok {}} + {"a timestamp not at the start of a line" corner "x \[ 1.0\] y" ok {}} +} {parse firstKernelLine $input} + +testTable oops { + {"a clean dmesg" positive "homa: loaded\nhoma: unloaded\n" ok {}} + {"a BUG" negative "ok\nBUG: unable to handle page fault\nok" ok + {{BUG: unable to handle page fault}}} + {"a WARNING and a KASAN report" negative "WARNING: CPU: 1 PID: 9\nKASAN: use-after-free" ok + {{WARNING: CPU: 1 PID: 9} {KASAN: use-after-free}}} + {"a general protection fault" negative "Oops: general protection fault" ok + {{Oops: general protection fault}}} + {"lower-case warning is not an oops" corner "warning: something benign" ok {}} + {"WARNING without a colon is not an oops" corner "WARNING something" ok {}} + {"empty text" boundary {} ok {}} +} {parse oops $input} + +testTable oopsCount { + {"a clean dmesg" positive "homa: loaded\nhoma: unloaded\n" ok 0} + {"two oopses; the RIP line is not counted" corner + "BUG: x\nRIP: 0010:f\nWARNING: CPU: 1" ok 2} + {"one KASAN report" negative "KASAN: use-after-free" ok 1} + {"empty text" boundary {} ok 0} +} {parse oopsCount $input} + +testTable oopsAlso { + {"the RIP line comes with its oops, in order" positive + {"BUG: x\nCPU: 1\nRIP: 0010:homa_peer_free\nok" RIP:} ok + {{BUG: x} {RIP: 0010:homa_peer_free}}} + {"a RIP line alone is still returned" corner {"RIP: 0010:f" RIP:} ok {{RIP: 0010:f}}} + {"an empty extra pattern adds nothing" boundary {"RIP: 0010:f" {}} ok {}} + {"a bad extra pattern is an error" negative {"x" (} errorGlob + {* compile regular expression pattern: parentheses () not balanced}} +} {parse oops {*}$input} + +testTable smokeDetail { + {"a pass" positive {"HOMA_SMOKE_INSMOD_OK\nHOMA_SMOKE_PASS" 0} ok HOMA_SMOKE_PASS} + {"a failure names its step" negative {"HOMA_SMOKE_INSMOD_OK\nHOMA_SMOKE_FAIL:ECHO" 1} ok + HOMA_SMOKE_FAIL:ECHO} + {"the last verdict wins" corner {"HOMA_SMOKE_FAIL:ECHO\nHOMA_SMOKE_PASS" 0} ok HOMA_SMOKE_PASS} + {"no verdict: the last step reached" negative + {"HOMA_SMOKE_INSMOD_OK\nHOMA_SMOKE_ECHO_OK" 124} ok + {no verdict (rc 124), last step: HOMA_SMOKE_ECHO_OK}} + {"no output at all" boundary {{} 125} ok {no verdict (rc 125), last step: }} +} {parse smokeDetail {*}$input} + +testTable countLines { + {"counts matching lines" positive {{^cycle .* ok} "cycle 1 ok\ncycle 2 ok\n"} ok 2} + {"no match" negative {{^cycle .* ok} "insmod: error\n"} ok 0} + {"empty text" boundary {{^cycle .* ok} {}} ok 0} + {"a match must be anchored per line" corner {{^cycle .* ok} "x cycle 1 ok"} ok 0} +} {parse countLines {*}$input} + +# ---- Counters ---------------------------------------------------------------- + +set keys {packets_sent_DATA eth1_rx_packets rpc_timeouts} + +testTable snap [subst { + {"per-core lines are summed" positive + {"packets_sent_DATA 10 sent\npackets_sent_DATA 5 sent\neth1_rx_packets 7" {$keys}} ok + {packets_sent_DATA 15 eth1_rx_packets 7 rpc_timeouts 0}} + {"other counters are ignored" positive + {"unknown_rpcs 9\npackets_sent_DATA 1" {$keys}} ok + {packets_sent_DATA 1 eth1_rx_packets 0 rpc_timeouts 0}} + {"no input gives zeros" boundary {{} {$keys}} ok + {packets_sent_DATA 0 eth1_rx_packets 0 rpc_timeouts 0}} + {"a non-numeric value is skipped" corner + {"packets_sent_DATA n/a\npackets_sent_DATA 4" {$keys}} ok + {packets_sent_DATA 4 eth1_rx_packets 0 rpc_timeouts 0}} + {"64-bit counters do not overflow" boundary + {"packets_sent_DATA 9223372036854775807\npackets_sent_DATA 1" {$keys}} ok + {packets_sent_DATA 9223372036854775808 eth1_rx_packets 0 rpc_timeouts 0}} +}] {parse snap {*}$input} + +testTable delta { + {"only counters that moved" positive {{a 1 b 2 c 3} {a 1 b 5 c 0}} ok {b 3 c -3}} + {"nothing moved" boundary {{a 1} {a 1}} ok {}} + {"a counter missing after counts as 0" corner {{a 4} {}} ok {a -4}} + {"a counter new after counts from 0" corner {{} {a 4}} ok {a 4}} + {"new counters follow the first snap's order" corner + {{a 1 b 1} {c 2 a 2 d 3}} ok {a 1 b -1 c 2 d 3}} + {"a counter in both but unchanged is dropped, whatever the order" boundary + {{a 1 b 2} {b 2 a 1}} ok {}} + {"64-bit counters subtract exactly" boundary + {{a 0} {a 9223372036854775808}} ok {a 9223372036854775808}} + {"both empty" boundary {{} {}} ok {}} +} {parse delta {*}$input} + +testTable get { + {"a present counter" positive {{a 3 b 4} b} ok 4} + {"an absent counter is 0" negative {{a 3} b} ok 0} + {"empty deltas" boundary {{} b} ok 0} +} {parse get {*}$input} + +# ---- cp_node -------------------------------------------------------------------- + +set cpLine {1 clients: 0.40 Kops/sec, 0.33 Gbps, RTT (us) P50 148.99 P99 26162.00 P99.9 30000.00} + +testTable field [subst { + {"Kops after clients:" positive {{$cpLine} clients:} ok 0.40} + {"P99, not P99.9" corner {{$cpLine} P99} ok 26162.00} + {"the last word has no value" boundary {{$cpLine} 30000.00} ok -} + {"an absent word" negative {{$cpLine} P10} ok -} + {"an empty line" boundary {{} P99} ok -} + {"a caller's placeholder" positive {{} P99 —} ok —} +}] {parse field {*}$input} + +testTable positive { + {"a positive float" positive 26162.00 ok 1} + {"zero, as a stalled cp_node prints it" negative 0.00 ok 0} + {"the smallest positive value" boundary 0.01 ok 1} + {"a missing field" negative - ok 0} + {"an empty string" boundary {} ok 0} + {"a negative number" corner -1 ok 0} +} {parse positive $input} + +# The input is {want echoes homa tcp}. +testTable trafficFailures [subst { + {"everything completed" positive {6 6 {$cpLine} {$cpLine}} ok {}} + {"echo short of six" negative {6 5 {$cpLine} {$cpLine}} ok echo} + {"echo timed out (no count)" corner {6 {} {$cpLine} {$cpLine}} ok echo} + {"Homa and TCP stalled" negative {6 6 {P50 0.00 P99 0.00} {}} ok {homa tcp}} + {"everything failed" negative {6 0 {} {}} ok {echo homa tcp}} + {"a different echo count is honoured" positive {3 3 {$cpLine} {$cpLine}} ok {}} + {"more echoes than wanted fails" corner {6 7 {$cpLine} {$cpLine}} ok echo} + {"want 0 and no echoes passes" boundary {0 0 {$cpLine} {$cpLine}} ok {}} + {"a non-numeric echo count fails" corner + {6 {grep: no such file} {$cpLine} {$cpLine}} ok echo} + {"a bad want is an error" negative {x 6 {$cpLine} {$cpLine}} error + {bad echo count "x"}} +}] {parse trafficFailures {*}$input} + +# ---- tc netem --------------------------------------------------------------------- + +set tcNetem "qdisc netem 8001: root refcnt 2 limit 100000 loss 5% + Sent 123456 bytes 96410 pkt (dropped 4559, overlimits 0 requeues 0) + backlog 0b 0p requeues 0" +set tcHoma "qdisc homa 8006: root refcnt 2 + Sent 999 bytes 77 pkt (dropped 1, overlimits 0 requeues 0)" + +testTable netemSent [subst { + {"packets and drops of the netem qdisc" positive {$tcNetem} ok {96410 4559}} + {"another qdisc's counters are not netem's" negative {$tcHoma} ok {}} + {"netem after another qdisc" corner "$tcHoma\n$tcNetem" ok {96410 4559}} + {"a netem line without counters" corner "qdisc netem 8001: root" ok {}} + {"no output" boundary {} ok {}} +}] {parse netemSent $input} + +testTable dropRate { + {"no loss configured, none seen" positive {1000 0 0} ok {rate 0.00 ok 1}} + {"no loss configured, one drop" negative {1000 1 0} ok {rate 0.10 ok 0}} + {"1 % configured, 0.76 % seen (GSO)" positive {216477 1662 1} ok {rate 0.76 ok 1}} + {"exactly the upper bound 2l + 0.5" boundary {975 25 1} ok {rate 2.50 ok 1}} + {"just over the upper bound" boundary {974 26 1} ok {rate 2.60 ok 0}} + {"exactly a quarter of the loss over 10k packets" boundary {9975 25 1} ok {rate 0.25 ok 1}} + {"under a quarter of the loss over 10k packets" boundary {9976 24 1} ok {rate 0.24 ok 0}} + {"0.1 % read as 0.04 % (big GSO batches, slow kernel)" positive {81937 34 0.1} ok + {rate 0.04 ok 1}} + {"netem off: 1 % configured, no drops in 10k packets" negative {10000 0 1} ok {rate 0.00 ok 0}} + {"a small sample needs no lower bound" boundary {9999 0 1} ok {rate 0.00 ok 1}} + {"no packets at all" corner {0 0 1} ok {rate 0.00 ok 1}} + {"a fractional loss" positive {326224 187 0.1} ok {rate 0.06 ok 1}} + {"a non-numeric loss" negative {1000 0 abc} error {not a number: "abc"}} + {"a negative packet count" negative {-1 0 1} error {not a count: "-1"}} +} {parse dropRate {*}$input} + +set tcLoss0 "qdisc netem 8002: root refcnt 2 limit 100000 + Sent 1500 bytes 1000 pkt (dropped 1, overlimits 0 requeues 0)" + +# The input is {node text loss}. +testTable netemCheck [subst { + {"5 % loss measured as 4.52 % passes" positive {node1 {$tcNetem} 5} ok + {ok 1 report {node1 4559/100969 (4.52%)}}} + {"no netem qdisc: drops unknown, nothing to judge" negative + {node1 {$tcHoma} 5} ok {ok 1 report {node1 ?/0 (0.00%)}}} + {"no output at all" boundary {node0 {} 0} ok + {ok 1 report {node0 ?/0 (0.00%)}}} + {"0 % configured, 1 drop seen" negative {node0 {$tcLoss0} 0} ok + {ok 0 report {node0 1/1001 (0.10%)}}} + {"netem after another qdisc" corner {node1 "$tcHoma\n$tcNetem" 5} ok + {ok 1 report {node1 4559/100969 (4.52%)}}} + {"far more drops than configured" negative {node1 {$tcNetem} 1} ok + {ok 0 report {node1 4559/100969 (4.52%)}}} + {"a bad loss is an error" negative {node1 {$tcNetem} abc} error + {not a number: "abc"}} +}] {parse netemCheck {*}$input} + +testTable netemLimitOk { + {"the limit took" positive + {{qdisc netem 8001: root refcnt 2 limit 100000 delay 1ms} 100000} ok 1} + {"the limit at the end of the line" boundary + {{qdisc netem 8001: root refcnt 2 limit 100000} 100000} ok 1} + {"tc's default limit" negative + {{qdisc netem 8001: root refcnt 2 limit 1000 delay 1ms} 100000} ok 0} + {"a longer limit that starts with ours" corner + {{qdisc netem 8001: root refcnt 2 limit 1000000} 100000} ok 0} + {"on a later line" corner + "{qdisc fq_codel 0: root\nqdisc netem 1: root refcnt 2 limit 100000} 100000" ok 1} + {"no netem qdisc" negative {{qdisc homa 1: root limit 100000} 100000} ok 0} + {"empty output" boundary {{} 100000} ok 0} +} {parse netemLimitOk {*}$input} + +testTable netemDetail { + {"the arguments, without tc's prefix" positive + "qdisc netem 8001: root refcnt 2 limit 100000 delay 1ms seed 17\n" ok + {limit 100000 delay 1ms seed 17}} + {"no netem qdisc" negative "qdisc fq_codel 0: root" ok {}} + {"empty output" boundary {} ok {}} +} {parse netemDetail $input} + +# ---- A driver's output, for homa-vm-compare --------------------------------------- + +testTable shape { + {"a phase line keeps its id, name and verdict" positive + { phase 3 smoke: PASS (120 ms): HOMA_SMOKE_PASS} ok {{phase 3 smoke: PASS}}} + {"a phase without a detail" boundary + { phase 13 stop servers: PASS (1171 ms)} ok {{phase 13 stop servers: PASS}}} + {"a sub-phase whose name has numbers" corner + { phase 8.2 traffic loss0.1: FAIL (13633 ms): echo 0/6} ok + {{phase 8.2 traffic loss0.1: FAIL}}} + {"a skipped phase" positive + { phase 15 dmesg node0: SKIP (0 ms): no shell on hvc0} ok + {{phase 15 dmesg node0: SKIP}}} + {"a guest's HOMA line loses its prefix, with numbers masked" positive + {[ 42.824466] homa-smoke[545]: HOMA_SMOKE_ECHO responses=6 (want 6)} ok + {{HOMA_SMOKE_ECHO responses=N (want N)}}} + {"cp_node's line: decimals and numbers inside words are masked" corner + {HOMA_PAIR_CPNODE_homa 1793386978.012646352 homa_w4 clients: 0.39 Kops/sec} ok + {{HOMA_PAIR_CPNODE_homa N homa_wN clients: N Kops/sec}}} + {"the verdict is kept as it is, counts included" positive + {VERDICT: FAIL (8 phases)} ok {{VERDICT: FAIL (8 phases)}}} + {"table rows, counters and other lines are dropped" negative { +| 3 | smoke | PASS | 120 | HOMA_SMOKE_PASS | + node0: packets_sent_DATA=94551 +pair lifecycle-7_3: logs in /tmp/x} ok {}} + {"VERDICT must start a line" corner {x VERDICT: PASS} ok {}} + {"no output" boundary {} ok {}} + {"several lines keep their order" positive {pair lifecycle: logs in /tmp/x + phase 1 start: PASS (101 ms): qemu pid 1, consoles on 2/3 +HOMA_PAIR_ECHO responses=6 + phase 13 stop: PASS (1207 ms) +VERDICT: PASS} ok + {{phase 1 start: PASS} {HOMA_PAIR_ECHO responses=N} {phase 13 stop: PASS} {VERDICT: PASS}}} +} {parse shape $input} + +tcltest::cleanupTests diff --git a/nix/microvms/tcl/tests/protocol.test b/nix/microvms/tcl/tests/protocol.test new file mode 100644 index 00000000..74464510 --- /dev/null +++ b/nix/microvms/tcl/tests/protocol.test @@ -0,0 +1,67 @@ +# protocol.test -- +# +# Table-driven tests for homa::lc::protocol, the end-marker protocol +# homa::lc::Console speaks to a guest's shell. Every row is {description +# category input outcome expected}; see testTable in helpers.tcl. + +source [file join [file dirname [info script]] helpers.tcl] +package require homa::lc::protocol + +# The input is {cmd id}. +testTable wrap { + {"command gets a numbered end marker" positive + {{uname -r} 3} ok {uname -r; echo "HOMA_LC_""END 3 rc=$?"}} + {"the first command is number 1" boundary + {true 1} ok {true; echo "HOMA_LC_""END 1 rc=$?"}} + {"a compound command is wrapped as a whole" corner + {{a && b || c} 7} ok {a && b || c; echo "HOMA_LC_""END 7 rc=$?"}} + {"an id must be a positive integer" negative + {true 0} error {bad command id "0"}} + {"a non-numeric id is rejected" negative + {true x} error {bad command id "x"}} + {"a command on two lines is rejected" negative + "{true\nfalse} 1" error {command has more than one line}} + {"an empty command is rejected (the shell would see \"; echo\")" boundary + {{} 1} error {empty command}} + {"a blank command is rejected too" corner + {{ } 1} error {empty command}} +} {homa::lc::protocol wrap {*}$input} + +# The input is {id line}; the result {keep text done rc}. +testTable protocolLine { + {"plain output is kept" positive + {3 hello} ok {1 hello 0 {}}} + {"our marker alone ends the command" positive + {3 {HOMA_LC_END 3 rc=0}} ok {0 {} 1 0}} + {"a non-zero rc is returned as is" positive + {3 {HOMA_LC_END 3 rc=124}} ok {0 {} 1 124}} + {"output before our marker is kept, and ends it" corner + {3 {abcHOMA_LC_END 3 rc=2}} ok {1 abc 1 2}} + {"a stale marker alone is dropped, not the end" negative + {3 {HOMA_LC_END 2 rc=0}} ok {0 {} 0 {}}} + {"output before a stale marker is kept" corner + {3 {xHOMA_LC_END 2 rc=0}} ok {1 x 0 {}}} + {"a stale marker then ours on one line: ours ends it" corner + {3 {HOMA_LC_END 2 rc=1HOMA_LC_END 3 rc=0}} ok {0 {} 1 0}} + {"a trailing CR is ignored" corner + "3 {HOMA_LC_END 3 rc=2\r}" ok {0 {} 1 2}} + {"a CR on an ordinary line is removed" corner + "3 {x\r}" ok {1 x 0 {}}} + {"a blank line is kept" boundary + {3 {}} ok {1 {} 0 {}}} + {"a CR-only line is kept as blank" boundary + "3 {\r}" ok {1 {} 0 {}}} + {"id 33 does not end command 3" boundary + {3 {HOMA_LC_END 33 rc=0}} ok {0 {} 0 {}}} + {"id 13 does not end command 3" boundary + {3 {HOMA_LC_END 13 rc=0}} ok {0 {} 0 {}}} + {"the shell's echo of the command is kept, not matched" corner + {3 {true; echo "HOMA_LC_""END 3 rc=$?"}} ok + {1 {true; echo "HOMA_LC_""END 3 rc=$?"} 0 {}}} + {"a non-numeric id is an error" negative + {x hello} error {bad command id "x"}} + {"id 0 is an error" negative + {0 hello} error {bad command id "0"}} +} {homa::lc::protocol line {*}$input} + +tcltest::cleanupTests diff --git a/nix/microvms/tcl/tests/report.test b/nix/microvms/tcl/tests/report.test new file mode 100644 index 00000000..92d37754 --- /dev/null +++ b/nix/microvms/tcl/tests/report.test @@ -0,0 +1,168 @@ +# report.test -- +# +# Table-driven tests for homa::lc::report, which formats phase results, +# counter deltas and homa-vm-matrix's rows. Every row is {description +# category input outcome expected}; see testTable in helpers.tcl. + +source [file join [file dirname [info script]] helpers.tcl] +package require homa::lc::report +namespace import ::homa::lc::report + +set cpLine {1 clients: 0.40 Kops/sec, 0.33 Gbps, RTT (us) P50 148.99 P99 26162.00 P99.9 30000.00} + +# ---- Phase results --------------------------------------------------------- + +testTable verdict { + {"rc 0 passes" positive 0 ok PASS} + {"skip is a SKIP" positive skip ok SKIP} + {"rc 1 fails" negative 1 ok FAIL} + {"a console timeout (124) fails" boundary 124 ok FAIL} + {"a closed console (125) fails" boundary 125 ok FAIL} + {"anything else is rejected" negative maybe error {bad rc "maybe"}} + {"a negative rc fails" corner -1 ok FAIL} + {"an empty rc is rejected" boundary {} error {bad rc ""}} +} {report verdict $input} + +testTable combine { + {"two successes" positive {0 0} ok 0} + {"a failure after a success" positive {0 1} ok 1} + {"a success does not clear a failure" positive {1 0} ok 1} + {"the later failure wins" corner {1 124} ok 124} + {"a closed console after a failure" corner {2 125} ok 125} + {"a negative rc is a failure" corner {0 -1} ok -1} + {"a non-numeric rc is an error" negative {0 x} error {bad rc "x"}} + {"a non-numeric earlier rc is an error" negative {skip 0} error {bad rc "skip"}} +} {report combine {*}$input} + +testTable phaseLine { + {"with a detail" positive {3 smoke PASS 120 HOMA_SMOKE_PASS} ok + { phase 3 smoke: PASS (120 ms): HOMA_SMOKE_PASS}} + {"without a detail" boundary {13 {stop servers} PASS 1171 {}} ok + { phase 13 stop servers: PASS (1171 ms)}} + {"a sub-phase number" corner {8.5 {traffic loss5} FAIL 13706 {failed: tcp}} ok + { phase 8.5 traffic loss5: FAIL (13706 ms): failed: tcp}} +} {report phaseLine {*}$input} + +testTable tableRow { + {"with a detail" positive {3 smoke PASS 120 HOMA_SMOKE_PASS} ok + {| 3 | smoke | PASS | 120 | HOMA_SMOKE_PASS |}} + {"an empty detail keeps the column" boundary {13 {stop servers} PASS 1171 {}} ok + {| 13 | stop servers | PASS | 1171 | |}} + {"a pipe in the detail is escaped" corner {5 link PASS 9 {a | b}} ok + {| 5 | link | PASS | 9 | a \| b |}} +} {report tableRow {*}$input} + +# ---- Counters ---------------------------------------------------------------- + +testTable deltaLine { + {"name=value pairs" positive {a 3 b -2} ok {a=3 b=-2}} + {"no deltas" boundary {} ok {}} +} {report deltaLine $input} + +# ---- Traffic runs -------------------------------------------------------------- + +set tcpLine {1 clients: 0.39 Kops/sec, RTT (us) P50 410.63 P99 27362.55} +set lossChecks { + node0 {ok 1 report {node0 10/200 (5.00%)}} + node1 {ok 1 report {node1 9/180 (5.00%)}} +} +set tail0 {RESENDs 0, resent 0, rpc_timeouts 0, tcp_retrans 0} +set rates {echo 6/6; Homa 0.40 Kops P50 148.99 P99 26162.00 us; TCP 0.39 Kops P99 27362.55 us} + +# The input is {measured want must}: measured is a dictionary of echoes, +# homa and tcp (cp_node's lines), deltas (node -> counters) and checks +# (node -> netemCheck's result, or empty without netem). The result is +# {failed detail}. +testTable trafficDetail [subst { + {"a clean baseline" positive + {{echoes 6 homa {$cpLine} tcp {$tcpLine} deltas {node0 {} node1 {}} checks {}} 6 {}} + ok {failed {} detail {$rates; $tail0}}} + {"loss, with RESENDs summed over both nodes" positive + {{echoes 6 homa {$cpLine} tcp {$tcpLine} + deltas {node0 {packets_sent_RESEND 5} node1 {packets_sent_RESEND 7 resent_packets 3}} + checks {$lossChecks}} 6 packets_sent_RESEND} + ok {failed {} detail {$rates; netem drops node0 10/200 (5.00%) node1 9/180 (5.00%);\ + RESENDs 12, resent 3, rpc_timeouts 0, tcp_retrans 0; packets_sent_RESEND 12}}} + {"loss without RESENDs fails" negative + {{echoes 6 homa {$cpLine} tcp {$tcpLine} deltas {node0 {} node1 {}} + checks {$lossChecks}} 6 packets_sent_RESEND} + ok {failed no-packets_sent_RESEND detail {$rates; netem drops node0 10/200 (5.00%)\ + node1 9/180 (5.00%); $tail0; packets_sent_RESEND 0; failed: no-packets_sent_RESEND}}} + {"everything missing" boundary + {{echoes {} homa {} tcp {} deltas {} checks {}} 6 {}} + ok {failed {echo homa tcp} detail {echo /6; Homa - Kops P50 - P99 - us;\ + TCP - Kops P99 - us; $tail0; failed: echo homa tcp}}} + {"a drop check failed on one node" corner + {{echoes 6 homa {$cpLine} tcp {$tcpLine} deltas {node1 {packets_sent_RESEND 1}} + checks {node0 {ok 1 report {node0 1/100 (1.00%)}} + node1 {ok 0 report {node1 60/100 (60.00%)}}}} 6 packets_sent_RESEND} + ok {failed drops-node1 detail {$rates; netem drops node0 1/100 (1.00%)\ + node1 60/100 (60.00%); RESENDs 1, resent 0, rpc_timeouts 0, tcp_retrans 0;\ + packets_sent_RESEND 1; failed: drops-node1}}} + {"the qdisc counter that must move, on one node only" corner + {{echoes 6 homa {$cpLine} tcp {$tcpLine} deltas {node0 {qdisc_tcp_packets 40}} + checks {}} 6 qdisc_tcp_packets} + ok {failed {} detail {$rates; $tail0; qdisc_tcp_packets 40}}} + {"a counter that went down does not count as moved" corner + {{echoes 6 homa {$cpLine} tcp {$tcpLine} deltas {node0 {pacer_homa_packets -2}} + checks {}} 6 pacer_homa_packets} + ok {failed no-pacer_homa_packets detail {$rates; $tail0; pacer_homa_packets -2;\ + failed: no-pacer_homa_packets}}} + {"a bad echo count is an error" negative + {{echoes 6 homa {} tcp {} deltas {} checks {}} x {}} error {bad echo count "x"}} +}] {report trafficDetail {*}$input} + +# ---- homa-vm-matrix ----------------------------------------------------------------- + +set pairOut "pair lifecycle: logs in /tmp/x +HOMA_PAIR_ECHO responses=6 (want 6) +HOMA_PAIR_CPNODE_homa $cpLine +HOMA_PAIR_CPNODE_tcp 1 clients: 0.39 Kops/sec, RTT (us) P50 410.63 P99 27362.55 +VERDICT: PASS" +set smokeOut "HOMA_SMOKE_ECHO responses=6 (want 6) +HOMA_SMOKE_GRO homa_gro_receive calls=1722 +VERDICT: PASS" + +# The input is {name kernels secs want out}. +testTable matrixRow [subst { + {"a pair test" positive {homa-pair-test-7_3-7_3 {7_3 → 7_3} 65 6 {$pairOut}} ok + {verdict PASS row {| homa-pair-test-7_3-7_3 | 7_3 → 7_3 | PASS | 65 | 6/6 | — |\ + 0.40 | 148.99 / 26162.00 | 0.39 | 410.63 / 27362.55 |}}} + {"a single-VM test has no cp_node" positive + {homa-microvm-test-7_3 7_3 51 6 {$smokeOut}} ok + {verdict PASS row {| homa-microvm-test-7_3 | 7_3 | PASS | 51 | 6/6 | 1722 |\ + — | — / — | — | — / — |}}} + {"a failed run" negative {t 7_3 9 6 "VERDICT: FAIL (3 phases)"} ok + {verdict FAIL row {| t | 7_3 | FAIL | 9 | 0/6 | — | — | — / — | — | — / — |}}} + {"a run with no output" boundary {t 7_3 0 6 {}} ok + {verdict FAIL row {| t | 7_3 | FAIL | 0 | 0/6 | — | — | — / — | — | — / — |}}} + {"VERDICT: PASS must start a line" corner {t 7_3 1 6 "x VERDICT: PASS"} ok + {verdict FAIL row {| t | 7_3 | FAIL | 1 | 0/6 | — | — | — / — | — | — / — |}}} + {"the cell shows the wanted count" positive + {t 7_3 2 3 "HOMA_SMOKE_ECHO responses=3 (want 3)\nVERDICT: PASS"} ok + {verdict PASS row {| t | 7_3 | PASS | 2 | 3/3 | — | — | — / — | — | — / — |}}} + {"a missing count shows 0 of the wanted count" negative + {t 7_3 2 3 "VERDICT: PASS"} ok + {verdict PASS row {| t | 7_3 | PASS | 2 | 0/3 | — | — | — / — | — | — / — |}}} +}] {report matrixRow {*}$input} + +# ---- homa-vm-compare ------------------------------------------------------------------ + +# The input is {old new}, two lists of lines. +testTable diff { + {"the same lines" positive {{a b c} {a b c}} ok {}} + {"a changed verdict" negative + {{{phase 3 smoke: PASS} b} {{phase 3 smoke: FAIL} b}} ok + {{- phase 3 smoke: PASS} {+ phase 3 smoke: FAIL}}} + {"a line added in the middle" positive {{a b} {a c b}} ok {{+ c}}} + {"a line removed" positive {{a b c} {a c}} ok {{- b}}} + {"both empty" boundary {{} {}} ok {}} + {"nothing before" boundary {{} {a b}} ok {{+ a} {+ b}}} + {"nothing after" boundary {{a} {}} ok {{- a}}} + {"a repeated line, once fewer" corner {{a a b} {a b}} ok {{- a}}} + {"two lines swapped" corner {{a b} {b a}} ok {{- a} {+ a}}} + {"lines with spaces are compared whole" corner + {{{phase 1 a: PASS}} {{phase 1 a: PASS}}} ok {}} +} {report diff {*}$input} + +tcltest::cleanupTests diff --git a/nix/microvms/tcl/tests/style.test b/nix/microvms/tcl/tests/style.test new file mode 100644 index 00000000..3f164ee9 --- /dev/null +++ b/nix/microvms/tcl/tests/style.test @@ -0,0 +1,235 @@ +# style.test -- +# +# Checks the package's sources against the Tcl Engineering Manual's +# conventions this package keeps: every procedure, method and constructor +# has a header with Arguments: (when it takes any), Results: and Side +# effects:, and nothing uses eval or exports with a pattern. Every row is +# {description category input outcome expected}; see testTable in +# helpers.tcl. + +source [file join [file dirname [info script]] helpers.tcl] + +# headerGaps -- +# +# Finds the procedures, methods and constructors whose header comment +# lacks a section. The header is the block of comment lines just above +# the declaration (blank lines between them are allowed). +# +# Arguments: +# text Tcl source. +# +# Results: +# The names of the commands with a gap, in order ("constructor" for a +# constructor), or an empty list. +# +# Side effects: +# None. + +proc headerGaps {text} { + set lines [split $text \n] + set gaps {} + for {set i 0} {$i < [llength $lines]} {incr i} { + set decl [string trim [lindex $lines $i]] + if {![regexp {^(proc|method|constructor)\s} $decl]} { + continue + } + # Without the body's opening brace, the declaration is a list. + if {[string index $decl end] eq "\{"} { + set decl [string range $decl 0 end-1] + } + if {[catch {llength $decl}]} { + continue + } + if {[lindex $decl 0] eq "constructor"} { + lassign [list constructor [lindex $decl 1]] name params + } else { + lassign [lrange $decl 1 2] name params + } + set j [expr {$i - 1}] + while {$j >= 0 && [string trim [lindex $lines $j]] eq ""} { + incr j -1 + } + set header {} + while {$j >= 0 && [regexp {^\s*#} [lindex $lines $j]]} { + set header "[lindex $lines $j]\n$header" + incr j -1 + } + set want {Results: {Side effects:}} + if {[llength $params]} { + lappend want Arguments: + } + foreach section $want { + if {![regexp -line "^\\s*#\\s*$section" $header]} { + lappend gaps $name + break + } + } + } + return $gaps +} + +# banned -- +# +# Finds the constructs this package does not use, outside comments: eval +# (build a list and run it, or use uplevel or tailcall) and "namespace +# export *" (name each public command). +# +# Arguments: +# text Tcl source. +# +# Results: +# A list with one entry per use: "eval" or "export *". +# +# Side effects: +# None. + +proc banned {text} { + set found {} + foreach line [split $text \n] { + if {[regexp {^\s*#} $line]} { + continue + } + # namespace eval runs a script in a namespace; it builds nothing. + regsub -all {namespace\s+eval\s} $line {} line + foreach match [regexp -all -inline {(?:^|[\s\[;\{])eval\s} $line] { + lappend found eval + } + if {[regexp {namespace\s+export\s+.*\*} $line]} { + lappend found {export *} + } + } + return $found +} + +testTable headerGaps { + {"a full header" positive { +# foo -- +# +# Does foo. +# +# Arguments: +# x a value. +# +# Results: +# None. +# +# Side effects: +# None. + +proc foo {x} {} +} ok {}} + {"no arguments, no Arguments: section needed" positive { +# foo -- +# +# Results: +# 1. +# +# Side effects: +# None. + +proc foo {} {return 1} +} ok {}} + {"no Side effects: section" negative { +# foo -- +# +# Arguments: +# x a value. +# +# Results: +# None. + +proc foo {x} {} +} ok foo} + {"arguments but no Arguments: section" negative { +# foo -- +# +# Results: +# None. +# +# Side effects: +# None. + +proc foo {x {y 1}} {} +} ok foo} + {"no header at all" negative {proc foo {} {}} ok foo} + {"methods and a constructor, indented, in a class" corner { +oo::class create C { + # constructor -- + # + # Arguments: + # x a value. + # + # Side effects: + # Sets x. + + constructor {x} { + } + + # get -- + # + # Results: + # x. + # + # Side effects: + # None. + + method get {} { + } +} +} ok constructor} + {"only the block just above counts: the next proc's is its own" corner { +# a -- +# +# Results: +# None. +# +# Side effects: +# None. + +proc a {} {} +proc b {} {} +} ok b} + {"a declaration in a comment is not one" corner {# proc foo {} {}} ok {}} + {"an empty file" boundary {} ok {}} +} { + headerGaps $input +} + +testTable banned { + {"eval as a command" negative {eval $cmd} ok eval} + {"eval in a substitution" negative {set x [eval $cmd]} ok eval} + {"a pattern export" negative {namespace export *} ok {{export *}}} + {"uplevel with a list is fine" positive {uplevel 1 [list expr $c]} ok {}} + {"eval in a comment is not a use" corner {# never eval a string} ok {}} + {"namespace eval is not eval" corner {namespace eval ::a {variable x}} ok {}} + {"but eval inside namespace eval is" negative {namespace eval ::a {eval $s}} ok eval} + {"a word that starts with eval is not eval" boundary {set evaluated 1} ok {}} + {"an explicit export list" positive {namespace export wrap line} ok {}} + {"an empty file" boundary {} ok {}} +} { + banned $input +} + +# Every source file: {headerGaps banned}, both empty. +testTable sources { + {"parse.tcl" positive parse.tcl ok {{} {}}} + {"protocol.tcl" positive protocol.tcl ok {{} {}}} + {"report.tcl" positive report.tcl ok {{} {}}} + {"console.tcl" positive console.tcl ok {{} {}}} + {"lc.tcl" positive lc.tcl ok {{} {}}} + {"bin/lifecycle.tcl" positive bin/lifecycle.tcl ok {{} {}}} + {"bin/pair-lifecycle.tcl" positive bin/pair-lifecycle.tcl ok {{} {}}} + {"bin/vm-matrix.tcl" positive bin/vm-matrix.tcl ok {{} {}}} + {"bin/vm-compare.tcl" positive bin/vm-compare.tcl ok {{} {}}} +} { + set chan [open [file join [file dirname $::here] $input]] + try { + chan configure $chan -encoding utf-8 + set text [read $chan] + } finally { + close $chan + } + list [headerGaps $text] [banned $text] +} + +tcltest::cleanupTests diff --git a/nix/microvms/tcl/tests/table.test b/nix/microvms/tcl/tests/table.test new file mode 100644 index 00000000..137e6fb8 --- /dev/null +++ b/nix/microvms/tcl/tests/table.test @@ -0,0 +1,157 @@ +# table.test -- +# +# Table-driven tests for homa::lc::Table, the phase table every driver +# prints and the matrix parses. Every row is {description category input +# outcome expected}; see testTable in helpers.tcl. The input is a list of +# phases, each {id name script}, run in order; times are replaced by N so +# the expected output is fixed. + +source [file join [file dirname [info script]] helpers.tcl] +package require homa::lc + +# runTable -- +# +# Runs phases through a new Table in a new directory, then its verdict. +# +# Arguments: +# phases a list of {id name script}; each script returns {rc detail}. +# +# Results: +# A dictionary: rcs (what each phase returned), exit (the verdict's exit +# code), out (what was printed) and rows (summary.md's rows, without +# its header), with each time replaced by N, and errors (what +# phase-errors.log holds, or "absent"). + +proc runTable {phases} { + set dir [tcltest::makeDirectory run] + set phaseTable [homa::lc::Table new] + set here [pwd] + cd $dir + try { + lassign [quietly { + list [lmap phase $phases { + lassign $phase id name script + $phaseTable phase $id $name $script + }] [$phaseTable verdict] + }] results printed + set summary [homa::lc::readFile summary.md] + set errorLog [expr {[file exists phase-errors.log] + ? [homa::lc::readFile phase-errors.log] : "absent"}] + } finally { + cd $here + $phaseTable destroy + tcltest::removeDirectory run + } + lassign $results phaseRcs code + return [dict create rcs $phaseRcs exit $code errors $errorLog \ + out [regsub -all {\(\d+ ms\)} $printed {(N ms)}] \ + rows [lmap row [lrange [parse lines $summary] 2 end] { + regsub {^((?:\|[^|]*){3}\|) \d+ \|} $row {\1 N |} + }]] +} + +testTable tableVerdict { + {"every phase passes" positive {{1 start {list 0 ok}} {2 serial {list 0 {}}}} ok + {{0 0} 0 {VERDICT: PASS}}} + {"one failure fails the run" negative {{1 start {list 0 {}}} {2 serial {list 1 late}}} ok + {{0 1} 1 {VERDICT: FAIL (1 phases)}}} + {"a skip is not a failure" corner {{3 smoke {list skip {no shell}}}} ok + {skip 0 {VERDICT: PASS}}} + {"timeouts and eof are failures" negative + {{3 smoke {list 124 {}}} {4 cycles {list 125 {}}}} ok + {{124 125} 1 {VERDICT: FAIL (2 phases)}}} + {"an error in a phase fails it, not the run" negative + {{3 smoke {error boom}} {4 cycles {list 0 {}}}} ok + {{1 0} 1 {VERDICT: FAIL (1 phases)}}} + {"no phases at all" boundary {} ok + {{} 0 {VERDICT: PASS}}} + {"a result without an rc fails its phase, not the run" corner + {{1 start {}} {2 serial {list 0 {}}}} ok + {{1 0} 1 {VERDICT: FAIL (1 phases)}}} + {"an rc that is not a number fails its phase" negative {{1 start {list ok {}}}} ok + {1 1 {VERDICT: FAIL (1 phases)}}} + {"a phase script may return its result" positive {{1 start {return [list 0 ok]}}} ok + {0 0 {VERDICT: PASS}}} + {"a returned failure is recorded" negative {{1 start {return [list 1 bad]}}} ok + {1 1 {VERDICT: FAIL (1 phases)}}} + {"break fails the phase, not the run" corner {{1 a break} {2 b {list 0 {}}}} ok + {{1 0} 1 {VERDICT: FAIL (1 phases)}}} + {"continue fails the phase, not the run" corner {{1 a continue} {2 b {list 0 {}}}} ok + {{1 0} 1 {VERDICT: FAIL (1 phases)}}} + {"a bare return fails its phase" boundary {{1 a return}} ok + {1 1 {VERDICT: FAIL (1 phases)}}} +} { + set result [runTable $input] + list [dict get $result rcs] [dict get $result exit] \ + [lindex [split [string trimright [dict get $result out] \n] \n] end] +} + +testTable tableOutput { + {"a phase line with its detail" positive {{3 smoke {list 0 HOMA_SMOKE_PASS}}} ok + { phase 3 smoke: PASS (N ms): HOMA_SMOKE_PASS}} + {"no detail, no colon" boundary {{2b virtio {list 0 {}}}} ok + { phase 2b virtio: PASS (N ms)}} + {"an error's message is the detail" negative {{3 smoke {error {no such file}}}} ok + { phase 3 smoke: FAIL (N ms): error: no such file}} + {"a bad rc is named in the detail" negative {{3 smoke {list ok {}}}} ok + { phase 3 smoke: FAIL (N ms): error: bad rc "ok"}} + {"break is named in the detail" corner {{3 smoke break}} ok + { phase 3 smoke: FAIL (N ms): error: break in phase script}} + {"continue is named in the detail" corner {{3 smoke continue}} ok + { phase 3 smoke: FAIL (N ms): error: continue in phase script}} + {"a bare return's empty rc is named" boundary {{3 smoke return}} ok + { phase 3 smoke: FAIL (N ms): error: bad rc ""}} + {"a returned result is printed like any other" positive {{3 smoke {return {0 done}}}} ok + { phase 3 smoke: PASS (N ms): done}} + {"a dotted id and a name with spaces" corner {{8.3 {netem loss node1} {list 0 {}}}} ok + { phase 8.3 netem loss node1: PASS (N ms)}} +} { + lindex [split [dict get [runTable $input] out] \n] 0 +} + +testTable tableSummary { + {"one row per phase, in order" positive {{1 start {list 0 {}}} {2 serial {list 1 late}}} ok + {{| 1 | start | PASS | N | |} {| 2 | serial | FAIL | N | late |}}} + {"no phases: just the header" boundary {} ok {}} + {"a pipe in the detail is escaped" corner {{3 smoke {list 0 a|b}}} ok + {{| 3 | smoke | PASS | N | a\|b |}}} +} { + dict get [runTable $input] rows +} + +# The input is the phases. The result is "absent" if there is no +# phase-errors.log, else the heading of each entry in it and whether it holds +# a Tcl stack trace. +testTable tableErrorLog { + {"no error, no phase-errors.log" positive {{1 a {list 0 {}}}} ok absent} + {"an error's stack goes to phase-errors.log" negative {{3 smoke {error boom}}} ok + {{{phase 3 smoke: boom}} 1}} + {"a bad rc is not a script error: no log" corner {{1 a {list ok {}}}} ok absent} + {"break is not a script error: no log" corner {{1 a break}} ok absent} + {"two errors are both logged, in order" boundary + {{1 a {error one}} {2 b {error two}}} ok + {{{phase 1 a: one} {phase 2 b: two}} 1}} +} { + set errors [dict get [runTable $input] errors] + if {$errors eq "absent"} { + return absent + } + list [lmap {- heading} [regexp -all -inline -line {^(phase .*)$} $errors] { + set heading + }] [string match "*invoked from within*" $errors] +} + +testTable tableScope { + {"a phase script sees and sets the caller's variables" positive 41 ok 42} +} { + set count $input + set phaseTable [homa::lc::Table new] + try { + quietly {$phaseTable phase 1 count {incr count; list 0 {}}} + } finally { + $phaseTable destroy + } + set count +} + +tcltest::cleanupTests diff --git a/nix/microvms/tcl/tests/wait.test b/nix/microvms/tcl/tests/wait.test new file mode 100644 index 00000000..cdbd3a11 --- /dev/null +++ b/nix/microvms/tcl/tests/wait.test @@ -0,0 +1,94 @@ +# wait.test -- +# +# Table-driven tests for homa::lc::waitFor, which the Node phases use to +# wait for QEMU's console port, the first kernel line and QEMU's exit +# while still handling events. Every row is {description category input +# outcome expected}; see testTable in helpers.tcl. + +source [file join [file dirname [info script]] helpers.tcl] +package require homa::lc + +# timing -- +# +# Classifies how long a wait of ms milliseconds took: "at once" (under +# 20 ms), "before deadline", "at deadline" (within 100 ms after it), or +# "late". +# +# Results: +# One of those four strings. +# +# Side effects: +# None. + +proc timing {elapsed ms} { + if {$elapsed < 20} { + return "at once" + } elseif {$elapsed < $ms} { + return "before deadline" + } elseif {$elapsed < $ms + 100} { + return "at deadline" + } + return late +} + +# The input is {ms interval at ?condition?}: ::ready becomes 1 after at ms +# (0: before the wait, never: not at all), and the condition defaults to +# $::ready. The result is {waitFor's result, timing}. +testTable waitFor { + {"already true: returns at once" positive {1000 50 0} ok {1 {at once}}} + {"true part-way through" positive {1000 50 200} ok {1 {before deadline}}} + {"never true: times out at the deadline" negative {200 50 never} ok {0 {at deadline}}} + {"0 ms and false: one check, at once" boundary {0 50 never} ok {0 {at once}}} + {"0 ms and true: one check, at once" boundary {0 50 0} ok {1 {at once}}} + {"true after the last interval: the check at the deadline sees it" boundary + {200 50 190} ok {1 {at deadline}}} + {"an interval longer than the wait does not overshoot it" corner {100 1000 never} ok + {0 {at deadline}}} + {"the condition is an expression in the caller's scope" corner + {1000 50 0 {$local == 7}} ok {1 {at once}}} + {"an error in the condition propagates" negative {1000 50 0 {[error boom]}} error boom} + {"a negative wait is an error" negative {-1 50 0} error {bad wait "-1"}} + {"an interval of 0 is an error" negative {100 0 0} error {bad interval "0"}} +} { + lassign $input ms interval at condition + if {$condition eq ""} { + set condition {$::ready} + } + set local 7 + set ::ready [expr {$at eq "0"}] + if {$at ni {0 never}} { + set timer [after $at {set ::ready 1}] + } + set start [clock milliseconds] + try { + set result [homa::lc::waitFor $ms $interval $condition] + } finally { + if {[info exists timer]} { + after cancel $timer + } + } + list $result [timing [expr {[clock milliseconds] - $start}] $ms] +} + +# A wait started from an event handler while another wait runs. Each must +# wait its own time: the inner one is not cut short when the outer one's +# deadline passes, and the outer one still ends once the inner one returns. +# The input is {outer inner}, in ms; the result is {outer's result, inner's +# result, inner's timing}. +testTable waitForNested { + {"a longer inner wait is not cut short by the outer one" corner {100 300} ok + {0 0 {at deadline}}} + {"a shorter inner wait ends on its own time" corner {300 100} ok + {0 0 {at deadline}}} +} { + lassign $input outer inner + after 10 [list apply {{inner} { + set start [clock milliseconds] + set ::innerResult [homa::lc::waitFor $inner 50 0] + set ::innerTiming [timing [expr {[clock milliseconds] - $start}] $inner] + }} $inner] + set outerResult [homa::lc::waitFor $outer 50 0] + list $outerResult $::innerResult $::innerTiming +} + +tcltest::cleanupTests diff --git a/nix/parts/README.md b/nix/parts/README.md new file mode 100644 index 00000000..f14c97b0 --- /dev/null +++ b/nix/parts/README.md @@ -0,0 +1,25 @@ +# nix/parts: the flake's outputs, one file per area + +[`flake.nix`](../../flake.nix) is short: it names its inputs and lists these files. Each one +is a [flake-parts](https://flake.parts) module that sets some of the flake's outputs. The +outputs are what `nix build .#<name>`, `nix run`, `nix develop` and `nix flake check` find. + +| File | Outputs | +|---|---| +| `scope.nix` | none: builds the package scope ([`../scope.nix`](../scope.nix)) and passes it to the others as `homa` | +| `build.nix` | `homa-module*`, `homa-build-matrix`, `homa-utils*`, `homa-tests`, `homa-tests-coverage`, `homa-image*`, cross builds (x86_64 hosts only); `default` is `homa-module` | +| `analysis.nix` | `analysis-*`; check `analysis-quick` | +| `fuzz.nix` | `fuzz*`; check `fuzz-selftest` | +| `vms.nix` | `homa-microvm*`, `homa-*-test*`, `homa-*lifecycle*`, `homa-pair-*`, `homa-vm-matrix`; check `tcl-tests` | +| `evidence.nix` | `homa-evidence`, `homa-evidence-full` | +| `dev.nix` | `devShells.default` (`nix develop`) and `formatter` (`nix fmt`) | + +## How a part works + +`perSystem` is a function that flake-parts calls once for each system in `flake.nix` +(`x86_64-linux`, `aarch64-linux`), with `pkgs` for that system. What it returns becomes +`packages.<system>.*`, `checks.<system>.*` and so on. Several parts may each set +`packages`; flake-parts merges them, and a name defined twice is an error. + +The parts hold no build logic. Each one picks outputs from the scope (`homa.analysisPkgs` +and so on), so the build code under `nix/` does not depend on flake-parts. diff --git a/nix/parts/analysis.nix b/nix/parts/analysis.nix new file mode 100644 index 00000000..98bc9bb2 --- /dev/null +++ b/nix/parts/analysis.nix @@ -0,0 +1,9 @@ +# nix/parts/analysis.nix +# +# Static analysis (../analysis): analysis-quick is a check, the rest are packages. + +{ + perSystem = { homa, ... }: { + inherit (homa.analysisPkgs) packages checks; + }; +} diff --git a/nix/parts/build.nix b/nix/parts/build.nix new file mode 100644 index 00000000..1b5be025 --- /dev/null +++ b/nix/parts/build.nix @@ -0,0 +1,16 @@ +# nix/parts/build.nix +# +# homa.ko for every kernel (and cross-compiled on x86_64 hosts), the util/ tools, the unit +# tests, and the OCI image. `nix build` with no target builds homa-module. + +{ + perSystem = { homa, lib, system, ... }: { + packages = + homa.modulePkgs.packages + // homa.userspacePkgs.packages + // homa.ociPkgs.packages + # The cross toolchains are built for x86_64 hosts only. + // lib.optionalAttrs (system == "x86_64-linux") homa.crossPkgs.packages + // { default = homa.modulePkgs.homa-module; }; + }; +} diff --git a/nix/parts/dev.nix b/nix/parts/dev.nix new file mode 100644 index 00000000..40290fa1 --- /dev/null +++ b/nix/parts/dev.nix @@ -0,0 +1,10 @@ +# nix/parts/dev.nix +# +# `nix develop` and `nix fmt`. + +{ + perSystem = { homa, pkgs, ... }: { + devShells.default = homa.devShell; + formatter = pkgs.nixpkgs-fmt; + }; +} diff --git a/nix/parts/evidence.nix b/nix/parts/evidence.nix new file mode 100644 index 00000000..7478c4d7 --- /dev/null +++ b/nix/parts/evidence.nix @@ -0,0 +1,10 @@ +# nix/parts/evidence.nix +# +# One report of every check (../evidence): homa-evidence, and homa-evidence-full with the +# VM tests. + +{ + perSystem = { homa, ... }: { + inherit (homa.evidencePkgs) packages; + }; +} diff --git a/nix/parts/fuzz.nix b/nix/parts/fuzz.nix new file mode 100644 index 00000000..b4b12928 --- /dev/null +++ b/nix/parts/fuzz.nix @@ -0,0 +1,9 @@ +# nix/parts/fuzz.nix +# +# The libFuzzer harnesses (../fuzz): fuzz-selftest is a check, the rest are packages. + +{ + perSystem = { homa, ... }: { + inherit (homa.fuzzPkgs) packages checks; + }; +} diff --git a/nix/parts/scope.nix b/nix/parts/scope.nix new file mode 100644 index 00000000..3760d683 --- /dev/null +++ b/nix/parts/scope.nix @@ -0,0 +1,12 @@ +# nix/parts/scope.nix +# +# Builds the Homa package scope (../scope.nix) once per system and hands it to every other +# part as the perSystem argument `homa`. + +{ inputs, ... }: + +{ + perSystem = { pkgs, ... }: { + _module.args.homa = import ../scope.nix { inherit pkgs inputs; }; + }; +} diff --git a/nix/parts/vms.nix b/nix/parts/vms.nix new file mode 100644 index 00000000..379ea247 --- /dev/null +++ b/nix/parts/vms.nix @@ -0,0 +1,9 @@ +# nix/parts/vms.nix +# +# The QEMU guests and the tests that drive them (../microvms); tcl-tests is a check. + +{ + perSystem = { homa, ... }: { + inherit (homa.microvmPkgs) packages checks; + }; +} diff --git a/nix/patches/default.nix b/nix/patches/default.nix new file mode 100644 index 00000000..f08ce9af --- /dev/null +++ b/nix/patches/default.nix @@ -0,0 +1,12 @@ +# nix/patches/default.nix +# +# Upstream-pending fixes the flake needs in order to build Homa against its default kernel. +# Each patch is a fork draft PR's commit, exported with `git format-patch`. The list is empty +# while upstream `main` builds on the default kernel as-is (the 7.1 port, fork draft #12, landed +# upstream as e65a5c0 on 2026-10-27). +# +# homaLib (../lib) applies them to every source subset, each patch cut down to the files in +# that subset. Once upstream `main` carries a fix, its patch stops applying and the build +# fails at patchPhase: that is the cue to delete it from this list. + +[ ] diff --git a/nix/scope.nix b/nix/scope.nix new file mode 100644 index 00000000..12a6a829 --- /dev/null +++ b/nix/scope.nix @@ -0,0 +1,52 @@ +# nix/scope.nix +# +# Every Homa derivation, in one package scope (lib.makeScope). Each file below is called +# with callPackage, which passes a file the arguments its argument list names: from this +# scope first, then from nixpkgs. A file states what it needs, and nothing has to thread +# pkgs, lib and kernelConfig through by hand. The flake-parts modules in parts/ read their +# outputs from here: +# +# homa = import ./scope.nix { inherit pkgs inputs; }; +# homa.modulePkgs.homa-module +# +# To swap a piece (another kernel, say), override the scope: homa.overrideScope (final: +# prev: { kernelConfig = ...; }) rebuilds everything that depends on it. + +{ pkgs, inputs }: + +pkgs.lib.makeScope pkgs.newScope (self: { + # homaLib (lib/): version, meta, the source subsets with patches/ applied, mkKbuild, + # perKernel. + homaLib = self.callPackage ./lib { }; + + # Foundation: the kernels, kernel trees from source, tool lists, dev environment. + kernelConfig = self.callPackage ./kernel/config.nix { }; + kernelSrc = self.callPackage ./kernel/src.nix { }; + packagesModule = self.callPackage ./dev/packages.nix { + inherit (self.kernelConfig) kernelPackages; + }; + inherit (self.packagesModule) pythonEnv; + envVars = self.callPackage ./dev/env-vars.nix { }; + shellFunctions = self.callPackage ./dev/shell-functions { }; + # The merged kernel tree that the unit tests and the mock fuzzers build against. + unitKdir = self.callPackage ./kernel/unit-kdir { }; + + # Each of these returns { packages, checks?, ... } for a part in parts/. + modulePkgs = self.callPackage ./build/module.nix { }; + crossPkgs = self.callPackage ./build/cross.nix { + inherit (self.modulePkgs) buildAgainstTree; + }; + userspacePkgs = self.callPackage ./build/userspace.nix { }; + ociPkgs = self.callPackage ./build/oci.nix { + inherit (self.userspacePkgs) homa-utils homa-utils-debug; + }; + analysisPkgs = self.callPackage ./analysis { }; + fuzzPkgs = self.callPackage ./fuzz { }; + microvmPkgs = self.callPackage ./microvms { + inherit (inputs) microvm nixpkgs; + inherit (self.modulePkgs) buildModule; + homaUtils = self.userspacePkgs.homa-utils; + }; + evidencePkgs = self.callPackage ./evidence { }; + devShell = self.callPackage ./dev/shell.nix { }; +})