From 20b65c83ce6db0d4920537d81e9455a3078155e6 Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Fri, 30 Oct 2026 18:20:05 -0700 Subject: [PATCH 01/11] nix: add a flake that builds Homa for many kernels and architectures An optional Nix layer, entirely under nix/ plus flake.nix and flake.lock: the Makefiles and sources are untouched, and nothing changes for anyone who doesn't use Nix. - nix build .#homa-module: homa.ko against a pinned kernel, and homa-module- for every nixpkgs kernel (6.1 to 7.3) plus kernel.org 7.0-7.2 trees (compile only); homa-build-matrix builds them all and writes a summary - homa-module-riscv64/-aarch64 and homa-cross-matrix: cross-compiled homa.ko and util/ - homa-utils (and an AddressSanitizer variant), homa-tests (the unit tests under ASan), homa-image (an OCI image of the tools) - nix develop: a shell with the kernel build deps and analysis tools, KDIR pointing at the pinned kernel, and homa-build/homa-test/homa-checkpatch helpers - nix/patches: a place for upstream-pending fixes the build needs (empty) Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 + flake.lock | 86 +++++++++++++ flake.nix | 58 +++++++++ nix/build/README.md | 35 ++++++ nix/build/cross.nix | 90 ++++++++++++++ nix/build/module.nix | 156 ++++++++++++++++++++++++ nix/build/oci.nix | 52 ++++++++ nix/build/userspace.nix | 155 +++++++++++++++++++++++ nix/dev/README.md | 22 ++++ nix/dev/env-vars.nix | 26 ++++ nix/dev/packages.nix | 42 +++++++ nix/dev/shell-functions/default.nix | 33 +++++ nix/dev/shell.nix | 34 ++++++ nix/kernel/README.md | 20 +++ nix/kernel/config.nix | 60 +++++++++ nix/kernel/src.nix | 106 ++++++++++++++++ nix/kernel/unit-kdir/default.nix | 54 ++++++++ nix/kernel/unit-kdir/homa_unit_compat.h | 7 ++ nix/kernel/unit-kdir/patch-autoconf.sh | 13 ++ nix/lib/README.md | 34 ++++++ nix/lib/default.nix | 92 ++++++++++++++ nix/parts/README.md | 21 ++++ nix/parts/build.nix | 16 +++ nix/parts/dev.nix | 10 ++ nix/parts/scope.nix | 12 ++ nix/patches/default.nix | 12 ++ nix/scope.nix | 44 +++++++ 27 files changed, 1292 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/build/README.md create mode 100644 nix/build/cross.nix create mode 100644 nix/build/module.nix create mode 100644 nix/build/oci.nix create mode 100644 nix/build/userspace.nix create mode 100644 nix/dev/README.md create mode 100644 nix/dev/env-vars.nix create mode 100644 nix/dev/packages.nix create mode 100644 nix/dev/shell-functions/default.nix create mode 100644 nix/dev/shell.nix create mode 100644 nix/kernel/README.md create mode 100644 nix/kernel/config.nix create mode 100644 nix/kernel/src.nix create mode 100644 nix/kernel/unit-kdir/default.nix create mode 100644 nix/kernel/unit-kdir/homa_unit_compat.h create mode 100755 nix/kernel/unit-kdir/patch-autoconf.sh create mode 100644 nix/lib/README.md create mode 100644 nix/lib/default.nix create mode 100644 nix/parts/README.md create mode 100644 nix/parts/build.nix create mode 100644 nix/parts/dev.nix create mode 100644 nix/parts/scope.nix create mode 100644 nix/patches/default.nix create mode 100644 nix/scope.nix diff --git a/.gitignore b/.gitignore index d646a9cd..b1fcae02 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ reports/ traces/ users/ saved_traces/ +result +result-* diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..4cfbe867 --- /dev/null +++ b/flake.lock @@ -0,0 +1,86 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1790898470, + "narHash": "sha256-zTwuwezD0w3hpga6a6P8emidzAZFyWqlNeRBlpWOOl8=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "024633cd702b10285db5cb19b40ad48d2399ba60", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "microvm": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "spectrum": "spectrum" + }, + "locked": { + "lastModified": 1791551030, + "narHash": "sha256-jTx3O3MKqPjv+cDfBFQ5zlKECBEwzF91scC+mSumZyE=", + "owner": "astro", + "repo": "microvm.nix", + "rev": "f13928fe9900e91709d7ce581eefa47eaafa79e1", + "type": "github" + }, + "original": { + "owner": "astro", + "repo": "microvm.nix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1792502884, + "narHash": "sha256-IoNisKcGbUjdd45qCqywGMCYmZ5EnkxezvLII0FWwNA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "02c087f3fe673c9e317f206a627cb6b910336dbe", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "microvm": "microvm", + "nixpkgs": "nixpkgs" + } + }, + "spectrum": { + "flake": false, + "locked": { + "lastModified": 1786032750, + "narHash": "sha256-+Ba/kIn5BjvV7PN8bWOo7RoqgTY/Yv2M7qeMf3wS2Ag=", + "ref": "refs/heads/main", + "rev": "77d2389874bb092b8dbf05ecc5b1030372e80e98", + "revCount": 1417, + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + }, + "original": { + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..e459f797 --- /dev/null +++ b/flake.nix @@ -0,0 +1,58 @@ +# +# flake.nix for Homa (HomaModule) — a modular Nix layer over the out-of-tree kernel module. +# +# ── Usage ────────────────────────────────────────────────────────────────────────────── +# nix develop # dev shell (toolchain + analysis tools) +# +# nix build .#homa-module # build homa.ko (release) against the pinned kernel +# nix build .#homa-module-debug # build homa.ko (debug: -g -O1) +# nix build .#homa-module-6_12 # per kernel: 6_1 6_6 6_12 6_18 7_3 (nixpkgs), +# # 7_0 7_1 7_2 (kernel.org source, compile only) +# nix build .#homa-build-matrix # every kernel above -> summary.md (never fails) +# nix build .#homa-module-riscv64 # cross-compiled homa.ko: riscv64, aarch64 (+ -strip) +# nix build .#homa-cross-matrix # both arches: homa.ko + util/ report -> summary.md +# nix build .#linux-kdir-riscv64-7_3 # the prepared kernel tree a cross build uses (+ aarch64) +# HOMA_LINUX_SRC=~/Downloads/linux nix build --impure .#homa-module-local +# nix build .#homa-utils # user-space tools from util/ +# nix build .#homa-utils-debug # user-space tools with AddressSanitizer +# nix build .#homa-tests # build + run the unit tests (ASan) +# +# nix build .#homa-image # OCI userspace image (docker load < result) +# nix build .#homa-image-debug +# +# nix flake show # list every output +# nix flake check # evaluate/build the checks +# +# If flakes are not enabled globally: +# nix --extra-experimental-features 'nix-command flakes' develop . +# ─────────────────────────────────────────────────────────────────────────────────────── +{ + description = "Homa: modular Nix flake (module + userspace derivations, microVM, static analysis, fuzzing)"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-parts = { + url = "github:hercules-ci/flake-parts"; + inputs.nixpkgs-lib.follows = "nixpkgs"; + }; + microvm = { + url = "github:astro/microvm.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + # flake-parts merges the modules in nix/parts/, one per area, each adding its packages + # and checks for every system (nix/parts/README.md). nix/scope.nix builds the + # derivations themselves. + outputs = inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + # Homa is a Linux kernel module; restrict to Linux systems (nixpkgs unstable also no + # longer supports x86_64-darwin). + systems = [ "x86_64-linux" "aarch64-linux" ]; + imports = [ + ./nix/parts/scope.nix + ./nix/parts/build.nix + ./nix/parts/dev.nix + ]; + }; +} diff --git a/nix/build/README.md b/nix/build/README.md new file mode 100644 index 00000000..4779b3fb --- /dev/null +++ b/nix/build/README.md @@ -0,0 +1,35 @@ +# nix/build: homa.ko, the user-space tools, the unit tests, the OCI image + +| File | Outputs | +|---|---| +| `module.nix` | `homa-module` (the default kernel), `homa-module-debug` (`-g -O1`), `homa-module-` for every nixpkgs kernel, `homa-module-local` (your tree, `--impure`), and `homa-build-matrix`, which tries every kernel and writes `summary.md` | +| `userspace.nix` | `homa-utils[-debug]` (`util/`; `-debug` adds ASan) and `homa-tests`, which builds and runs the unit tests in `test/` | +| `cross.nix` | `homa-module-{riscv64,aarch64}[-strip]`, `homa-utils-{riscv64,aarch64}`, `linux-kdir--` and `homa-cross-matrix` (x86_64 hosts only) | +| `oci.nix` | `homa-image[-debug]`: the user-space tools in an OCI image (`docker load < result`) | + +```sh +nix build .#homa-module # result/lib/modules//homa.ko +nix build .#homa-build-matrix && cat result/summary.md +nix build .#homa-tests # fails if a unit test fails +``` + +## How the module build works + +Homa's own `Makefile` does the work, as it would by hand: `make KDIR=`. +`homaLib.mkKbuild` ([`../lib`](../lib)) wraps that in a derivation: the source is +`homaLib.src.module`, the kernel tree comes from [`../kernel`](../kernel), and kbuild's +own flags replace nixpkgs' hardening flags. A build is one function call per kernel: + +```nix +buildModule { kernelPackages = pkgs.linuxPackages_6_12; enableDebug = true; } +``` + +`homa-build-matrix` does not fail when a kernel fails. Its job is the table: which kernels +Homa builds on today. + +## Cross builds + +`cross.nix` uses nixpkgs' `pkgsCross.`, a copy of nixpkgs whose compilers target +another architecture. Calling `userspace.nix` through `pkgsCross.riscv64.callPackage` is +all it takes to cross-compile the tools; the module builds against a riscv64 or aarch64 +kernel tree prepared from source. diff --git a/nix/build/cross.nix b/nix/build/cross.nix new file mode 100644 index 00000000..5d06e941 --- /dev/null +++ b/nix/build/cross.nix @@ -0,0 +1,90 @@ +# nix/build/cross.nix +# +# Cross-compile checks (xdp2 pattern): build Homa for other architectures on an x86_64 +# host, with nixpkgs' cached cross toolchains (pkgsCross), and no emulation. +# +# homa-module- homa.ko for the default kernel (7.3), cross-compiled against a +# prepared tree (kernel/src.nix mkTree) using nixpkgs' +# kernel config; checks homa.ko's ELF machine +# homa-module--strip ... the same for the upstream-shaped source (__STRIP__=1) +# homa-utils- util/ (cp_node, demo_*, ...) built with the cross toolchain +# homa-cross-matrix every arch: module results plus a `make -k` report of util/, +# in summary.md; fails only if a homa.ko does not build +# +# is riscv64 or aarch64. The kernel trees take about 2 minutes each (modules_prepare, +# no vmlinux), versus about 30 for a full cross kernel build. +# +# nix/scope.nix calls this file with callPackage, passing buildAgainstTree from module.nix. + +{ pkgs +, lib +, kernelConfig +, kernelSrc +, homaLib +, unitKdir +, buildAgainstTree +}: + +let + # arch: the kernel's ARCH=; machine: what readelf -h prints for the target. + targets = { + riscv64 = { cross = pkgs.pkgsCross.riscv64; arch = "riscv"; machine = "RISC-V"; }; + aarch64 = { cross = pkgs.pkgsCross.aarch64-multiplatform; arch = "arm64"; machine = "AArch64"; }; + }; + + # Same source as the default x86 kernel, so every arch builds the same kernel version. + kernel = kernelConfig.kernelPackages.kernel; + series = kernelConfig.seriesOf kernelConfig.kernelPackages; + + perTarget = name: t: + let + cc = t.cross.stdenv.cc; + tree = kernelSrc.mkTree { + name = "${name}-${series}"; + inherit (kernel) version src; + configfile = t.cross.linuxPackages_latest.kernel.configfile; + makeFlags = [ "ARCH=${t.arch}" "CROSS_COMPILE=${cc.targetPrefix}" ]; + extraInputs = [ cc ]; + }; + module = buildAgainstTree name tree { inherit (t) machine; }; + moduleStrip = buildAgainstTree "${name}-strip" tree { inherit (t) machine; strip = true; }; + # The same file as the native tools, called from the cross package set, so its + # stdenv is the cross one. + userspace = t.cross.callPackage ./userspace.nix { inherit homaLib unitKdir; }; + utils = userspace.homa-utils; + utilsReport = userspace.buildUtils { report = true; }; + in + { inherit tree module moduleStrip utils utilsReport; inherit (t) machine; }; + + all = lib.mapAttrs perTarget targets; + + homa-cross-matrix = pkgs.runCommand "homa-cross-matrix" { } '' + mkdir -p "$out" + { + echo "# Cross-compile matrix (Linux ${kernel.version})" + ${lib.concatMapStrings (name: let a = all.${name}; in '' + echo + echo "## ${name}" + echo + echo "- homa.ko: PASS (${a.machine}), ${a.module}" + echo "- homa.ko (__STRIP__=1): PASS (${a.machine}), ${a.moduleStrip}" + echo "- compiler warnings in homa.ko: $(grep -cE "^[^ ]+:[0-9]+:[0-9]+: warning:" ${a.module}/build.log || true)" + echo + cat ${a.utilsReport}/report.md + cp ${a.utilsReport}/build.log "$out/utils-${name}.log" + '') (lib.attrNames all)} + } > "$out/summary.md" + cat "$out/summary.md" + ''; +in +{ + packages = lib.concatMapAttrs + (name: a: { + "homa-module-${name}" = a.module; + "homa-module-${name}-strip" = a.moduleStrip; + "homa-utils-${name}" = a.utils; + "linux-kdir-${name}-${series}" = a.tree; + }) + all + // { inherit homa-cross-matrix; }; +} diff --git a/nix/build/module.nix b/nix/build/module.nix new file mode 100644 index 00000000..c0f7a4c2 --- /dev/null +++ b/nix/build/module.nix @@ -0,0 +1,156 @@ +# nix/build/module.nix +# +# The homa.ko out-of-tree kernel module, built with the standard nixpkgs kbuild idiom +# (kernel.moduleBuildDependencies + KDIR pointing at the kernel's build tree). +# +# Debug vs non-debug is a single boolean (`enableDebug`), mirroring xdp2's `enableAsserts`: +# release: MY_CFLAGS='-O2' +# debug: MY_CFLAGS='-g -O1 -fno-omit-frame-pointer' +# Homa has no formal DEBUG macro, so at the C level "debug" means opt-level/symbols only. +# Real memory-safety coverage comes from the microVM's KASAN kernel (see docs/DESIGN.md §4). +# +# `buildModule` is also reused by nix/microvms/ to build the .ko against the SAME +# kernelPackages the guest boots (co-pinning avoids vermagic drift). +# +# Usage (nix/scope.nix calls this file with callPackage): +# modulePkgs.homa-module # release .ko (default kernel) +# modulePkgs.packages # { homa-module, homa-module-debug, homa-module-..., +# # homa-build-matrix } +# modulePkgs.buildModule { kernelPackages = ; enableDebug = false; } +# +# Per-kernel targets: +# homa-module- nixpkgs kernels (kernel/config.nix matrix): a real .ko, loadable in the +# matching microVM. +# homa-module- kernel-src trees (7.0-7.2, local): compile check only, built with +# KBUILD_MODPOST_WARN=1 because those trees have no Module.symvers. +# homa-build-matrix one derivation that tries every kernel above and writes a +# pass/fail table plus logs; it never fails itself. + +{ pkgs +, lib +, homaLib +, kernelConfig +, kernelSrc +}: + +let + inherit (homaLib) mkKbuild perKernel; + + buildModule = { kernelPackages, enableDebug ? false }: + let + kernel = kernelPackages.kernel; + modDirVersion = kernel.modDirVersion; + myCflags = if enableDebug then "-g -O1 -fno-omit-frame-pointer" else "-O2"; + in + mkKbuild { + pname = "homa-module${lib.optionalString enableDebug "-debug"}"; + # The top Makefile's default target `all` runs: + # $(MAKE) -C $(KDIR) M=$(shell pwd) modules + # so we only need to point KDIR at this kernel's build tree. + kdir = kernelConfig.kdirOf kernelPackages; + nativeBuildInputs = kernel.moduleBuildDependencies; + env.MY_CFLAGS = myCflags; + + installPhase = '' + runHook preInstall + mkdir -p "$out/lib/modules/${modDirVersion}" + cp homa.ko "$out/lib/modules/${modDirVersion}/" + # Record what this .ko was built against, for the microVM / debugging. + { + echo "modDirVersion=${modDirVersion}" + echo "enableDebug=${lib.boolToString enableDebug}" + echo "MY_CFLAGS=${myCflags}" + } > "$out/lib/modules/${modDirVersion}/homa.ko.buildinfo" + runHook postInstall + ''; + + # Never strip or patchelf a kernel module. + dontStrip = true; + dontPatchELF = true; + + meta = homaLib.meta // { + description = "Homa transport protocol Linux kernel module (homa.ko)" + + lib.optionalString enableDebug " [debug]"; + }; + }; + + homa-module = buildModule { inherit (kernelConfig) kernelPackages; }; + homa-module-debug = buildModule { inherit (kernelConfig) kernelPackages; enableDebug = true; }; + + # Compile-test matrix: homa-module- against each nixpkgs kernel (fallback for + # API drift on the newest kernel). + matrixPkgs = perKernel "homa-module" + (_: kp: buildModule { kernelPackages = kp; }) + kernelConfig.matrix; + + # Compile-only builds against kernels prepared from source (nix/kernel/src.nix). A cross + # tree (cross.nix) brings its ARCH=/CROSS_COMPILE= flags and its cross gcc along. + # strip: build the upstream-shaped source (make __STRIP__=1) + # machine: if set, readelf must report this e_machine for homa.ko (e.g. "RISC-V") + buildAgainstTree = name: tree: { strip ? false, machine ? null }: mkKbuild { + pname = "homa-module-${name}"; + kdir = "${tree}/build"; + nativeBuildInputs = tree.nativeBuildInputs; + buildInputs = tree.buildInputs; + makeFlags = [ "KBUILD_MODPOST_WARN=1" ] ++ tree.makeFlags + ++ lib.optional strip "__STRIP__=1"; + # Keep the log, so homa-cross-matrix can count compiler warnings. + buildPhase = '' + runHook preBuild + make -j"$NIX_BUILD_CORES" $makeFlags 2>&1 | tee build.log + runHook postBuild + ''; + installPhase = '' + ${lib.optionalString (machine != null) '' + readelf -h homa.ko | grep -q 'Machine: *${machine}' \ + || { readelf -h homa.ko; echo "homa.ko is not ${machine}"; exit 1; } + ''} + mkdir -p "$out" + cp homa.ko build.log "$out/" + ''; + dontFixup = true; + }; + srcPkgs = perKernel "homa-module" + (name: tree: buildAgainstTree name tree { }) + kernelSrc.trees; + + # Every kernel, one report. Each build runs in its own copy of the source; a failure is + # recorded, not propagated, so one broken kernel does not hide the others. + matrixKdirs = + lib.mapAttrsToList (n: kp: { name = n; version = kp.kernel.version; kdir = kernelConfig.kdirOf kp; extra = ""; }) + kernelConfig.matrix + ++ lib.mapAttrsToList (n: t: { name = n; version = t.version; kdir = "${t}/build"; extra = "KBUILD_MODPOST_WARN=1"; }) + kernelSrc.trees; + + homa-build-matrix = mkKbuild { + pname = "homa-build-matrix"; + nativeBuildInputs = kernelConfig.kernelPackages.kernel.moduleBuildDependencies + ++ (with pkgs; [ elfutils openssl ]); + buildPhase = '' + mkdir -p "$out/logs" + printf '| Kernel | Version | Result |\n|---|---|---|\n' > "$out/summary.md" + fail=0 + ${lib.concatMapStrings (k: '' + rm -rf "../w-${k.name}"; cp -r . "../w-${k.name}"; chmod -R u+w "../w-${k.name}" + if make -C "../w-${k.name}" KDIR=${k.kdir} ${k.extra} \ + -j"$NIX_BUILD_CORES" > "$out/logs/${k.name}.log" 2>&1; then + r=PASS + else + r=FAIL; fail=$((fail + 1)) + fi + echo "| ${k.name} | ${k.version} | $r |" >> "$out/summary.md" + rm -rf "../w-${k.name}" + '') matrixKdirs} + echo "$fail" > "$out/failures" + cat "$out/summary.md" + ''; + dontInstall = true; + dontFixup = true; + }; +in +{ + inherit buildModule buildAgainstTree homa-module; + packages = { + inherit homa-module homa-module-debug homa-build-matrix; + } // srcPkgs // matrixPkgs; +} diff --git a/nix/build/oci.nix b/nix/build/oci.nix new file mode 100644 index 00000000..bdbc84cc --- /dev/null +++ b/nix/build/oci.nix @@ -0,0 +1,52 @@ +# nix/build/oci.nix +# +# A lightweight OCI image bundling Homa's user-space tools + Python tooling (secondary to +# the microVM, which is the real kernel-module test vehicle). Uses dockerTools.buildLayeredImage +# (reproducible, layer-cached) per the xdp2 pattern. +# +# nix build .#homa-image && docker load < result +# nix build .#homa-image-debug && docker load < result +# +# In the scope (nix/scope.nix) as ociPkgs. + +{ pkgs +, lib +, pythonEnv +, homa-utils +, homa-utils-debug +}: + +let + mkImage = { name, utils }: + pkgs.dockerTools.buildLayeredImage { + inherit name; + tag = "latest"; + created = "1970-01-01T00:00:00Z"; # reproducible + + contents = [ + utils + pythonEnv + pkgs.bashInteractive + pkgs.coreutils + ]; + + config = { + Cmd = [ "/bin/bash" ]; + Env = [ "MPLBACKEND=Agg" ]; + Labels = { + "org.opencontainers.image.title" = name; + "org.opencontainers.image.description" = + "Homa user-space tools and Python analysis tooling"; + "org.opencontainers.image.source" = + "https://github.com/PlatformLab/HomaModule"; + "org.opencontainers.image.licenses" = "BSD-2-Clause"; + }; + }; + }; + + homa-image = mkImage { name = "homa-image"; utils = homa-utils; }; + homa-image-debug = mkImage { name = "homa-image-debug"; utils = homa-utils-debug; }; +in +{ + packages = { inherit homa-image homa-image-debug; }; +} diff --git a/nix/build/userspace.nix b/nix/build/userspace.nix new file mode 100644 index 00000000..1c8a943f --- /dev/null +++ b/nix/build/userspace.nix @@ -0,0 +1,155 @@ +# nix/build/userspace.nix +# +# User-space derivations: +# homa-utils - the util/ benchmark/test binaries (release, mirrors util/Makefile -O3) +# homa-utils-debug - the same, with AddressSanitizer injected via the cc-wrapper +# homa-tests - build + run the test/ kselftest-style unit harness (ASan) +# +# The test harness compiles the kernel C files in user space against the pinned kernel's +# headers, so it needs KDIR + ARCH=x86 (test/Makefile hardcodes x86 include paths). This is +# kernel-version sensitive; if it fails against the newest kernel that is recorded in +# nix/STATUS.md and the static-analysis report (see docs/DESIGN.md §5). +# +# nix/scope.nix calls this file with callPackage (and cross.nix from each cross package set). + +{ pkgs +, lib +, homaLib +, unitKdir +}: + +let + inherit (homaLib) version; + + # report = true (cross.nix) runs `make -k` and never fails: $out/report.md says which + # util/ objects and programs built, with each failing file's first error. + buildUtils = { debug ? false, report ? false }: + pkgs.stdenv.mkDerivation { + pname = "homa-utils${lib.optionalString debug "-debug"}${lib.optionalString report "-report"}"; + inherit version; + src = homaLib.src.util; + + nativeBuildInputs = [ pkgs.gnumake ]; + hardeningDisable = [ "all" ]; + + # Inject debug/ASan without editing util/Makefile (the cc-wrapper honours these). + # NIX_CFLAGS_LINK, not NIX_LDFLAGS: these go to gcc when it links, NIX_LDFLAGS to ld, + # which rejects -fsanitize. + env = lib.optionalAttrs debug { + NIX_CFLAGS_COMPILE = "-g -fno-omit-frame-pointer -fsanitize=address"; + NIX_CFLAGS_LINK = "-fsanitize=address"; + }; + + dontConfigure = true; + + # util/Makefile calls g++ by name. When cross-compiling (cross.nix) that must be the + # target compiler, so put a g++ that is $CXX first on PATH. + buildPhase = '' + runHook preBuild + ${lib.optionalString (pkgs.stdenv.hostPlatform != pkgs.stdenv.buildPlatform) '' + mkdir -p .cross-bin + ln -s "$(command -v "$CXX")" .cross-bin/g++ + ln -s "$(command -v "$CC")" .cross-bin/gcc + ln -s "$(command -v "$CC")" .cross-bin/cc + export PATH="$PWD/.cross-bin:$PATH" + ''} + ${if report then '' + mkdir -p "$out" + make -C util -k -j"$NIX_BUILD_CORES" all > "$out/build.log" 2>&1 || true + { + echo "| File | Result | First error |" + echo "|---|---|---|" + for f in util/*.c util/*.cc homa_receiver.cc; do + o="util/$(basename "''${f%.*}").o" + if [ -f "$o" ]; then + echo "| $f | PASS | |" + else + # Rebuild it alone: with -j the errors interleave, and some are in headers. + err=$(make -C util "$(basename "$o")" 2>&1 \ + | grep -m 1 -E '^[^ ]+:[0-9]+:[0-9]+: error:' | sed 's/|/\\|/g' || true) + echo "| $f | FAIL | $err |" + fi + done + echo + bins=$(make -s -C util -p -n all 2>/dev/null | sed -n 's/^BINS := //p') + built=0 + for b in $bins; do [ -x "util/$b" ] && built=$((built + 1)); done + echo "Programs built: $built of $(echo "$bins" | wc -w)" + } > "$out/report.md" + '' else '' + make -C util -j"$NIX_BUILD_CORES" all + ''} + runHook postBuild + ''; + + dontInstall = report; + installPhase = '' + runHook preInstall + mkdir -p "$out/bin" + # Install every executable the Makefile produced under util/. + for f in util/*; do + if [ -f "$f" ] && [ -x "$f" ]; then + install -m755 "$f" "$out/bin/" + fi + done + runHook postInstall + ''; + + meta = homaLib.meta // { + description = "Homa user-space test/benchmark tools (util/)" + + lib.optionalString debug " [ASan]"; + }; + }; + + homa-utils = buildUtils { }; + homa-utils-debug = buildUtils { debug = true; }; + + homa-tests = pkgs.stdenv.mkDerivation { + pname = "homa-tests"; + inherit version; + src = homaLib.src.test; + + # test/linux_objs (picks the vendored rbtree/rhashtable/xxhash for the kernel version) + # is a #!/bin/bash script, which does not exist inside the build sandbox. + postPatch = "patchShebangs test/linux_objs"; + + nativeBuildInputs = [ pkgs.gnumake pkgs.perl pkgs.bash ]; + hardeningDisable = [ "all" ]; + + dontConfigure = true; + + buildPhase = '' + runHook preBuild + # unitKdir merges nixpkgs' split source/ + build/ trees and adapts the kernel + # config to the mock layer (see nix/kernel/unit-kdir/default.nix). + make -C test \ + KDIR=${unitKdir} \ + CC="gcc ${unitKdir.extraCFlags}" \ + -j"$NIX_BUILD_CORES" unit + runHook postBuild + ''; + + # Run the unit tests as the check step (default IPv6 path). + doCheck = true; + checkPhase = '' + runHook preCheck + (cd test && ./unit) + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + mkdir -p "$out/bin" + install -m755 test/unit "$out/bin/homa-unit" + runHook postInstall + ''; + + meta = homaLib.meta // { + description = "Homa kselftest-style unit tests (built + run under AddressSanitizer)"; + }; + }; +in +{ + inherit buildUtils homa-utils homa-utils-debug; + packages = { inherit homa-utils homa-utils-debug homa-tests; }; +} diff --git a/nix/dev/README.md b/nix/dev/README.md new file mode 100644 index 00000000..babe45e3 --- /dev/null +++ b/nix/dev/README.md @@ -0,0 +1,22 @@ +# nix/dev: the development shell + +`nix develop` opens a bash shell with every tool on `PATH` and `$KDIR` pointing at the +default kernel's build tree. Nothing is installed on the host; leaving the shell leaves +no trace. + +| File | In the scope as | What it is | +|---|---|---| +| `packages.nix` | `packagesModule` | the tool lists: build inputs, analysis and debugging tools, `pythonEnv` (python3 with the packages Homa's scripts import), and the Tcl drivers' tools (Tcl 8.6, Expect, tcllib, nagelfar, tclint; Tcl's setup hook sets `TCLLIBPATH`) | +| `env-vars.nix` | `envVars` | shell code that exports `KDIR`, `LINUX_VERSION`, `ARCH`, and puts `pythonEnv` first on `PATH` | +| `shell-functions/` | `shellFunctions` | `homa-build`, `homa-test`, `homa-clean`, `homa-checkpatch`, `homa-lint` | +| `shell.nix` | `devShell` | the `mkShell` that puts them together | + +```sh +nix develop +homa-build # make against $KDIR +homa-test # the unit tests +homa-checkpatch # the kernel's checkpatch.pl +homa-lint # the analysis-quick tier: ruff, shellcheck, clang-format +``` + +`nix fmt` formats Nix files with `nixpkgs-fmt` ([`../parts/dev.nix`](../parts/dev.nix)). diff --git a/nix/dev/env-vars.nix b/nix/dev/env-vars.nix new file mode 100644 index 00000000..67f2c179 --- /dev/null +++ b/nix/dev/env-vars.nix @@ -0,0 +1,26 @@ +# nix/dev/env-vars.nix +# +# A shell-script fragment (returned as a string) that the dev shell splices into its +# shellHook. Sets the environment Homa's Makefiles expect. +# +# In the scope (nix/scope.nix) as envVars; dev/shell.nix puts it in the shellHook. + +{ kernelConfig +, pythonEnv +}: + +'' + # Homa's test/Makefile hardcodes x86 include paths. + export ARCH="''${ARCH:-x86}" + + # Default KDIR to the flake-pinned kernel, but let callers override it (e.g. to build + # against the running host kernel: make KDIR=/lib/modules/$(uname -r)/build). + export KDIR="''${KDIR:-${kernelConfig.kdir}}" + export LINUX_VERSION="''${LINUX_VERSION:-${kernelConfig.modDirVersion}}" + + # Headless matplotlib for the plotting scripts. + export MPLBACKEND="''${MPLBACKEND:-Agg}" + + # Make the pinned python (with matplotlib/numpy) the one on PATH inside the shell. + export PATH="${pythonEnv}/bin:$PATH" +'' diff --git a/nix/dev/packages.nix b/nix/dev/packages.nix new file mode 100644 index 00000000..6acfc5e2 --- /dev/null +++ b/nix/dev/packages.nix @@ -0,0 +1,42 @@ +# nix/dev/packages.nix +# +# The dev shell's tools, grouped by role: +# pythonEnv python3 + the only third-party deps Homa's scripts use +# allPackages everything, for the dev shell +# +# In the scope (nix/scope.nix) as packagesModule. + +{ pkgs +, lib +, kernelPackages +}: + +let + # Homa's Python tooling imports only matplotlib and numpy beyond the stdlib. + pythonEnv = pkgs.python3.withPackages (p: [ p.matplotlib p.numpy ]); + + # Out-of-tree kernel module build dependencies (the key kbuild idiom) + make/perl for the + # test harness dependency generation (test/mergedep.pl). + buildTools = [ + pkgs.gnumake + pkgs.perl + pkgs.bc + ] ++ kernelPackages.kernel.moduleBuildDependencies; + + # C/C++ static-analysis + formatting tools. `smatch` is not in every nixpkgs, so guard it. + cTools = [ + pkgs.clang-tools # clang-tidy, clang-format + pkgs.cppcheck + pkgs.sparse + pkgs.bear # compile_commands.json generation + pkgs.gdb + ] ++ lib.optional (pkgs ? smatch) pkgs.smatch; + + pyTools = [ pkgs.ruff pkgs.mypy pythonEnv ]; + shTools = [ pkgs.shellcheck pkgs.shfmt ]; + docTools = [ pkgs.groff pkgs.ghostscript ]; # man/ PDF generation +in +{ + inherit pythonEnv; + allPackages = buildTools ++ cTools ++ pyTools ++ shTools ++ docTools; +} diff --git a/nix/dev/shell-functions/default.nix b/nix/dev/shell-functions/default.nix new file mode 100644 index 00000000..d7598efd --- /dev/null +++ b/nix/dev/shell-functions/default.nix @@ -0,0 +1,33 @@ +# nix/dev/shell-functions/default.nix +# +# Bash helper functions spliced into the dev shell's shellHook (xdp2 pattern). Returns one +# concatenated string of function definitions. +# +# In the scope (nix/scope.nix) as shellFunctions; dev/shell.nix puts it in the shellHook. + +{}: + +'' + # Build homa.ko against $KDIR (env-vars.nix defaults KDIR to the pinned kernel; override + # it to retarget, e.g. KDIR=/lib/modules/$(uname -r)/build homa-build). + homa-build() { + make KDIR="''${KDIR}" "$@" + } + + # Build and run the unit tests (AddressSanitizer) against $KDIR. + homa-test() { + make -C test KDIR="''${KDIR}" LINUX_VERSION="''${LINUX_VERSION}" ARCH="''${ARCH}" "$@" \ + && ./test/unit + } + + homa-clean() { + make clean 2>/dev/null || true + make -C test clean 2>/dev/null || true + make -C util clean 2>/dev/null || true + } + + # Kernel checkpatch (needs a Linux source tree; set LINUX_SRC_DIR). + homa-checkpatch() { + make checkpatch "$@" + } +'' diff --git a/nix/dev/shell.nix b/nix/dev/shell.nix new file mode 100644 index 00000000..c78f1dd1 --- /dev/null +++ b/nix/dev/shell.nix @@ -0,0 +1,34 @@ +# nix/dev/shell.nix +# +# The Homa development shell. Composes the toolchain from packages.nix, the environment +# fragment from env-vars.nix, and the bash helper functions from shell-functions/*.nix. +# +# In the scope (nix/scope.nix) as devShell; nix/parts/dev.nix makes it devShells.default. + +{ pkgs +, packagesModule +, envVars +, shellFunctions # bash function definitions (shell-functions/) +}: + +pkgs.mkShell { + packages = packagesModule.allPackages; + + # Disable hardening in the shell so an in-shell `make` matches the derivations. + hardeningDisable = [ "all" ]; + + shellHook = '' + ${envVars} + ${shellFunctions} + + cat <<'EOF' + Homa dev shell. Kernel module + userspace + analysis toolchain is on PATH. + + homa-build build homa.ko against $KDIR (override KDIR to retarget) + homa-test build and run the unit tests (ASan) + homa-clean make clean + homa-checkpatch run kernel checkpatch (needs a Linux source tree) + + EOF + ''; +} diff --git a/nix/kernel/README.md b/nix/kernel/README.md new file mode 100644 index 00000000..e8d43a32 --- /dev/null +++ b/nix/kernel/README.md @@ -0,0 +1,20 @@ +# nix/kernel: which kernels Homa is built and tested against + +| File | In the scope as | What it is | +|---|---|---| +| `config.nix` | `kernelConfig` | the default kernel (nixpkgs `linuxPackages_latest`), its `kdir`, the per-series `matrix` of nixpkgs kernels, and `vmMatrix`, the series the microVMs boot (those in `vmSeries`) | +| `src.nix` | `kernelSrc` | kernel trees prepared from kernel.org tarballs (7.0 to 7.2) or a local checkout (`HOMA_LINUX_SRC`, with `--impure`), for compile checks; its `mkTree` also makes the riscv64 and aarch64 trees in [`../build/cross.nix`](../build/cross.nix) | +| `unit-kdir/` | `unitKdir` | one merged tree for the unit tests (`test/`) and the kernel-mock fuzzers, with the compiler flags they need on a nixpkgs kernel | + +To change the default kernel, edit `kernelPackages` in `config.nix`; every module build, +VM and test follows. To add a kernel to the VM tests, add its series to `vmSeries`. The +VM targets (`homa-microvm-test-` and so on) and the port allocation follow from +that list. + +## Why the trees need preparing + +A kernel module builds against a kernel's build tree (`KDIR`): its headers, its `.config` +and the generated files. nixpkgs ships this for the kernels it packages, as +`kernel.dev`. For a kernel nixpkgs does not package, `src.nix` runs +`make olddefconfig modules_prepare` on the source itself. nixpkgs splits a tree into `source/` and `build/`; Homa's +`test/Makefile` expects one tree, which `unit-kdir` provides as a tree of symlinks. diff --git a/nix/kernel/config.nix b/nix/kernel/config.nix new file mode 100644 index 00000000..f6c46d28 --- /dev/null +++ b/nix/kernel/config.nix @@ -0,0 +1,60 @@ +# nix/kernel/config.nix +# +# Single source of truth for kernel selection. +# +# Returns the default kernelPackages (nixpkgs `linux_latest`, overridable), the derived +# KDIR / modDirVersion used by the out-of-tree module build, and a "matrix" of every kernel +# series nixpkgs ships (6.1 ... 7.3), used for per-kernel module builds and microVMs. +# +# In the scope (nix/scope.nix) as kernelConfig: +# kernelConfig.kernelPackages # default (nixpkgs linux_latest, 7.3) +# kernelConfig.kdir # "${kernel.dev}/lib/modules/${modDirVersion}/build" +# kernelConfig.matrix # { "6_1" = ; ...; "7_3" = ; } (only those present) +# kernelConfig.vmMatrix # the subset the module builds on, so microVMs are useful +# kernelConfig.unitKernelPackages # the unit harness / mock fuzzers' kernel (6.18), which +# # kernel/unit-kdir builds its KDIR from + +{ pkgs +, lib + # Override the default kernel by passing another kernelPackages set. + # + # Default = nixpkgs' newest kernel (7.3 at the time of writing): Homa is being ported + # forward to 7.x. Upstream `main` builds here since the 7.1 port landed (e65a5c0). +, kernelPackages ? pkgs.linuxPackages_latest + # The unit harness and kernel-mock fuzzers need a kernel whose headers test/mock.c + # supports; kept separate so it can lag the default. +, unitKernelPackages ? pkgs.linuxPackages_6_18 +}: + +let + # KDIR for a given kernelPackages: the build tree shipped in the kernel's `dev` output. + kdirOf = kp: "${kp.kernel.dev}/lib/modules/${kp.kernel.modDirVersion}/build"; + + # "7.3.1" -> "7_3" + seriesOf = kp: lib.replaceStrings [ "." ] [ "_" ] (lib.versions.majorMinor kp.kernel.version); + + # Matrix: every kernel series nixpkgs still ships, keyed by series. Some attrs exist but + # throw on access (EOL kernels), so each is probed with tryEval and only usable ones kept. + # Series nixpkgs has dropped (7.0-7.2) come from kernel.org tarballs: see kernel/src.nix. + candidates = [ "6_1" "6_6" "6_12" "6_18" "latest" ]; + kpFor = name: pkgs."linuxPackages_${name}" or null; + usable = name: + let + kp = kpFor name; + probe = builtins.tryEval (kp != null && builtins.isString kp.kernel.modDirVersion); + in + probe.success && probe.value; + matrix = lib.listToAttrs + (map (n: lib.nameValuePair (seriesOf (kpFor n)) (kpFor n)) (lib.filter usable candidates)); + + # Series that get microVM and pair-test targets: those where homa.ko builds on `main`. + # 6.1/6.6 are below the supported range and 6.18-stable (6.18.40+, 6.18.57+) backported the + # 7.x socket API changes, so they stay in `matrix` (and homa-build-matrix) for visibility. + vmSeries = [ "6_12" "7_3" ]; + vmMatrix = lib.filterAttrs (s: _: lib.elem s vmSeries) matrix; +in +{ + inherit kernelPackages matrix vmMatrix kdirOf seriesOf unitKernelPackages; + inherit (kernelPackages.kernel) modDirVersion; + kdir = kdirOf kernelPackages; +} diff --git a/nix/kernel/src.nix b/nix/kernel/src.nix new file mode 100644 index 00000000..b654a03b --- /dev/null +++ b/nix/kernel/src.nix @@ -0,0 +1,106 @@ +# nix/kernel/src.nix +# +# Kernel build trees prepared from SOURCE, for compile-testing homa.ko against kernels that +# nixpkgs no longer ships (7.0-7.2 are EOL-removed) or has not released (a local checkout of +# net-next / mainline). +# +# Each tree is the kernel source plus `make olddefconfig modules_prepare` in an O= build +# directory, configured from nixpkgs' newest kernel config. That is enough for +# `make -C M=... modules`, but not a bootable kernel: there is no vmlinux and no +# Module.symvers, so module builds against it need KBUILD_MODPOST_WARN=1. About 2 minutes +# per tree, versus ~30 for a full kernel build. +# +# Sources: +# tarballs pinned kernel.org releases below (hashes checked against sha256sums.asc) +# local HOMA_LINUX_SRC=/path/to/linux, only with --impure; e.g. +# HOMA_LINUX_SRC=~/Downloads/linux nix build --impure .#homa-build-matrix +# +# In the scope (nix/scope.nix) as kernelSrc: +# kernelSrc.trees # { "7_0" = ; "7_1" = ; "7_2" = ; [ local = ; ] } +# "${kernelSrc.trees."7_1"}/build" # KDIR + +{ pkgs +, lib +}: + +let + # Config source for every tree; olddefconfig fills in or drops symbols per version. + configKernel = pkgs.linuxPackages_latest.kernel; + + tarballs = { + "7_0" = { version = "7.0.14"; hash = "sha256-3pmZt4TSKT8A05xi2PkqCKuKVLxOgP/SUKDAnLB6D5g="; }; + "7_1" = { version = "7.1.13"; hash = "sha256-YU2V+v3LXM4rZiDn7cavu2Br/9RlVAVYaBXYRoeEGtc="; }; + "7_2" = { version = "7.2.12"; hash = "sha256-uxhGio3Il4qgDKpkRudT97baP+5Wtrfzixb5QwugLU8="; }; + }; + + # Large source directories that out-of-tree module builds never read. + prune = [ "Documentation" "drivers" "sound" "samples" "fs" "LICENSES" ]; + + # mkTree is also used by cross.nix: configfile is that arch's nixpkgs kernel config, and + # makeFlags carries ARCH= and CROSS_COMPILE= (the cross gcc goes in extraInputs). + mkTree = + { name + , version + , src + , configfile ? configKernel.configfile + , makeFlags ? [ ] + , extraInputs ? [ ] + }: pkgs.stdenv.mkDerivation { + pname = "linux-kdir-${name}"; + inherit version src; + + nativeBuildInputs = with pkgs; [ bc bison flex perl python3 pkg-config ] + ++ configKernel.moduleBuildDependencies ++ extraInputs; + buildInputs = with pkgs; [ elfutils openssl zlib ]; + hardeningDisable = [ "all" ]; + dontConfigure = true; + + # The O= build tree records the absolute source path, so the source must live in $out + # too. Build into $out directly. + buildPhase = '' + runHook preBuild + mkdir -p "$out" + cp -r . "$out/source" + chmod -R u+w "$out/source" + patchShebangs "$out/source/scripts" "$out/source/tools" + cd "$out/source" + mkdir -p "$out/build" + cp ${configfile} "$out/build/.config" + make O="$out/build" ${toString makeFlags} olddefconfig + make O="$out/build" ${toString makeFlags} -j"$NIX_BUILD_CORES" modules_prepare + # Keep enough source for M= builds (headers, scripts, arch, Kbuild/Kconfig files). + rm -rf ${lib.concatMapStringsSep " " (d: "\"$out/source/${d}\"") prune} + runHook postBuild + ''; + dontInstall = true; + dontFixup = true; + + # module.nix's buildAgainstTree passes these on to make. (Not name: a passthru name + # would hide the derivation's own.) + passthru = { inherit makeFlags; }; + }; + + fromTarball = name: { version, hash }: mkTree { + inherit name version; + src = pkgs.fetchurl { + url = "https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-${version}.tar.xz"; + inherit hash; + }; + }; + + localPath = builtins.getEnv "HOMA_LINUX_SRC"; + localTree = mkTree { + name = "local"; + version = "local"; + src = builtins.path { + name = "linux-local-src"; + path = localPath; + filter = p: _: baseNameOf p != ".git"; + }; + }; +in +{ + inherit mkTree; + trees = lib.mapAttrs fromTarball tarballs + // lib.optionalAttrs (localPath != "") { local = localTree; }; +} diff --git a/nix/kernel/unit-kdir/default.nix b/nix/kernel/unit-kdir/default.nix new file mode 100644 index 00000000..3f0dd84d --- /dev/null +++ b/nix/kernel/unit-kdir/default.nix @@ -0,0 +1,54 @@ +# nix/kernel/unit-kdir/default.nix +# +# A KDIR tailored for Homa's user-space unit harness (test/) and the kernel-mock fuzzers. +# +# Homa's test/Makefile hardcodes $(KDIR)/include/... and $(KDIR)/arch/x86/include/..., but +# nixpkgs splits a kernel into source/ (checked-in headers) and build/ (generated). This +# derivation merges them into one symlink tree, and swaps in a patched copy of +# include/generated/autoconf.h so that the distro config matches what test/mock.[ch] +# assumes (see patch-autoconf.sh): no gcc named-address-space per-cpu, no memory +# allocation profiling, and CONFIG_DEBUG_PREEMPT (mock.c defines preempt_count_add()). +# +# It also exposes the extra compiler flags needed against the nixpkgs kernel: +# -fcf-protection=branch the config enables X86_KERNEL_IBT (nocf_check attributes) +# -include homa_unit_compat.h small API renames vs. the vendored test/*_v6_17_8.c sources +# -Wno-error=... diagnostics newer gcc emits on Homa that are not +# harness problems (reported separately by the analysis) +# +# Nothing in the source tree is modified: everything is passed via KDIR= and CC=. +# +# Usage: +# unitKdir (in nix/scope.nix), used by userspace.nix and fuzz/tools/fuzz-mock.nix: +# make -C test KDIR=${unitKdir} CC="gcc ${unitKdir.extraCFlags}" unit + +{ pkgs +, lib +, kernelConfig +}: + +let + unitKp = kernelConfig.unitKernelPackages; + buildK = kernelConfig.kdirOf unitKp; + compatHeader = ./homa_unit_compat.h; + + kdir = pkgs.runCommand "homa-unit-kdir-${unitKp.kernel.modDirVersion}" { } '' + srcK="$(dirname "${buildK}")/source" + mkdir -p "$out/include/generated" "$out/arch/x86/include" + cp -rs "$srcK/include/." "$out/include/" + cp -rs "$srcK/arch/x86/include/." "$out/arch/x86/include/" + cp -rs "${buildK}/include/generated/." "$out/include/generated/" + ln -sfn "${buildK}/arch/x86/include/generated" "$out/arch/x86/include/generated" + chmod -R u+w "$out/include/generated" + rm -f "$out/include/generated/autoconf.h" + ${pkgs.bash}/bin/bash ${./patch-autoconf.sh} \ + "${buildK}/include/generated/autoconf.h" "$out/include/generated/autoconf.h" + ''; +in +kdir // { + extraCFlags = lib.concatStringsSep " " [ + "-fcf-protection=branch" + "-include ${compatHeader}" + "-Wno-error=unused-but-set-variable" + "-Wno-error=incompatible-pointer-types" + ]; +} diff --git a/nix/kernel/unit-kdir/homa_unit_compat.h b/nix/kernel/unit-kdir/homa_unit_compat.h new file mode 100644 index 00000000..14bac43b --- /dev/null +++ b/nix/kernel/unit-kdir/homa_unit_compat.h @@ -0,0 +1,7 @@ +/* Force-included when building the unit harness against a newer kernel than + * the vendored test/ rhashtable sources were taken from. + */ +#ifndef kvmalloc_node_noprof +#define kvmalloc_node_noprof(_size, _flags, _node) \ + kvmalloc_node_align_noprof(_size, 1, _flags, _node) +#endif diff --git a/nix/kernel/unit-kdir/patch-autoconf.sh b/nix/kernel/unit-kdir/patch-autoconf.sh new file mode 100755 index 00000000..534c62f5 --- /dev/null +++ b/nix/kernel/unit-kdir/patch-autoconf.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# patch-autoconf.sh +# Adapts a distro kernel config to what Homa's user-space mock layer (test/mock.[ch]) assumes. +set -eu +awk ' +BEGIN { + split("CONFIG_USE_X86_SEG_SUPPORT CONFIG_CC_HAS_NAMED_AS CONFIG_CC_HAS_NAMED_AS_FIXED_SANITIZERS CONFIG_MEM_ALLOC_PROFILING CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT CONFIG_MEM_ALLOC_PROFILING_DEBUG", a, " ") + for (i in a) off[a[i]] = 1 +} +$1 == "#define" && ($2 in off) { print "/* unit-test KDIR: " $2 " disabled */"; next } +{ print } +END { print "#ifndef CONFIG_DEBUG_PREEMPT\n#define CONFIG_DEBUG_PREEMPT 1 /* unit-test KDIR */\n#endif" } +' "$1" > "$2" diff --git a/nix/lib/README.md b/nix/lib/README.md new file mode 100644 index 00000000..46405f2d --- /dev/null +++ b/nix/lib/README.md @@ -0,0 +1,34 @@ +# nix/lib: homaLib, what every Homa build shares + +`default.nix` is a set of values and functions, not a package. The scope +([`../scope.nix`](../scope.nix)) builds it once as `homaLib`, and any file that names +`homaLib` in its argument list gets it. + +| Name | What it is | +|---|---| +| `src.module` | Homa's top-level C sources, Makefiles and Kconfig: what kbuild reads | +| `src.util`, `src.test` | the same, plus `util/` (and `test/`) | +| `src.fuzz` | plus `util/`, `test/` and `nix/fuzz/` | +| `src.all` | the whole tree, for the linters | +| `version`, `meta` | for every Homa derivation | +| `mkKbuild { kdir, makeFlags, ... }` | a derivation that runs Homa's Makefile against a kernel tree | +| `perKernel prefix f set` | `{ "-" = f series value; }` for a set keyed by kernel series | + +## Why the source is cut into subsets + +A Nix build depends on every byte of its source. Built from the whole repository, `homa.ko` +would rebuild whenever a README changed. Each subset is made with `lib.fileset` and holds +only what one kind of build reads, so a change outside it leaves the build cached. Markdown +files are left out of every subset but `all`. For the same reason, `version` does not +contain the git revision. + +Each subset also has the patches in [`../patches`](../patches) applied. A patch is cut down +to the files in the subset first (`filterdiff`), so a fix that also adds a unit test still +applies to `src.module`, which has no `test/`. + +## Nix idioms used here + +- `lib.fileset.toSource { root; fileset; }` copies only the listed files into the store. +- `pkgs.applyPatches` is a derivation whose output is its source with patches applied. +- `mkKbuild`'s `args // { ... }` merges attribute sets: the right side wins, which is how + `makeFlags` and `nativeBuildInputs` are appended to instead of replaced. diff --git a/nix/lib/default.nix b/nix/lib/default.nix new file mode 100644 index 00000000..164a38d0 --- /dev/null +++ b/nix/lib/default.nix @@ -0,0 +1,92 @@ +# nix/lib/default.nix +# +# homaLib: what the module, userspace, analysis and fuzz builds share (see README.md). +# +# version, meta for every Homa derivation. The version leaves out the git revision: +# with it, every commit would rebuild everything, even one that only +# edits a README. +# src. Homa's source cut down to what one kind of build reads, with the +# patches in ../patches applied: +# module the top-level C sources, Makefiles and Kconfig +# util + util/ +# test + util/ and test/ +# fuzz + util/, test/ and nix/fuzz/ +# all the whole tree (the linters scan everything) +# So editing a README, or anything under nix/, does not rebuild homa.ko. +# mkKbuild a derivation that runs Homa's Makefile (kbuild) against a kernel tree +# perKernel { "-" = f series value; } for a set keyed by series + +{ pkgs, lib }: + +let + root = ../..; + + # Top-level files only: Homa's sources sit at the root next to its subdirectories. + topLevel = lib.fileset.unions (map (name: root + "/${name}") (lib.attrNames + (lib.filterAttrs + (name: type: type == "regular" + && (lib.any (ext: lib.hasSuffix ext name) [ ".c" ".h" ".cc" ] + || lib.elem name [ "Makefile" "Makefile.upstream" "Kconfig" ])) + (builtins.readDir root)))); + + # A patch touches Homa's C sources: the top level, test/ and util/. Each subset gets the + # parts of each patch that are inside it (filterdiff), so a fix that also adds a unit test + # still applies to the module's source. + patches = import ../patches; + patchDirs = [ "test" "util" ]; + patchFor = dirs: patch: pkgs.runCommand "${baseNameOf patch}" + { nativeBuildInputs = [ pkgs.patchutils ]; } '' + filterdiff -p1 ${lib.concatMapStringsSep " " (d: "-x '${d}/*'") + (lib.subtractLists dirs patchDirs)} ${patch} > $out + ''; + + subset = name: dirs: fileset: pkgs.applyPatches { + name = "homa-src-${name}"; + src = lib.fileset.toSource { inherit root fileset; }; + patches = map (patchFor dirs) patches; + }; + # A directory without its Markdown files: no build reads them. + dirNoDocs = d: lib.fileset.difference (root + "/${d}") + (lib.fileset.fileFilter (f: f.hasExt "md") (root + "/${d}")); + withDirs = name: dirs: subset name dirs + (lib.fileset.unions ([ topLevel ] ++ map dirNoDocs dirs)); + + version = "0-unstable"; + src = { + module = withDirs "module" [ ]; + util = withDirs "util" [ "util" ]; + test = withDirs "test" [ "test" "util" ]; + fuzz = withDirs "fuzz" [ "test" "util" "nix/fuzz" ]; + all = subset "all" patchDirs root; + }; +in +{ + inherit version src; + + meta = { + homepage = "https://github.com/PlatformLab/HomaModule"; + license = lib.licenses.bsd2; + platforms = lib.platforms.linux; + }; + + # mkKbuild { pname, kdir, ... }: runs `make` in Homa's module source with KDIR=kdir + # (when given) and MY_CFLAGS=-O2. Any other attribute goes to mkDerivation; makeFlags and + # nativeBuildInputs are appended to, the rest replace these defaults. + mkKbuild = { kdir ? null, makeFlags ? [ ], nativeBuildInputs ? [ ], ... }@args: + pkgs.stdenv.mkDerivation ({ + inherit version; + src = src.module; + dontConfigure = true; + # The cc-wrapper's hardening flags clash with kbuild's own. + hardeningDisable = [ "all" ]; + # Through the environment, not makeFlags: stdenv word-splits makeFlags, which would + # turn a debug build's "-g -O1" into make's own -O option. + env.MY_CFLAGS = "-O2"; + } // removeAttrs args [ "kdir" ] // { + nativeBuildInputs = [ pkgs.gnumake ] ++ nativeBuildInputs; + makeFlags = lib.optional (kdir != null) "KDIR=${kdir}" ++ makeFlags; + }); + + perKernel = prefix: f: lib.mapAttrs' (series: value: + lib.nameValuePair "${prefix}-${series}" (f series value)); +} diff --git a/nix/parts/README.md b/nix/parts/README.md new file mode 100644 index 00000000..2336effd --- /dev/null +++ b/nix/parts/README.md @@ -0,0 +1,21 @@ +# nix/parts: the flake's outputs, one file per area + +[`flake.nix`](../../flake.nix) is short: it names its inputs and lists these files. Each one +is a [flake-parts](https://flake.parts) module that sets some of the flake's outputs. The +outputs are what `nix build .#`, `nix run`, `nix develop` and `nix flake check` find. + +| File | Outputs | +|---|---| +| `scope.nix` | none: builds the package scope ([`../scope.nix`](../scope.nix)) and passes it to the others as `homa` | +| `build.nix` | `homa-module*`, `homa-build-matrix`, `homa-utils*`, `homa-tests`, `homa-image*`, cross builds (x86_64 hosts only); `default` is `homa-module` | +| `dev.nix` | `devShells.default` (`nix develop`) and `formatter` (`nix fmt`) | + +## How a part works + +`perSystem` is a function that flake-parts calls once for each system in `flake.nix` +(`x86_64-linux`, `aarch64-linux`), with `pkgs` for that system. What it returns becomes +`packages..*`, `checks..*` and so on. Several parts may each set +`packages`; flake-parts merges them, and a name defined twice is an error. + +The parts hold no build logic. Each one picks outputs from the scope (`homa.analysisPkgs` +and so on), so the build code under `nix/` does not depend on flake-parts. diff --git a/nix/parts/build.nix b/nix/parts/build.nix new file mode 100644 index 00000000..1b5be025 --- /dev/null +++ b/nix/parts/build.nix @@ -0,0 +1,16 @@ +# nix/parts/build.nix +# +# homa.ko for every kernel (and cross-compiled on x86_64 hosts), the util/ tools, the unit +# tests, and the OCI image. `nix build` with no target builds homa-module. + +{ + perSystem = { homa, lib, system, ... }: { + packages = + homa.modulePkgs.packages + // homa.userspacePkgs.packages + // homa.ociPkgs.packages + # The cross toolchains are built for x86_64 hosts only. + // lib.optionalAttrs (system == "x86_64-linux") homa.crossPkgs.packages + // { default = homa.modulePkgs.homa-module; }; + }; +} diff --git a/nix/parts/dev.nix b/nix/parts/dev.nix new file mode 100644 index 00000000..40290fa1 --- /dev/null +++ b/nix/parts/dev.nix @@ -0,0 +1,10 @@ +# nix/parts/dev.nix +# +# `nix develop` and `nix fmt`. + +{ + perSystem = { homa, pkgs, ... }: { + devShells.default = homa.devShell; + formatter = pkgs.nixpkgs-fmt; + }; +} diff --git a/nix/parts/scope.nix b/nix/parts/scope.nix new file mode 100644 index 00000000..3760d683 --- /dev/null +++ b/nix/parts/scope.nix @@ -0,0 +1,12 @@ +# nix/parts/scope.nix +# +# Builds the Homa package scope (../scope.nix) once per system and hands it to every other +# part as the perSystem argument `homa`. + +{ inputs, ... }: + +{ + perSystem = { pkgs, ... }: { + _module.args.homa = import ../scope.nix { inherit pkgs inputs; }; + }; +} diff --git a/nix/patches/default.nix b/nix/patches/default.nix new file mode 100644 index 00000000..f08ce9af --- /dev/null +++ b/nix/patches/default.nix @@ -0,0 +1,12 @@ +# nix/patches/default.nix +# +# Upstream-pending fixes the flake needs in order to build Homa against its default kernel. +# Each patch is a fork draft PR's commit, exported with `git format-patch`. The list is empty +# while upstream `main` builds on the default kernel as-is (the 7.1 port, fork draft #12, landed +# upstream as e65a5c0 on 2026-10-27). +# +# homaLib (../lib) applies them to every source subset, each patch cut down to the files in +# that subset. Once upstream `main` carries a fix, its patch stops applying and the build +# fails at patchPhase: that is the cue to delete it from this list. + +[ ] diff --git a/nix/scope.nix b/nix/scope.nix new file mode 100644 index 00000000..2ba84f4b --- /dev/null +++ b/nix/scope.nix @@ -0,0 +1,44 @@ +# nix/scope.nix +# +# Every Homa derivation, in one package scope (lib.makeScope). Each file below is called +# with callPackage, which passes a file the arguments its argument list names: from this +# scope first, then from nixpkgs. A file states what it needs, and nothing has to thread +# pkgs, lib and kernelConfig through by hand. The flake-parts modules in parts/ read their +# outputs from here: +# +# homa = import ./scope.nix { inherit pkgs inputs; }; +# homa.modulePkgs.homa-module +# +# To swap a piece (another kernel, say), override the scope: homa.overrideScope (final: +# prev: { kernelConfig = ...; }) rebuilds everything that depends on it. + +{ pkgs, inputs }: + +pkgs.lib.makeScope pkgs.newScope (self: { + # homaLib (lib/): version, meta, the source subsets with patches/ applied, mkKbuild, + # perKernel. + homaLib = self.callPackage ./lib { }; + + # Foundation: the kernels, kernel trees from source, tool lists, dev environment. + kernelConfig = self.callPackage ./kernel/config.nix { }; + kernelSrc = self.callPackage ./kernel/src.nix { }; + packagesModule = self.callPackage ./dev/packages.nix { + inherit (self.kernelConfig) kernelPackages; + }; + inherit (self.packagesModule) pythonEnv; + envVars = self.callPackage ./dev/env-vars.nix { }; + shellFunctions = self.callPackage ./dev/shell-functions { }; + # The merged kernel tree that the unit tests and the mock fuzzers build against. + unitKdir = self.callPackage ./kernel/unit-kdir { }; + + # Each of these returns { packages, checks?, ... } for a part in parts/. + modulePkgs = self.callPackage ./build/module.nix { }; + crossPkgs = self.callPackage ./build/cross.nix { + inherit (self.modulePkgs) buildAgainstTree; + }; + userspacePkgs = self.callPackage ./build/userspace.nix { }; + ociPkgs = self.callPackage ./build/oci.nix { + inherit (self.userspacePkgs) homa-utils homa-utils-debug; + }; + devShell = self.callPackage ./dev/shell.nix { }; +}) From 5e2648f01199616831b03c1ca35ba8a7a896fb57 Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Fri, 30 Oct 2026 18:20:25 -0700 Subject: [PATCH 02/11] nix: tiered static analysis Three tiers, each a build whose output holds every tool's report and a summary with per-tool finding counts: - analysis-quick: ruff, shellcheck + shfmt, clang-format against the kernel's style (no kernel needed; a `nix flake check` gate) - analysis-standard: + cppcheck, + clang-tidy (bugprone, cert, clang-analyzer, misc) over a compile DB captured from a real kbuild - analysis-deep: + sparse with endianness checks, + smatch Each tool is also its own output (analysis-sparse, ...). homa-lint in the dev shell runs analysis-quick. Co-Authored-By: Claude Opus 5.5 --- flake.nix | 5 ++ nix/analysis/README.md | 40 +++++++++ nix/analysis/clang-format.nix | 25 ++++++ nix/analysis/clang-tidy.nix | 32 +++++++ nix/analysis/compile-db.nix | 38 +++++++++ nix/analysis/cppcheck.nix | 33 ++++++++ nix/analysis/default.nix | 124 ++++++++++++++++++++++++++++ nix/analysis/kbuild-check.nix | 36 ++++++++ nix/analysis/python.nix | 28 +++++++ nix/analysis/shell.nix | 32 +++++++ nix/dev/shell-functions/default.nix | 8 ++ nix/dev/shell.nix | 1 + nix/parts/README.md | 1 + nix/parts/analysis.nix | 9 ++ nix/scope.nix | 1 + 15 files changed, 413 insertions(+) create mode 100644 nix/analysis/README.md create mode 100644 nix/analysis/clang-format.nix create mode 100644 nix/analysis/clang-tidy.nix create mode 100644 nix/analysis/compile-db.nix create mode 100644 nix/analysis/cppcheck.nix create mode 100644 nix/analysis/default.nix create mode 100644 nix/analysis/kbuild-check.nix create mode 100644 nix/analysis/python.nix create mode 100644 nix/analysis/shell.nix create mode 100644 nix/parts/analysis.nix diff --git a/flake.nix b/flake.nix index e459f797..b66c66a5 100644 --- a/flake.nix +++ b/flake.nix @@ -20,6 +20,10 @@ # nix build .#homa-image # OCI userspace image (docker load < result) # nix build .#homa-image-debug # +# nix build .#analysis-quick # python + shell + clang-format (no kernel needed) +# nix build .#analysis-standard # + clang-tidy + cppcheck +# nix build .#analysis-deep # + sparse (endian) + smatch +# # nix flake show # list every output # nix flake check # evaluate/build the checks # @@ -52,6 +56,7 @@ imports = [ ./nix/parts/scope.nix ./nix/parts/build.nix + ./nix/parts/analysis.nix ./nix/parts/dev.nix ]; }; diff --git a/nix/analysis/README.md b/nix/analysis/README.md new file mode 100644 index 00000000..2d0cad3b --- /dev/null +++ b/nix/analysis/README.md @@ -0,0 +1,40 @@ +# nix/analysis: static analysis in three tiers + +Each tool is a derivation whose output is a report (`report.txt`) and a finding count +(`count.txt`). A tier runs several tools and adds a `summary.txt`. The tools report +findings; they do not fail on them. Only `analysis-quick` is a `nix flake check`. + +| Tier | Tools | Needs a kernel | +|---|---|---| +| `analysis-quick` | ruff (Python), shellcheck and shfmt (shell), clang-format | no | +| `analysis-standard` | + cppcheck, clang-tidy (from a compile database) | yes | +| `analysis-deep` | + sparse (with endianness checks), smatch | yes | + +```sh +nix build .#analysis-standard && cat result/summary.txt +nix build .#analysis-sparse && less result/report.txt # one tool +``` + +| File | What it is | +|---|---| +| `default.nix` | `mkAnalysisTool`, the tiers, and the per-tool `analysis-` packages | +| `python.nix`, `shell.nix`, `clang-format.nix`, `cppcheck.nix` | one `mkAnalysisTool` call each | +| `compile-db.nix` | `compile_commands.json`, recorded with `bear` around a kbuild build | +| `clang-tidy.nix` | clang-tidy over that compile database | +| `kbuild-check.nix` | a checker kbuild runs on every file (`make C=2 CHECK=...`): sparse and smatch | + +## Adding a tool + +```nix +mytool = mkAnalysisTool { + name = "mytool"; + tools = [ pkgs.mytool ]; + script = "mytool . > $out/report.txt"; + count = ": warning: "; # an extended regex matching one finding +}; +``` + +`script` runs in a writable copy of Homa's source (`homaLib.src.module` unless `src` says +otherwise). `kbuild = true` adds the compiler and the default kernel's build dependencies. +Each tool file is called with `lib.callPackageWith`, so it names the arguments it needs +(`mkAnalysisTool`, `pkgs`, `kernelConfig`, ...) and gets just those. diff --git a/nix/analysis/clang-format.nix b/nix/analysis/clang-format.nix new file mode 100644 index 00000000..e8ca5ff1 --- /dev/null +++ b/nix/analysis/clang-format.nix @@ -0,0 +1,25 @@ +# nix/analysis/clang-format.nix +# +# clang-format compliance check for the core C sources/headers. Homa follows Linux kernel +# style (tabs, checkpatch-strict), so we check against the built-in LinuxKernel style rather +# than imposing a new one (avoids massive churn). + +{ pkgs, mkAnalysisTool }: + +mkAnalysisTool { + name = "clang-format"; + tools = [ pkgs.clang-tools ]; + script = '' + # Core module sources/headers (top-level homa_*.c/.h + timetrace). + mapfile -t cfiles < <(find . -maxdepth 1 -type f \( -name 'homa_*.c' -o -name 'homa_*.h' \ + -o -name 'homa.h' -o -name 'timetrace.c' -o -name 'timetrace.h' \) | sort) + { + echo "# clang-format (style=LinuxKernel): ''${#cfiles[@]} files checked" + for f in "''${cfiles[@]}"; do + clang-format --style=LinuxKernel --dry-run --Werror "$f" >/dev/null 2>&1 \ + || echo "would-reformat: $f" + done + } > "$out/report.txt" + ''; + count = "^would-reformat: "; +} diff --git a/nix/analysis/clang-tidy.nix b/nix/analysis/clang-tidy.nix new file mode 100644 index 00000000..84f94e9d --- /dev/null +++ b/nix/analysis/clang-tidy.nix @@ -0,0 +1,32 @@ +# nix/analysis/clang-tidy.nix +# +# clang-tidy over the core C sources using the compile DB from compile-db.nix. The DB's +# absolute paths (from the compile-db build sandbox) are rewritten onto this copy of the +# tree. Kernel-code clang-tidy via a captured DB is inherently best-effort; if the DB is +# empty (module didn't build) this yields 0 findings, which is recorded honestly. + +{ pkgs, mkAnalysisTool, compile-db }: + +mkAnalysisTool { + name = "clang-tidy"; + tools = [ pkgs.clang-tools pkgs.jq ]; + script = '' + # Rewrite the DB's recorded source root onto our copy so clang-tidy finds the files. + srcroot=$(cat ${compile-db}/srcroot) + sed "s|$srcroot|$PWD|g" ${compile-db}/compile_commands.json > compile_commands.json + + mapfile -t files < <(jq -r '.[].file' compile_commands.json 2>/dev/null \ + | grep -E 'homa_.*\.c$' | sort -u || true) + + { + echo "# clang-tidy (bugprone-*,cert-*,clang-analyzer-*,misc-*): ''${#files[@]} file(s) in DB" + for f in "''${files[@]}"; do + [ -f "$f" ] || continue + clang-tidy -p "$PWD" \ + --checks='-*,bugprone-*,cert-*,clang-analyzer-*,misc-*' \ + "$f" 2>&1 || true + done + } > "$out/report.txt" + ''; + count = ": (warning|error): "; +} diff --git a/nix/analysis/compile-db.nix b/nix/analysis/compile-db.nix new file mode 100644 index 00000000..850fb22a --- /dev/null +++ b/nix/analysis/compile-db.nix @@ -0,0 +1,38 @@ +# nix/analysis/compile-db.nix +# +# Builds a compile_commands.json for the module by running the kbuild compile under `bear`. +# This is kernel-dependent (needs moduleBuildDependencies + KDIR) and is the input to +# clang-tidy. If the module fails to build (kernel API drift), an empty DB is emitted so the +# downstream analysis still evaluates (the drift is recorded, not fatal). +# +# Also records the build's source root ($out/srcroot) so clang-tidy can rewrite the DB's +# absolute paths onto its own copy of the tree. + +{ pkgs, homaLib, kernelConfig }: + +homaLib.mkKbuild { + pname = "homa-compile-db"; + kdir = kernelConfig.kdir; + nativeBuildInputs = [ pkgs.bear ] ++ kernelConfig.kernelPackages.kernel.moduleBuildDependencies; + + buildPhase = '' + runHook preBuild + echo "$PWD" > srcroot + bear --output compile_commands.json -- make $makeFlags || true + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + mkdir -p "$out" + cp srcroot "$out/srcroot" + if [ -f compile_commands.json ]; then + cp compile_commands.json "$out/compile_commands.json" + else + echo '[]' > "$out/compile_commands.json" + fi + runHook postInstall + ''; + + dontFixup = true; +} diff --git a/nix/analysis/cppcheck.nix b/nix/analysis/cppcheck.nix new file mode 100644 index 00000000..788a6496 --- /dev/null +++ b/nix/analysis/cppcheck.nix @@ -0,0 +1,33 @@ +# nix/analysis/cppcheck.nix +# +# cppcheck over the core C sources. Runs standalone (no compile DB needed) so it stays in +# the `standard` tier without requiring a kernel build. Kernel headers are absent, so many +# macros are unknown — we suppress missingInclude/unknownMacro noise and focus on cppcheck's +# own flow/bounds/leak checks. + +{ pkgs, mkAnalysisTool }: + +mkAnalysisTool { + name = "cppcheck"; + tools = [ pkgs.cppcheck ]; + script = '' + mapfile -t cfiles < <(find . -maxdepth 1 -type f -name 'homa_*.c' | sort) + cfiles+=("./timetrace.c") + { + echo "# cppcheck (warning,performance,portability; std=c11)" + cppcheck \ + --enable=warning,performance,portability \ + --std=c11 \ + --language=c \ + -I. \ + --suppress=missingInclude \ + --suppress=missingIncludeSystem \ + --suppress=unknownMacro \ + --inline-suppr \ + --quiet \ + --template='{file}:{line}: {severity}: {id}: {message}' \ + "''${cfiles[@]}" 2>&1 || true + } > "$out/report.txt" + ''; + count = "^[^:]+:[0-9]+: "; +} diff --git a/nix/analysis/default.nix b/nix/analysis/default.nix new file mode 100644 index 00000000..8c6074f2 --- /dev/null +++ b/nix/analysis/default.nix @@ -0,0 +1,124 @@ +# nix/analysis/default.nix +# +# Tiered static analysis (xdp2/rocm pattern). Each tool is one mkAnalysisTool call (below) +# that writes $out/report.txt and $out/count.txt; the tier targets gather those into one +# summary.txt with per-tool finding counts. +# +# analysis-quick python (ruff) + shell (shellcheck/shfmt) + clang-format (no kernel) +# analysis-standard + cppcheck + clang-tidy (compile DB) +# analysis-deep + sparse (endianness) + smatch +# +# Per-tool packages are also exposed (analysis-ruff, analysis-cppcheck, ...). nix/scope.nix +# calls this file with callPackage; each tool file is called the same way, and gets +# mkAnalysisTool. + +{ pkgs +, lib +, homaLib +, kernelConfig +}: + +let + # mkAnalysisTool { name, tools, script, count, src?, kbuild? }: copies the source (by + # default Homa's module sources), runs script, which writes "$out/report.txt", then + # writes into count.txt how many report lines match the extended regex count. + # kbuild = true adds what a build against the default kernel needs, a C compiler + # included (runCommandCC; plain runCommand has none). + mkAnalysisTool = { name, tools, script, count, src ? homaLib.src.module, kbuild ? false }: + (if kbuild then pkgs.runCommandCC else pkgs.runCommand) "homa-analysis-${name}" + { + # stdenv already has coreutils, findutils, grep, sed and awk on PATH. + nativeBuildInputs = tools + ++ lib.optionals kbuild + ([ pkgs.gnumake ] ++ kernelConfig.kernelPackages.kernel.moduleBuildDependencies); + hardeningDisable = lib.optional kbuild "all"; + } '' + cp -r ${src} src && chmod -R +w src && cd src + mkdir -p "$out" + ${script} + count=$(grep -cE '${count}' "$out/report.txt" || true) + echo "''${count:-0}" > "$out/count.txt" + ''; + + # callPackageWith: like callPackage, with these added to what a file can name. + call = lib.callPackageWith (pkgs // { inherit homaLib kernelConfig mkAnalysisTool; }); + kbuildCheck = call ./kbuild-check.nix { }; + + python = call ./python.nix { }; + shell = call ./shell.nix { }; + clang-format = call ./clang-format.nix { }; + cppcheck = call ./cppcheck.nix { }; + compile-db = call ./compile-db.nix { }; + clang-tidy = call ./clang-tidy.nix { inherit compile-db; }; + # sparse: the kernel's own semantic checker, with endianness checks on. It flags + # mismatched __be16/__be32/__be64 handling and __force casts in the wire code, which + # fuzzing can miss. + sparse = kbuildCheck { + name = "sparse"; + checker = "${pkgs.sparse}/bin/sparse"; + flags = "-D__CHECK_ENDIAN__ -Wsparse-all"; + count = ":[0-9]+:[0-9]+: (warning|error):"; + }; + # smatch: kernel-oriented flow analysis, when nixpkgs has it. + smatch = kbuildCheck { + name = "smatch"; + checker = "${pkgs.smatch}/bin/smatch -p=kernel"; + count = ":[0-9]+:[0-9]+: (warn|error):"; + }; + hasSmatch = pkgs ? smatch; + + # A tier is a runCommand that gathers named tool reports into one directory + summary. + mkTier = tierName: tools: + let + links = lib.concatMapStringsSep "\n" + (t: '' + cp "${t.drv}/report.txt" "$out/${t.name}.report.txt" + c=$(cat "${t.drv}/count.txt") + printf ' %-14s %s finding(s)\n' "${t.name}:" "$c" >> "$out/summary.txt" + total=$((total + c)) + '') + tools; + in + pkgs.runCommand "homa-${tierName}" { } '' + mkdir -p "$out" + total=0 + echo "=== Homa ${tierName} ===" > "$out/summary.txt" + ${links} + echo "" >> "$out/summary.txt" + echo "Total: $total finding(s)" >> "$out/summary.txt" + echo "$total" > "$out/count.txt" + cat "$out/summary.txt" + ''; + + quickTools = [ + { name = "ruff"; drv = python; } + { name = "shell"; drv = shell; } + { name = "clang-format"; drv = clang-format; } + ]; + standardTools = quickTools ++ [ + { name = "cppcheck"; drv = cppcheck; } + { name = "clang-tidy"; drv = clang-tidy; } + ]; + deepTools = standardTools ++ [ + { name = "sparse"; drv = sparse; } + ] ++ lib.optional hasSmatch { name = "smatch"; drv = smatch; }; + + analysis-quick = mkTier "analysis-quick" quickTools; + analysis-standard = mkTier "analysis-standard" standardTools; + analysis-deep = mkTier "analysis-deep" deepTools; +in +{ + packages = { + inherit analysis-quick analysis-standard analysis-deep; + analysis-ruff = python; + analysis-shell = shell; + analysis-clang-format = clang-format; + analysis-cppcheck = cppcheck; + analysis-clang-tidy = clang-tidy; + analysis-compile-db = compile-db; + analysis-sparse = sparse; + } // lib.optionalAttrs hasSmatch { analysis-smatch = smatch; }; + + # `analysis-quick` is kernel-independent, so it makes a good `nix flake check` gate. + checks = { inherit analysis-quick; }; +} diff --git a/nix/analysis/kbuild-check.nix b/nix/analysis/kbuild-check.nix new file mode 100644 index 00000000..2b7fff32 --- /dev/null +++ b/nix/analysis/kbuild-check.nix @@ -0,0 +1,36 @@ +# nix/analysis/kbuild-check.nix +# +# A checker that kbuild runs on every source file (make C=2 CHECK=...), against the default +# kernel: sparse and smatch (deep tier). Best-effort: if the module does not build against +# that kernel, the report still holds what the checker and kbuild printed. But it fails if +# kbuild never ran the checker at all (no "CHECK" line), so a broken setup cannot pass as +# zero findings. +# +# kbuildCheck { name, checker, flags ? "", count } +# checker the command kbuild runs per file (CHECK=) +# flags passed to it (CF=) +# count extended regex for one finding in its output + +{ kernelConfig, mkAnalysisTool }: + +{ name, checker, flags ? "", count }: + +mkAnalysisTool { + inherit name count; + tools = [ ]; + kbuild = true; + script = '' + # C=2 checks every file, not just those make rebuilds. + make KDIR=${kernelConfig.kdir} MY_CFLAGS='-O2' \ + C=2 CHECK="${checker}" CF="${flags}" > checker.stdout.log 2> checker.stderr.log || true + { + echo "# ${name} ${flags}" + cat checker.stderr.log checker.stdout.log + } > "$out/report.txt" + if ! grep -qE '^ +CHECK ' checker.stdout.log; then + cat "$out/report.txt" + echo "${name}: kbuild never ran the checker" + exit 1 + fi + ''; +} diff --git a/nix/analysis/python.nix b/nix/analysis/python.nix new file mode 100644 index 00000000..f2dfd24c --- /dev/null +++ b/nix/analysis/python.nix @@ -0,0 +1,28 @@ +# nix/analysis/python.nix +# +# Python static analysis with ruff. Homa's scripts include EXTENSIONLESS python programs +# (util/cp_* with a `#!/usr/bin/python3` shebang), so files are discovered by shebang as +# well as by `.py` extension — a plain `*.py` glob would miss them. + +{ pkgs, homaLib, mkAnalysisTool }: + +mkAnalysisTool { + name = "python"; + tools = [ pkgs.ruff ]; + src = homaLib.src.all; + script = '' + export RUFF_CACHE_DIR="$TMPDIR/ruff" + # A shebang must be the first line (grep -l would match a "#!" on any line). + { find . -type f -exec awk 'FNR == 1 && /^#!.*python/ { print FILENAME } { nextfile }' {} + + find . -name '*.py'; } | sort -u > "$out/files.txt" + mapfile -t pyfiles < "$out/files.txt" + { + echo "# ruff check ($(wc -l < "$out/files.txt") files)" + if [ "''${#pyfiles[@]}" -gt 0 ]; then + ruff check --output-format=concise "''${pyfiles[@]}" 2>&1 || true + fi + } > "$out/report.txt" + ''; + # ruff's concise format: path:line:col: code message + count = "^[^:]+:[0-9]+:[0-9]+:"; +} diff --git a/nix/analysis/shell.nix b/nix/analysis/shell.nix new file mode 100644 index 00000000..696dc548 --- /dev/null +++ b/nix/analysis/shell.nix @@ -0,0 +1,32 @@ +# nix/analysis/shell.nix +# +# Shell static analysis with shellcheck + shfmt. Homa has NO *.sh files — its shell scripts +# are extensionless (util/send_many, util/get_traces, cloudlab/bin/*), so discovery is by +# shebang. + +{ pkgs, homaLib, mkAnalysisTool }: + +mkAnalysisTool { + name = "shell"; + tools = [ pkgs.shellcheck pkgs.shfmt ]; + src = homaLib.src.all; + script = '' + # The shebang must be the first line (grep -l would match a "#!" on any line). + find . -type f -exec awk 'FNR == 1 && /^#!.*(bash|\/bin\/sh|env sh)/ { print FILENAME } + { nextfile }' {} + | sort -u > "$out/files.txt" + mapfile -t shfiles < "$out/files.txt" + { + echo "# shellcheck ($(wc -l < "$out/files.txt") files)" + if [ "''${#shfiles[@]}" -gt 0 ]; then + shellcheck --severity=warning "''${shfiles[@]}" 2>&1 || true + fi + echo "" + echo "# shfmt -d (formatting diffs)" + if [ "''${#shfiles[@]}" -gt 0 ]; then + shfmt -d "''${shfiles[@]}" 2>&1 || true + fi + } > "$out/report.txt" + ''; + # One per shellcheck finding: "In line N:" + count = "^In .* line [0-9]+:"; +} diff --git a/nix/dev/shell-functions/default.nix b/nix/dev/shell-functions/default.nix index d7598efd..3eecc24a 100644 --- a/nix/dev/shell-functions/default.nix +++ b/nix/dev/shell-functions/default.nix @@ -30,4 +30,12 @@ homa-checkpatch() { make checkpatch "$@" } + + # Quick static analysis: the analysis-quick tier (ruff, shellcheck, clang-format in the + # kernel's style), so the shell and the flake report the same findings. + homa-lint() { + local out + out=$(nix build "$(git rev-parse --show-toplevel)#analysis-quick" --no-link --print-out-paths) \ + && cat "$out/summary.txt" && echo "reports: $out" + } '' diff --git a/nix/dev/shell.nix b/nix/dev/shell.nix index c78f1dd1..492c67ba 100644 --- a/nix/dev/shell.nix +++ b/nix/dev/shell.nix @@ -28,6 +28,7 @@ pkgs.mkShell { homa-test build and run the unit tests (ASan) homa-clean make clean homa-checkpatch run kernel checkpatch (needs a Linux source tree) + homa-lint the analysis-quick tier: ruff, shellcheck, clang-format EOF ''; diff --git a/nix/parts/README.md b/nix/parts/README.md index 2336effd..f002d6ab 100644 --- a/nix/parts/README.md +++ b/nix/parts/README.md @@ -8,6 +8,7 @@ outputs are what `nix build .#`, `nix run`, `nix develop` and `nix flake c |---|---| | `scope.nix` | none: builds the package scope ([`../scope.nix`](../scope.nix)) and passes it to the others as `homa` | | `build.nix` | `homa-module*`, `homa-build-matrix`, `homa-utils*`, `homa-tests`, `homa-image*`, cross builds (x86_64 hosts only); `default` is `homa-module` | +| `analysis.nix` | `analysis-*`; check `analysis-quick` | | `dev.nix` | `devShells.default` (`nix develop`) and `formatter` (`nix fmt`) | ## How a part works diff --git a/nix/parts/analysis.nix b/nix/parts/analysis.nix new file mode 100644 index 00000000..98bc9bb2 --- /dev/null +++ b/nix/parts/analysis.nix @@ -0,0 +1,9 @@ +# nix/parts/analysis.nix +# +# Static analysis (../analysis): analysis-quick is a check, the rest are packages. + +{ + perSystem = { homa, ... }: { + inherit (homa.analysisPkgs) packages checks; + }; +} diff --git a/nix/scope.nix b/nix/scope.nix index 2ba84f4b..cc6a50a5 100644 --- a/nix/scope.nix +++ b/nix/scope.nix @@ -40,5 +40,6 @@ pkgs.lib.makeScope pkgs.newScope (self: { ociPkgs = self.callPackage ./build/oci.nix { inherit (self.userspacePkgs) homa-utils homa-utils-debug; }; + analysisPkgs = self.callPackage ./analysis { }; devShell = self.callPackage ./dev/shell.nix { }; }) From 2b056d402dcf2049d743d8e51ed92c10427d8427 Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Fri, 30 Oct 2026 18:22:00 -0700 Subject: [PATCH 03/11] nix: libFuzzer harnesses for the wire format and the receive path - fuzz: standalone harnesses (ASan + UBSan) for a decoder of Homa's wire format, with seed corpora and a dictionary; fuzz-run runs each for 20 s, fuzz-deep for as long as you like in fork mode - fuzz-mock: harnesses over the real homa_softirq receive path, built on the unit tests' mock layer, one per packet type as well - fuzz-cov: llvm-cov coverage of the decoder - fuzz-selftest: a canary that fails if the harnesses stop finding a planted bug (a `nix flake check` gate) Crashes are sorted by sanitizer signature, and known findings are listed in nix/fuzz/known-findings.txt. Co-Authored-By: Claude Opus 5.5 --- flake.nix | 8 + nix/fuzz/README.md | 37 +++++ nix/fuzz/corpus-seeds/dissector/seed0 | Bin 0 -> 64 bytes nix/fuzz/default.nix | 117 ++++++++++++++ nix/fuzz/dict/homa_wire.dict | 17 ++ nix/fuzz/harness/fuzz_dissector.cc | 49 ++++++ nix/fuzz/harness/fuzz_guard.h | 40 +++++ nix/fuzz/harness/fuzz_selftest.cc | 30 ++++ nix/fuzz/harness/homa_wire_spec.h | 129 +++++++++++++++ nix/fuzz/harness/mock/fuzz_compat.c | 25 +++ nix/fuzz/harness/mock/fuzz_softirq.c | 222 ++++++++++++++++++++++++++ nix/fuzz/known-findings.txt | 4 + nix/fuzz/tools/build-mock-fuzzers.sh | 123 ++++++++++++++ nix/fuzz/tools/fuzz-cov.nix | 40 +++++ nix/fuzz/tools/fuzz-mock.nix | 68 ++++++++ nix/fuzz/tools/fuzz-selftest.nix | 43 +++++ nix/fuzz/tools/gen-mock-seeds.py | 101 ++++++++++++ nix/fuzz/tools/run-fuzzers.sh | 83 ++++++++++ nix/parts/README.md | 1 + nix/parts/fuzz.nix | 9 ++ nix/scope.nix | 1 + 21 files changed, 1147 insertions(+) create mode 100644 nix/fuzz/README.md create mode 100644 nix/fuzz/corpus-seeds/dissector/seed0 create mode 100644 nix/fuzz/default.nix create mode 100644 nix/fuzz/dict/homa_wire.dict create mode 100644 nix/fuzz/harness/fuzz_dissector.cc create mode 100644 nix/fuzz/harness/fuzz_guard.h create mode 100644 nix/fuzz/harness/fuzz_selftest.cc create mode 100644 nix/fuzz/harness/homa_wire_spec.h create mode 100644 nix/fuzz/harness/mock/fuzz_compat.c create mode 100644 nix/fuzz/harness/mock/fuzz_softirq.c create mode 100644 nix/fuzz/known-findings.txt create mode 100755 nix/fuzz/tools/build-mock-fuzzers.sh create mode 100644 nix/fuzz/tools/fuzz-cov.nix create mode 100644 nix/fuzz/tools/fuzz-mock.nix create mode 100644 nix/fuzz/tools/fuzz-selftest.nix create mode 100644 nix/fuzz/tools/gen-mock-seeds.py create mode 100644 nix/fuzz/tools/run-fuzzers.sh create mode 100644 nix/parts/fuzz.nix diff --git a/flake.nix b/flake.nix index b66c66a5..2d1057e7 100644 --- a/flake.nix +++ b/flake.nix @@ -24,6 +24,13 @@ # nix build .#analysis-standard # + clang-tidy + cppcheck # nix build .#analysis-deep # + sparse (endian) + smatch # +# nix build .#fuzz # build all libFuzzer harnesses +# FUZZ_TIME=300 ./result/bin/run-fuzzers # ... then run them for 5 min +# nix build .#fuzz-run # bounded run -> summary.txt / count.txt / crashes +# nix run .#fuzz-deep -- [outdir] # fork mode + dictionary (FUZZ_DEEP_TIME) +# nix build .#fuzz-cov # coverage of the wire (de)serialization functions +# nix build .#fuzz-selftest # canary gate (fails if the harness is vacuous) +# # nix flake show # list every output # nix flake check # evaluate/build the checks # @@ -57,6 +64,7 @@ ./nix/parts/scope.nix ./nix/parts/build.nix ./nix/parts/analysis.nix + ./nix/parts/fuzz.nix ./nix/parts/dev.nix ]; }; diff --git a/nix/fuzz/README.md b/nix/fuzz/README.md new file mode 100644 index 00000000..ed9d1d29 --- /dev/null +++ b/nix/fuzz/README.md @@ -0,0 +1,37 @@ +# nix/fuzz: libFuzzer harnesses for Homa's wire format + +The harnesses feed random bytes to the code that decodes Homa packets, built with +libFuzzer, AddressSanitizer and UndefinedBehaviorSanitizer. [`../docs/DESIGN.md`](../docs/DESIGN.md) +§7 explains what each one covers and why. + +| Output | What it is | +|---|---| +| `fuzz` | the standalone harnesses (`fuzz_dissector`, `fuzz_selftest`), which need no kernel, plus `bin/run-fuzzers` | +| `fuzz-run` | a 20 s run of each: `summary.txt`, `count.txt`, `crashes/` | +| `fuzz-deep` | a command: fork mode and the dictionary, for `FUZZ_DEEP_TIME` seconds per harness | +| `fuzz-cov` | llvm-cov coverage of the wire decoder | +| `fuzz-selftest` | a canary: fails unless the harness finds a planted bug (a `nix flake check`) | +| `fuzz-mock` | the kernel-mock harnesses: Homa's real receive path on the unit tests' mock kernel, plus `bin/run-mock-fuzzers` | + +```sh +nix build .#fuzz && FUZZ_TIME=300 ./result/bin/run-fuzzers out/ +nix run .#fuzz-deep -- out/ +nix build .#fuzz-mock && ./result/bin/run-mock-fuzzers 600 out/ +``` + +| Path | What it is | +|---|---| +| `default.nix` | the outputs above, and the helpers they share: `compileHarness`, `mkFuzzDrv`, `launcher` | +| `tools/run-fuzzers.sh` | the one runner every output uses: runs each harness, reproduces each crash, keeps one per sanitizer signature | +| `tools/fuzz-{cov,selftest,mock}.nix` | the builds with more to them than one compile | +| `tools/build-mock-fuzzers.sh`, `tools/gen-mock-seeds.py` | the kernel-mock build and its seed corpus; both run by hand too | +| `harness/` | the harness sources; `harness/mock/` for the kernel-mock ones | +| `corpus-seeds/`, `dict/` | starting inputs and the libFuzzer dictionary | +| `known-findings.txt` | sanitizer reports `fuzz-mock` accepts while their fix is pending | + +## The runner's output + +`run-fuzzers [outdir]` leaves, for each harness, its working corpus and log, and under +`crashes//` one `crash-` input with its symbolized `repro-.txt` per +distinct crash. `summary.txt` has a line per harness. The environment sets the run: +`FUZZ_TIME` (seconds per harness), `FUZZ_FORKS`, `FUZZ_MAXLEN`, `FUZZ_DICT`. diff --git a/nix/fuzz/corpus-seeds/dissector/seed0 b/nix/fuzz/corpus-seeds/dissector/seed0 new file mode 100644 index 0000000000000000000000000000000000000000..1ba57cfcad1e7876b78e21cffce7de192418a80a GIT binary patch literal 64 QcmZQzfCGU5IFon=029yvVE_OC literal 0 HcmV?d00001 diff --git a/nix/fuzz/default.nix b/nix/fuzz/default.nix new file mode 100644 index 00000000..2c2c35f8 --- /dev/null +++ b/nix/fuzz/default.nix @@ -0,0 +1,117 @@ +# nix/fuzz/default.nix +# +# Fuzzing composition root (rocm-systems pattern). Focus on Homa's wire-format +# (de)serialization — see docs/DESIGN.md §7. +# +# fuzz standalone wire-decoder + selftest harnesses (kernel-free, always builds), +# with bin/run-fuzzers (FUZZ_TIME=300 ./result/bin/run-fuzzers [outdir]) +# fuzz-run a 20 s run of each of those -> summary.txt / count.txt / crashes +# fuzz-deep nix run .#fuzz-deep -- [outdir]: fork mode + dictionary, for as long as +# FUZZ_DEEP_TIME says +# fuzz-cov llvm-cov of the wire decoder +# fuzz-selftest canary gate (fails if the harness is vacuous) +# fuzz-mock kernel-mock harnesses: real homa_softirq receive path + per-type pinned +# variants on the unit-test mock layer (gate: all build + seeds run clean), +# with bin/run-mock-fuzzers [seconds] [outdir] +# +# fuzz, fuzz-run, fuzz-deep and fuzz-mock's runner all run harnesses through one script, +# tools/run-fuzzers.sh, which also sorts the crashes by sanitizer signature. + +{ pkgs +, lib +, homaLib +, unitKdir +}: + +let + src = homaLib.src.fuzz; + sanFlags = "-std=c++17 -g -O1 -fno-omit-frame-pointer -fsanitize=fuzzer,address,undefined"; + symbolizer = "${lib.getBin pkgs.llvm}/bin/llvm-symbolizer"; + + # homa-fuzz [outdir]: see tools/run-fuzzers.sh. + runner = pkgs.writeShellApplication { + name = "homa-fuzz"; + runtimeInputs = with pkgs; [ coreutils findutils gnugrep gnused ]; + runtimeEnv.ASAN_SYMBOLIZER_PATH = symbolizer; + text = builtins.readFile ./tools/run-fuzzers.sh; + }; + + # A command for a build's bin/: runs the harnesses installed next to it, seeded from the + # build's corpus/. setup runs first, and may set FUZZ_* and the arguments ("$@" is the + # outdir). + launcher = name: setup: pkgs.writeShellApplication { + inherit name; + text = '' + here=$(dirname "$(readlink -f "$0")") + ${setup} + exec ${lib.getExe runner} "$here" "$here/../corpus" "$@" + ''; + }; + + # The clang++ command for one harness in nix/fuzz/harness. + compileHarness = { name, out ? "out/bin/${name}", flags ? "" }: + "clang++ ${sanFlags} ${flags} -I nix/fuzz/harness nix/fuzz/harness/${name}.cc -o ${out}"; + + # A derivation over the fuzz sources, built with clang (then args' own + # nativeBuildInputs) and no hardening flags (they clash with the sanitizers). + mkFuzzDrv = args: pkgs.stdenv.mkDerivation ({ + version = "0"; + inherit src; + hardeningDisable = [ "all" ]; + dontConfigure = true; + dontFixup = true; + } // args // { + nativeBuildInputs = [ pkgs.clang ] ++ args.nativeBuildInputs or [ ]; + }); + + # callPackageWith: like callPackage, with these added to what a file can name. + call = file: lib.callPackageWith (pkgs // { inherit mkFuzzDrv compileHarness launcher unitKdir; }) + file + { }; + + # The kernel-independent harnesses: the standalone wire-format spec decoder and the + # self-test canary. They need no kernel headers, so this always builds. + standaloneNames = [ "fuzz_selftest" "fuzz_dissector" ]; + fuzz = mkFuzzDrv { + pname = "homa-fuzzers"; + buildPhase = '' + runHook preBuild + mkdir -p out/bin out/corpus + ${lib.concatMapStrings (name: '' + ${compileHarness { inherit name; }} + mkdir -p out/corpus/${name} + cp nix/fuzz/corpus-seeds/${lib.removePrefix "fuzz_" name}/* out/corpus/${name}/ \ + 2>/dev/null || true + '') standaloneNames} + cp ${launcher "run-fuzzers" ""}/bin/run-fuzzers out/bin/ + runHook postBuild + ''; + installPhase = ''cp -r out "$out"''; + }; + + fuzz-run = pkgs.runCommand "homa-fuzz-run" { } '' + FUZZ_TIME=20 ${lib.getExe runner} ${fuzz}/bin ${fuzz}/corpus "$out" + ''; + + # A command, not a build, so that its time limit can be set when it runs. + fuzz-deep = pkgs.writeShellApplication { + name = "fuzz-deep"; + text = '' + # fuzz-deep [outdir]: FUZZ_DEEP_TIME (60 s per harness), FUZZ_DEEP_FORKS (2), + # FUZZ_DEEP_MAXLEN (4096). + export FUZZ_TIME=''${FUZZ_DEEP_TIME:-60} FUZZ_FORKS=''${FUZZ_DEEP_FORKS:-2} \ + FUZZ_MAXLEN=''${FUZZ_DEEP_MAXLEN:-4096} FUZZ_DICT=${src}/nix/fuzz/dict/homa_wire.dict + exec ${lib.getExe runner} ${fuzz}/bin ${fuzz}/corpus "''${1:-fuzz-deep-out}" + ''; + }; + + fuzz-cov = call ./tools/fuzz-cov.nix; + fuzz-selftest = call ./tools/fuzz-selftest.nix; + fuzz-mock = call ./tools/fuzz-mock.nix; +in +{ + packages = { inherit fuzz fuzz-run fuzz-deep fuzz-cov fuzz-selftest fuzz-mock; }; + + # fuzz-selftest is a real gate and kernel-free, so it belongs in `nix flake check`. + checks = { inherit fuzz-selftest; }; +} diff --git a/nix/fuzz/dict/homa_wire.dict b/nix/fuzz/dict/homa_wire.dict new file mode 100644 index 00000000..fb49dd92 --- /dev/null +++ b/nix/fuzz/dict/homa_wire.dict @@ -0,0 +1,17 @@ +# nix/fuzz/dict/homa_wire.dict — libFuzzer dictionary for Homa wire formats. +# The packet-type byte (enum homa_packet_type) lives at offset 11 of every header; feeding +# these values steers the fuzzer toward exercising each per-type parser. +type_data="\x10" +type_grant="\x11" +type_resend="\x12" +type_rpc_unknown="\x13" +type_busy="\x14" +type_cutoffs="\x15" +type_freeze="\x16" +type_need_ack="\x17" +type_ack="\x18" +# doff for a standard 5-word TCP-style header (non-DATA packets expect 5<<4). +doff_five="\x50" +# Common sentinel offsets/lengths that hit boundary conditions. +zero32="\x00\x00\x00\x00" +neg32="\xff\xff\xff\xff" diff --git a/nix/fuzz/harness/fuzz_dissector.cc b/nix/fuzz/harness/fuzz_dissector.cc new file mode 100644 index 00000000..a771eb2f --- /dev/null +++ b/nix/fuzz/harness/fuzz_dissector.cc @@ -0,0 +1,49 @@ +/* nix/fuzz/harness/fuzz_dissector.cc — Focus C (kernel-independent) + * + * A standalone Homa wire-format decoder driven by libFuzzer. It classifies the packet type, + * validates the per-type header length (mirroring homa_plumbing.c's header_lengths[]), and + * for DATA packets walks the trailing segment headers doing the same offset/length + * arithmetic the kernel does — all under ASan/UBSan, so an out-of-bounds read or overflow in + * that arithmetic is caught. This needs no kernel headers or Wireshark, so it always builds, + * and it is the reference oracle used by fuzz_differential. + */ +#include "homa_wire_spec.h" +#include +#include + +using namespace homa_spec; + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + Decoded d = classify(data, size); + if (!d.valid_type || !d.long_enough) + return 0; /* the kernel would drop these early; nothing more to decode */ + + /* Read the common header safely (we know size >= header_length(type) >= sizeof(common)). */ + common_hdr common; + memcpy(&common, data, sizeof(common)); + + if (d.type == DATA) { + data_hdr dh; + memcpy(&dh, data, sizeof(dh)); + + /* message_length / segment walk (big-endian on the wire). Convert defensively. */ + auto be32 = [](uint32_t v) -> uint32_t { + return ((v & 0x000000ffu) << 24) | ((v & 0x0000ff00u) << 8) | + ((v & 0x00ff0000u) >> 8) | ((v & 0xff000000u) >> 24); + }; + uint32_t message_length = be32(dh.message_length); + (void)message_length; + + /* Walk segment headers in the tail, bounded strictly by `size`. */ + size_t pos = sizeof(data_hdr) - sizeof(seg_hdr); /* first seg is inside data_hdr */ + while (pos + sizeof(seg_hdr) <= size) { + seg_hdr seg; + memcpy(&seg, data + pos, sizeof(seg)); + uint32_t off = be32(seg.offset); + (void)off; + /* Advance by a seg header + at least one byte to guarantee progress. */ + pos += sizeof(seg_hdr) + 1; + } + } + return 0; +} diff --git a/nix/fuzz/harness/fuzz_guard.h b/nix/fuzz/harness/fuzz_guard.h new file mode 100644 index 00000000..1eab17ab --- /dev/null +++ b/nix/fuzz/harness/fuzz_guard.h @@ -0,0 +1,40 @@ +/* nix/fuzz/harness/fuzz_guard.h + * + * Sanitizer-independent overflow canary (adapted from the rocm-systems fuzzing design). + * A GuardedBuffer places known canary words immediately before and after an N-byte + * buffer; Check() aborts (so libFuzzer records the input) if either canary was smashed. + * Used by fuzz_selftest to prove the harness framework is not vacuous. + */ +#ifndef HOMA_FUZZ_GUARD_H +#define HOMA_FUZZ_GUARD_H + +#include +#include +#include +#include + +namespace homa_fuzz { + +static constexpr uint32_t kCanaryLo = 0xDEADBEEFu; +static constexpr uint32_t kCanaryHi = 0xCAFEBABEu; + +[[noreturn]] inline void Fail(const char *why) { + fprintf(stderr, "canary smashed: %s\n", why); + std::abort(); +} + +template +struct GuardedBuffer { + uint32_t lo = kCanaryLo; + uint8_t data[N] = {0}; + uint32_t hi = kCanaryHi; + + void Check() const { + if (lo != kCanaryLo) Fail("underflow (lo canary)"); + if (hi != kCanaryHi) Fail("overflow (hi canary)"); + } +}; + +} // namespace homa_fuzz + +#endif /* HOMA_FUZZ_GUARD_H */ diff --git a/nix/fuzz/harness/fuzz_selftest.cc b/nix/fuzz/harness/fuzz_selftest.cc new file mode 100644 index 00000000..484e48c2 --- /dev/null +++ b/nix/fuzz/harness/fuzz_selftest.cc @@ -0,0 +1,30 @@ +/* nix/fuzz/harness/fuzz_selftest.cc + * + * A deliberately fixed-size copy loop guarded by fuzz_guard.h. This harness is + * self-contained (no kernel/mock/Wireshark deps) so it ALWAYS builds — it exists to prove + * the libFuzzer + sanitizer toolchain and the canary net actually catch an overflow. + * + * The `nix build .#fuzz-selftest` gate injects a regression (widening the copy past the + * buffer) and asserts this harness aborts. In its shipped form the bound is correct, so a + * normal fuzz run finds nothing. + */ +#include "fuzz_guard.h" +#include +#include + +/* SELFTEST_COPY_LEN is the number of bytes copied into an 8-byte buffer. Shipped value (8) + * is safe; the selftest gate rewrites it larger to confirm the canary fires. */ +#ifndef SELFTEST_COPY_LEN +#define SELFTEST_COPY_LEN 8 +#endif + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + homa_fuzz::GuardedBuffer<8> buf; + size_t n = size < (size_t)SELFTEST_COPY_LEN ? size : (size_t)SELFTEST_COPY_LEN; + /* Intentionally copy up to SELFTEST_COPY_LEN bytes; if the gate widens that past 8 it + * overruns buf.data and smashes buf.hi. */ + for (size_t i = 0; i < n; i++) + buf.data[i] = data[i]; + buf.Check(); + return 0; +} diff --git a/nix/fuzz/harness/homa_wire_spec.h b/nix/fuzz/harness/homa_wire_spec.h new file mode 100644 index 00000000..f668662e --- /dev/null +++ b/nix/fuzz/harness/homa_wire_spec.h @@ -0,0 +1,129 @@ +/* nix/fuzz/harness/homa_wire_spec.h + * + * A dependency-free (no kernel, no Wireshark) mirror of Homa's on-wire header layout, taken + * verbatim from homa_wire.h (non-__STRIP__ variant). It lets the dissector/spec harness and + * the differential reference side decode raw bytes and reason about per-type header lengths + * exactly as homa_plumbing.c's header_lengths[] does. All multi-byte fields are big-endian + * on the wire; this decoder only needs the type byte and struct sizes, so it does not byte-swap. + * + * Kept in lock-step with homa_wire.h by construction (same field order/sizes + a static_assert + * on the type-byte offset). + */ +#ifndef HOMA_WIRE_SPEC_H +#define HOMA_WIRE_SPEC_H + +#include +#include + +namespace homa_spec { + +/* enum homa_packet_type (homa_wire.h) */ +enum PacketType : uint8_t { + DATA = 0x10, + GRANT = 0x11, + RESEND = 0x12, + RPC_UNKNOWN = 0x13, + BUSY = 0x14, + CUTOFFS = 0x15, + FREEZE = 0x16, + NEED_ACK = 0x17, + ACK = 0x18, + MIN_TYPE = 0x10, + MAX_TYPE = 0x18, +}; + +#define HOMA_MAX_PRIORITIES 8 +#define HOMA_MAX_ACKS_PER_PKT 5 + +#pragma pack(push, 1) + +struct common_hdr { + uint16_t sport; + uint16_t dport; + uint32_t sequence; + char ack[3]; + uint8_t type; /* offset 11 */ + uint8_t doff; + uint8_t flags; + uint16_t window; + uint16_t checksum; + uint16_t urgent; + uint64_t sender_id; +}; + +struct ack_t { + uint64_t client_id; + uint16_t server_port; +}; + +struct seg_hdr { uint32_t offset; }; + +struct data_hdr { + common_hdr common; + uint32_t message_length; + uint32_t incoming; + ack_t ack; + uint16_t cutoff_version; + uint8_t retransmit; + char pad[3]; + seg_hdr seg; +}; + +struct grant_hdr { common_hdr common; uint32_t offset; uint8_t priority; }; +struct resend_hdr { common_hdr common; uint32_t offset; uint32_t length; uint8_t priority; }; +struct rpc_unknown_hdr { common_hdr common; }; +struct busy_hdr { common_hdr common; }; +struct cutoffs_hdr { common_hdr common; uint32_t unsched_cutoffs[HOMA_MAX_PRIORITIES]; uint16_t cutoff_version; }; +struct freeze_hdr { common_hdr common; }; +struct need_ack_hdr { common_hdr common; }; +struct ack_hdr { common_hdr common; uint16_t num_acks; ack_t acks[HOMA_MAX_ACKS_PER_PKT]; }; + +#pragma pack(pop) + +static_assert(offsetof(common_hdr, type) == 11, "type byte must be at offset 11"); + +/* Minimum header length required for each type (mirrors header_lengths[] indexed by + * type - DATA in homa_plumbing.c). */ +inline size_t header_length(uint8_t type) { + switch (type) { + case DATA: return sizeof(data_hdr); + case GRANT: return sizeof(grant_hdr); + case RESEND: return sizeof(resend_hdr); + case RPC_UNKNOWN: return sizeof(rpc_unknown_hdr); + case BUSY: return sizeof(busy_hdr); + case CUTOFFS: return sizeof(cutoffs_hdr); + case FREEZE: return sizeof(freeze_hdr); + case NEED_ACK: return sizeof(need_ack_hdr); + case ACK: return sizeof(ack_hdr); + default: return 0; + } +} + +struct Decoded { + bool valid_type; /* type byte is a known Homa packet type */ + uint8_t type; + bool long_enough; /* buffer >= header_length(type) */ + size_t need; /* required header length for this type */ +}; + +/* Classify a raw packet the way the kernel's early parse does: read the type byte, look up + * the required header length, and check the buffer is long enough. */ +inline Decoded classify(const uint8_t *data, size_t size) { + Decoded d{}; + if (size <= offsetof(common_hdr, type)) { + d.valid_type = false; + d.type = 0; + d.long_enough = false; + d.need = 0; + return d; + } + d.type = data[offsetof(common_hdr, type)]; + d.valid_type = (d.type >= MIN_TYPE && d.type <= MAX_TYPE); + d.need = header_length(d.type); + d.long_enough = d.valid_type && size >= d.need; + return d; +} + +} // namespace homa_spec + +#endif /* HOMA_WIRE_SPEC_H */ diff --git a/nix/fuzz/harness/mock/fuzz_compat.c b/nix/fuzz/harness/mock/fuzz_compat.c new file mode 100644 index 00000000..aa6bff55 --- /dev/null +++ b/nix/fuzz/harness/mock/fuzz_compat.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: BSD-2-Clause or GPL-2.0+ +/* + * fuzz_compat.c - link-time glue for the kernel-mock fuzzers. + * + * With -fsanitize=address and CONFIG_GENERIC_ENTRY (but no KASAN), the + * kernel's fortify-string.h renames memcpy/memset/memmove to the KASAN + * software-tag symbols. Forward them to libc, where ASan intercepts them. + */ + +#include + +void *__hwasan_memcpy(void *p, const void *q, size_t size) +{ + return memcpy(p, q, size); +} + +void *__hwasan_memset(void *p, int c, size_t size) +{ + return memset(p, c, size); +} + +void *__hwasan_memmove(void *p, const void *q, size_t size) +{ + return memmove(p, q, size); +} diff --git a/nix/fuzz/harness/mock/fuzz_softirq.c b/nix/fuzz/harness/mock/fuzz_softirq.c new file mode 100644 index 00000000..90b0f19f --- /dev/null +++ b/nix/fuzz/harness/mock/fuzz_softirq.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: BSD-2-Clause or GPL-2.0+ +/* + * fuzz_softirq.c - libFuzzer harness that feeds packets through Homa's real + * receive path (homa_softirq -> homa_dispatch_pkts -> homa_*_pkt) on top of + * the unit-test mock layer (test/mock.c, test/utils.c). + * + * Each input sets up a small amount of RPC state (mirroring the + * homa_incoming unit-test fixture), then delivers up to MAX_PKTS packets + * whose bytes come from the fuzzer. Afterwards everything is torn down and + * the mock layer's leak checks (skbs, kmallocs, pages, locks, ...) are run; + * any FAIL() from those checks, or from a sanitizer, aborts the run. + * + * Input format: + * byte 0 scenario (see setup_scenario) + * byte 1 flags: bit 0 = aim sport/dport/sender_id at the scenario's + * RPC; bit 1 = IPv4 instead of IPv6; bit 2 = with scenario 6, + * aim at the server RPC instead of the client RPC + * then repeated [u16 little-endian length][length bytes] packets + * + * Build with -DPIN_TYPE= to force every packet's type byte to a + * single opcode, so that the fuzzer spends its time inside one handler. + */ + +#define KSELFTEST_NOT_MAIN 1 +#include "homa_impl.h" +#include "homa_grant.h" +#include "homa_qdisc.h" +#include "homa_rpc.h" +#include "homa_sock.h" +#include "homa_wire.h" +#include "kselftest_harness.h" +#include "ccutils.h" +#include "mock.h" +#include "utils.h" + +#define MAX_PKTS 8 + +/* test/mock.c reports leaks and lock imbalances with FAIL(), which + * clears __current_test->passed. main.c normally owns this storage. + */ +static struct __test_metadata fuzz_meta = {.name = "fuzz"}; +struct __test_metadata *__current_test = &fuzz_meta; +struct __test_metadata *__test_list; +unsigned int __test_count; +unsigned int __fixture_count; +int __constructor_order; + +struct fuzz_env { + struct in6_addr client_ip; + struct in6_addr server_ip; + struct homa homa; + struct homa_net *hnet; + struct homa_sock hsk; + struct homa_sock hsk2; + + /* Packets are aimed at this RPC when flags bit 0 is set. */ + struct in6_addr *saddr; + struct in6_addr *daddr; + u16 sport; + u16 dport; + u64 sender_id; +}; + +enum { + CLIENT_PORT = 40000, + SERVER_PORT = 99, + CLIENT_ID = 1234, + SERVER_ID = 1235, +}; + +static void aim_at_client(struct fuzz_env *env) +{ + env->saddr = &env->server_ip; + env->daddr = &env->client_ip; + env->sport = SERVER_PORT; + env->dport = env->hsk.port; + env->sender_id = SERVER_ID; +} + +static void aim_at_server(struct fuzz_env *env) +{ + env->saddr = &env->client_ip; + env->daddr = &env->server_ip; + env->sport = CLIENT_PORT; + env->dport = SERVER_PORT; + env->sender_id = CLIENT_ID; +} + +static void setup_scenario(struct fuzz_env *env, u8 scenario, u8 flags) +{ + aim_at_server(env); + switch (scenario % 8) { + case 0: + /* No RPCs at all: exercises the unknown-RPC paths. */ + break; + case 1: + unit_client_rpc(&env->hsk, UNIT_OUTGOING, &env->client_ip, + &env->server_ip, SERVER_PORT, CLIENT_ID, 2000, + 10000); + aim_at_client(env); + break; + case 2: + unit_client_rpc(&env->hsk, UNIT_RCVD_ONE_PKT, &env->client_ip, + &env->server_ip, SERVER_PORT, CLIENT_ID, 2000, + 10000); + aim_at_client(env); + break; + case 3: + unit_server_rpc(&env->hsk2, UNIT_RCVD_ONE_PKT, &env->client_ip, + &env->server_ip, CLIENT_PORT, SERVER_ID, 10000, + 100); + break; + case 4: + unit_server_rpc(&env->hsk2, UNIT_IN_SERVICE, &env->client_ip, + &env->server_ip, CLIENT_PORT, SERVER_ID, 10000, + 100); + break; + case 5: + unit_server_rpc(&env->hsk2, UNIT_OUTGOING, &env->client_ip, + &env->server_ip, CLIENT_PORT, SERVER_ID, 100, + 20000); + break; + case 6: + unit_client_rpc(&env->hsk, UNIT_OUTGOING, &env->client_ip, + &env->server_ip, SERVER_PORT, CLIENT_ID, 20000, + 1000); + unit_server_rpc(&env->hsk2, UNIT_RCVD_ONE_PKT, &env->client_ip, + &env->server_ip, CLIENT_PORT, SERVER_ID + 2, + 10000, 100); + if (flags & 4) + env->sender_id = CLIENT_ID + 2; + else + aim_at_client(env); + break; + case 7: + unit_client_rpc(&env->hsk, UNIT_RCVD_MSG, &env->client_ip, + &env->server_ip, SERVER_PORT, CLIENT_ID, 2000, + 1000); + aim_at_client(env); + break; + } +} + +static void deliver(struct fuzz_env *env, const u8 *data, size_t len, + u8 flags) +{ + struct homa_common_hdr *h; + struct sk_buff *skb; + + skb = mock_skb_alloc(env->saddr, env->daddr, NULL, len, 0); + h = skb_put(skb, len); + memcpy(h, data, len); + if (len >= sizeof(*h)) { +#ifdef PIN_TYPE + h->type = PIN_TYPE; +#endif + if (flags & 1) { + h->sport = htons(env->sport); + h->dport = htons(env->dport); + h->sender_id = cpu_to_be64(env->sender_id); + } + } + + /* FREEZE is a debugging aid that freezes the timetrace; it isn't + * part of the protocol, so keep the fuzzer out of it. + */ + if (len > offsetof(struct homa_common_hdr, type) && h->type == FREEZE) + h->type = DATA; + homa_softirq(skb); +} + +int LLVMFuzzerTestOneInput(const u8 *data, size_t size) +{ + static struct fuzz_env env; + u8 scenario, flags; + int npkts; + + if (size < 2) + return 0; + scenario = data[0]; + flags = data[1]; + data += 2; + size -= 2; + + memset(&env, 0, sizeof(env)); + fuzz_meta.passed = 1; + mock_ipv6_default = !(flags & 2); + mock_ipv6 = mock_ipv6_default; + env.client_ip = unit_get_in_addr("196.168.0.1"); + env.server_ip = unit_get_in_addr("1.2.3.4"); + homa_init(&env.homa); + env.hnet = mock_hnet(0, &env.homa); + env.homa.num_priorities = 1; + env.homa.poll_cycles = 0; + env.homa.qshared->fifo_fraction = 0; + env.homa.unsched_bytes = 10000; + env.homa.grant->window = 10000; + mock_sock_init(&env.hsk, env.hnet, 0); + mock_sock_init(&env.hsk2, env.hnet, SERVER_PORT); + setup_scenario(&env, scenario, flags); + unit_log_clear(); + + for (npkts = 0; npkts < MAX_PKTS && size >= 2; npkts++) { + size_t len = data[0] | (data[1] << 8); + + data += 2; + size -= 2; + if (len > size) + len = size; + if (len > 1500) + len = 1500; + deliver(&env, data, len, flags); + data += len; + size -= len; + } + + homa_destroy(&env.homa); + unit_teardown(); + if (!fuzz_meta.passed) + abort(); + return 0; +} diff --git a/nix/fuzz/known-findings.txt b/nix/fuzz/known-findings.txt new file mode 100644 index 00000000..abec9452 --- /dev/null +++ b/nix/fuzz/known-findings.txt @@ -0,0 +1,4 @@ +# Sanitizer SUMMARY locations already triaged (see https://github.com/randomizedcoder/HomaModule/blob/nix-flake/analysis-reports/round2/fuzzing-report.md). +# The fuzz-mock smoke gate tolerates these; any other finding fails the build. +# One "file:line" per line, matched against the sanitizer SUMMARY line. +# (none: homa_incoming.c:63, small_msg_bytes[-1], fixed upstream by 1d38026 / PR #121) diff --git a/nix/fuzz/tools/build-mock-fuzzers.sh b/nix/fuzz/tools/build-mock-fuzzers.sh new file mode 100755 index 00000000..2f13d76e --- /dev/null +++ b/nix/fuzz/tools/build-mock-fuzzers.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# nix/fuzz/tools/build-mock-fuzzers.sh +# +# Builds the kernel-mock libFuzzer harnesses (nix/fuzz/harness/mock/*.c) by +# compiling Homa and the unit-test mock layer exactly the way test/Makefile +# does, but with clang + libFuzzer + ASan/UBSan. +# +# Usage: build-mock-fuzzers.sh [extra-cflags] +# a merged KDIR such as nix/kernel/unit-kdir produces +# [extra-cflags] e.g. unitKdir.extraCFlags +# +# Writes /bin/, /status.txt and /build.log. +# Exits non-zero if the shared objects or any harness fail to build. + +set -u +SRC=$(cd "$1" && pwd) +KDIR=$2 +OUT=$3 +EXTRA=${4:-} +CC=${CC:-clang} +CXX=${CXX:-clang++} +JOBS=${JOBS:-$(nproc)} + +mkdir -p "$OUT/bin" "$OUT/obj" +OBJ="$OUT/obj" +LOG="$OUT/build.log" +STATUS="$OUT/status.txt" +: > "$LOG" +: > "$STATUS" + +INC="-I$SRC/test -I$SRC \ + -I$KDIR/arch/x86/include -I$KDIR/arch/x86/include/generated \ + -I$KDIR/include -I$KDIR/arch/x86/include/uapi \ + -I$KDIR/arch/x86/include/generated/uapi -I$KDIR/include/uapi \ + -I$KDIR/include/generated/uapi -include $KDIR/include/linux/kconfig.h" +DEFS="-D__KERNEL__ -D__UNIT_TEST__ -DKBUILD_MODNAME='\"homa\"' -DCC_USING_FENTRY" +# alignment is off: list_for_each_entry() on an empty list forms container_of(head) for +# cacheline-aligned types, and mock kmalloc() is plain malloc(). +SAN="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize=alignment -fno-sanitize-recover=undefined" +# clang makes this an error by default; it is the 6.18 recvmsg() signature change. +CFLAGS="-std=gnu11 -fms-extensions -fno-pie -g -O1 -fno-omit-frame-pointer -w \ + -Wno-error=incompatible-function-pointer-types $SAN $DEFS $INC $EXTRA" +CXXFLAGS="-std=c++11 -g -O1 -fno-omit-frame-pointer -w $SAN $DEFS -I$SRC/test -I$SRC" + +HOMA_SRCS="homa_devel homa_interest homa_incoming homa_outgoing homa_peer + homa_pool homa_plumbing homa_rpc homa_sock homa_timer homa_tx_pool + homa_utils timetrace homa_grant homa_hijack homa_metrics homa_offload + homa_qdisc" +LINUX_OBJS=$(bash "$SRC/test/linux_objs" "$KDIR/include/generated/uapi/linux/version.h") + +compile() { # + eval "$CC -c $1 \"$2\" -o \"$3\"" >> "$LOG" 2>&1 || + { echo "FAILED $2" >> "$STATUS"; return 1; } +} + +pids="" +shared="" +for m in $HOMA_SRCS; do + compile "$CFLAGS" "$SRC/$m.c" "$OBJ/$m.o" & pids="$pids $!" + shared="$shared $OBJ/$m.o" + while [ "$(jobs -rp | wc -l)" -ge "$JOBS" ]; do wait -n; done +done +for o in $LINUX_OBJS; do + b=${o%.o} + case $b in + # As in test/Makefile: these need -O2 and no ASan. + rhashtable*|xxhash*) f="$CFLAGS -O2 -fno-sanitize=address" ;; + *) f="$CFLAGS" ;; + esac + compile "$f" "$SRC/test/$b.c" "$OBJ/$o" & pids="$pids $!" + shared="$shared $OBJ/$o" +done +for m in mock utils; do + compile "$CFLAGS" "$SRC/test/$m.c" "$OBJ/$m.o" & pids="$pids $!" + shared="$shared $OBJ/$m.o" +done +# Plain C, no kernel headers. +$CC -c -O1 -g "$SRC/nix/fuzz/harness/mock/fuzz_compat.c" -o "$OBJ/fuzz_compat.o" \ + >> "$LOG" 2>&1 || echo "FAILED fuzz_compat.c" >> "$STATUS" +shared="$shared $OBJ/fuzz_compat.o" +eval "$CXX -c $CXXFLAGS \"$SRC/test/ccutils.cc\" -o \"$OBJ/ccutils.o\"" \ + >> "$LOG" 2>&1 || echo "FAILED ccutils.cc" >> "$STATUS" +shared="$shared $OBJ/ccutils.o" + +ok=1 +for p in $pids; do wait "$p" || ok=0; done +grep -q FAILED "$STATUS" && ok=0 +if [ $ok != 1 ]; then + echo "shared objects failed; see build.log" >> "$STATUS" + cat "$STATUS" + grep -m20 -E 'error|fatal' "$LOG" + exit 1 +fi +echo "shared objects OK ($(basename "$KDIR"))" >> "$STATUS" + +# name:source:extra-defines. The PIN_TYPE variants keep the fuzzer inside a +# single packet handler. +HARNESSES=" +fuzz_softirq:fuzz_softirq.c: +fuzz_data_pkt:fuzz_softirq.c:-DPIN_TYPE=DATA +fuzz_grant_pkt:fuzz_softirq.c:-DPIN_TYPE=GRANT +fuzz_resend_pkt:fuzz_softirq.c:-DPIN_TYPE=RESEND +fuzz_rpc_unknown_pkt:fuzz_softirq.c:-DPIN_TYPE=RPC_UNKNOWN +fuzz_busy_pkt:fuzz_softirq.c:-DPIN_TYPE=BUSY +fuzz_cutoffs_pkt:fuzz_softirq.c:-DPIN_TYPE=CUTOFFS +fuzz_need_ack_pkt:fuzz_softirq.c:-DPIN_TYPE=NEED_ACK +fuzz_ack_pkt:fuzz_softirq.c:-DPIN_TYPE=ACK +fuzz_start_msg_pkt:fuzz_softirq.c:-DPIN_TYPE=START_MSG +" +rc=0 +for h in $HARNESSES; do + name=${h%%:*}; rest=${h#*:}; file=${rest%%:*}; defs=${rest#*:} + if compile "$CFLAGS $defs" "$SRC/nix/fuzz/harness/mock/$file" "$OBJ/$name.o" && + $CXX -no-pie -fsanitize=fuzzer,address,undefined $shared "$OBJ/$name.o" \ + -o "$OUT/bin/$name" >> "$LOG" 2>&1; then + echo "BUILT $name" >> "$STATUS" + else + echo "FAILED $name" >> "$STATUS" + rc=1 + fi +done +cat "$STATUS" +exit $rc diff --git a/nix/fuzz/tools/fuzz-cov.nix b/nix/fuzz/tools/fuzz-cov.nix new file mode 100644 index 00000000..e90a3f03 --- /dev/null +++ b/nix/fuzz/tools/fuzz-cov.nix @@ -0,0 +1,40 @@ +# nix/fuzz/tools/fuzz-cov.nix +# +# Source-based coverage of the standalone wire decoder (fuzz_dissector), replayed over its +# corpus, reported with llvm-cov. Gives a concrete coverage number for the wire +# (de)serialization decode logic. Kernel-free. + +{ pkgs, mkFuzzDrv, compileHarness }: + +mkFuzzDrv { + pname = "homa-fuzz-cov"; + nativeBuildInputs = [ pkgs.llvm ]; + + buildPhase = '' + runHook preBuild + mkdir -p "$out" + ${compileHarness { + name = "fuzz_dissector"; + out = "fuzz_dissector_cov"; + flags = "-fprofile-instr-generate -fcoverage-mapping"; + }} + + work="$PWD/work"; mkdir -p "$work" + cp nix/fuzz/corpus-seeds/dissector/* "$work/" 2>/dev/null || true + LLVM_PROFILE_FILE="$PWD/dissector.profraw" \ + ./fuzz_dissector_cov -runs=200000 -max_total_time=20 "$work" > "$out/cov.log" 2>&1 || true + + llvm-profdata merge -sparse dissector.profraw -o dissector.profdata 2>/dev/null || true + { + echo "# fuzz-cov: coverage of the Homa wire decoder (fuzz_dissector)" + llvm-cov report ./fuzz_dissector_cov -instr-profile=dissector.profdata 2>&1 || true + } > "$out/summary.txt" + llvm-cov show ./fuzz_dissector_cov -instr-profile=dissector.profdata \ + > "$out/dissector.show.txt" 2>&1 || true + cp dissector.profdata "$out/" 2>/dev/null || true + tail -n 20 "$out/summary.txt" + runHook postBuild + ''; + + dontInstall = true; +} diff --git a/nix/fuzz/tools/fuzz-mock.nix b/nix/fuzz/tools/fuzz-mock.nix new file mode 100644 index 00000000..ad80e93c --- /dev/null +++ b/nix/fuzz/tools/fuzz-mock.nix @@ -0,0 +1,68 @@ +# nix/fuzz/tools/fuzz-mock.nix +# +# Builds the KERNEL-MOCK libFuzzer harnesses (nix/fuzz/harness/mock/*.c). These feed fuzzer +# bytes through Homa's real receive path (homa_softirq -> homa_dispatch_pkts -> homa_*_pkt) +# on top of the unit-test mock layer (test/mock.c), compiled with clang + libFuzzer + +# ASan/UBSan against the same merged KDIR the unit tests use (nix/kernel/unit-kdir). +# +# The build logic lives in build-mock-fuzzers.sh so it can also be run by hand. This target +# is a gate: it fails unless every harness builds and runs its seed corpus cleanly. +# +# Output: $out/bin/, $out/corpus/ (generated seeds), $out/status.txt, and +# $out/bin/run-mock-fuzzers [seconds-per-harness] [outdir] for a time-boxed campaign (fork mode, +# one job per core). + +{ pkgs, unitKdir, mkFuzzDrv, launcher }: + +let + runner = launcher "run-mock-fuzzers" '' + FUZZ_TIME=''${1:-60} + FUZZ_FORKS=''${FUZZ_FORKS:-$(nproc)} + export FUZZ_TIME FUZZ_FORKS FUZZ_MAXLEN=4096 + set -- "''${2:-mock-fuzz-out}" + ''; +in +mkFuzzDrv { + pname = "homa-fuzzers-mock"; + nativeBuildInputs = [ pkgs.llvmPackages.compiler-rt pkgs.python3 ]; + + buildPhase = '' + runHook preBuild + CC=clang CXX=clang++ JOBS=$NIX_BUILD_CORES \ + bash nix/fuzz/tools/build-mock-fuzzers.sh . ${unitKdir} out "${unitKdir.extraCFlags}" + python3 nix/fuzz/tools/gen-mock-seeds.py out/corpus + runHook postBuild + ''; + + # Smoke gate: every harness must get through the seeds with no sanitizer report and no + # mock-layer leak, except findings already listed in nix/fuzz/known-findings.txt. + doCheck = true; + checkPhase = '' + runHook preCheck + known=$(grep -v '^#' nix/fuzz/known-findings.txt | grep . || true) + for f in out/bin/fuzz_*; do + name=$(basename "$f") + if "$f" -runs=0 out/corpus > "smoke-$name.log" 2>&1; then + echo "smoke: $name OK" + continue + fi + summary=$(grep -m1 SUMMARY "smoke-$name.log" || true) + if [ -n "$summary" ] && [ -n "$known" ] && echo "$summary" | grep -qF "$known"; then + echo "smoke: $name known finding: $summary" + else + cat "smoke-$name.log" + echo "smoke: $name FAILED" + exit 1 + fi + done + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + mkdir -p "$out" + cp -r out/bin out/corpus out/status.txt out/build.log "$out/" + cp ${runner}/bin/run-mock-fuzzers "$out/bin/" + runHook postInstall + ''; +} diff --git a/nix/fuzz/tools/fuzz-selftest.nix b/nix/fuzz/tools/fuzz-selftest.nix new file mode 100644 index 00000000..5d2ba61c --- /dev/null +++ b/nix/fuzz/tools/fuzz-selftest.nix @@ -0,0 +1,43 @@ +# nix/fuzz/tools/fuzz-selftest.nix +# +# Canary gate (rocm-systems idea): compile fuzz_selftest with the copy bound deliberately +# widened past the 8-byte guarded buffer, feed it an overflowing input, and REQUIRE that it +# aborts (via the fuzz_guard canary or ASan). If it does not, the harness/toolchain is +# vacuous and this build fails. This is the one fuzz target that is a real pass/fail gate. + +{ mkFuzzDrv, compileHarness }: + +mkFuzzDrv { + pname = "homa-fuzz-selftest"; + + buildPhase = '' + runHook preBuild + mkdir -p "$out" + # Inject the regression: widen the copy to 32 bytes into an 8-byte buffer. + ${compileHarness { + name = "fuzz_selftest"; + out = "selftest_broken"; + flags = "-DSELFTEST_COPY_LEN=32"; + }} + head -c 32 /dev/zero | tr '\0' 'A' > overflow.bin + + set +e + ASAN_OPTIONS=abort_on_error=1 ./selftest_broken overflow.bin > "$out/selftest.log" 2>&1 + rc=$? + set -e + + if [ "$rc" -eq 0 ]; then + echo "SELFTEST FAILED: harness did NOT catch the injected overflow" | tee "$out/verdict.txt" >&2 + exit 1 + fi + if grep -qiE 'canary smashed|AddressSanitizer|stack-buffer-overflow|global-buffer-overflow' "$out/selftest.log"; then + echo "SELFTEST PASS: injected overflow detected (rc=$rc)" | tee "$out/verdict.txt" + else + echo "SELFTEST FAILED: process aborted (rc=$rc) but no overflow signature found" | tee "$out/verdict.txt" >&2 + exit 1 + fi + runHook postBuild + ''; + + dontInstall = true; +} diff --git a/nix/fuzz/tools/gen-mock-seeds.py b/nix/fuzz/tools/gen-mock-seeds.py new file mode 100644 index 00000000..fea958e8 --- /dev/null +++ b/nix/fuzz/tools/gen-mock-seeds.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Generate seed inputs for the kernel-mock fuzzers (nix/fuzz/harness/mock). + +Usage: gen-mock-seeds.py + +Each seed is: scenario byte, flags byte, then [u16 LE length][packet] records +(see fuzz_softirq.c). Flags bit 0 is set so the harness aims the packets at +the scenario's RPC; only the type-specific fields need to be plausible. +""" + +import os +import struct +import sys + +DATA, GRANT, RESEND, RPC_UNKNOWN, BUSY, CUTOFFS = 0x10, 0x11, 0x12, 0x13, 0x14, 0x15 +NEED_ACK, ACK, START_MSG = 0x17, 0x18, 0x19 + + +def common(ptype): + # sport, dport, sequence, ack[3], type, doff, flags, window, checksum, + # urgent, sender_id (the harness overwrites ports and sender_id). + return struct.pack(">HHI3sBBBHHHQ", 0, 0, 0, b"\0\0\0", ptype, 0, 0, 0, 0, 0, 0) + + +def homa_ack(client_id=0, port=0): + return struct.pack(">QH", client_id, port) + + +def data(msg_length, offset, payload=1000, retransmit=0): + hdr = (common(DATA) + struct.pack(">II", msg_length, 0) + homa_ack() + + struct.pack(">HHB3s", 0, 0, retransmit, b"\0\0\0") + + struct.pack(">I", offset)) + return hdr + bytes((i & 0xff for i in range(payload))) + + +def grant(offset, priority=0): + return common(GRANT) + struct.pack(">IB", offset, priority) + + +def resend(offset, length, priority=0): + return common(RESEND) + struct.pack(">IIB", offset, length, priority) + + +def start_msg(msg_length): + return common(START_MSG) + struct.pack(">I", msg_length) + + +def ack(acks): + body = b"".join(homa_ack(c, p) for c, p in acks) + body += homa_ack() * (5 - len(acks)) + return common(ACK) + struct.pack(">H", len(acks)) + body + + +def cutoffs(version=1): + vals = [1000000, 12288, 8192, 4096, 2048, 1024, 512, 256] + return common(CUTOFFS) + struct.pack(">8IH", *vals, version) + + +def seed(scenario, *pkts, flags=1): + out = bytes([scenario, flags]) + for p in pkts: + out += struct.pack(" [outdir] +# +# Runs every fuzz_* harness in bindir, one after another, seeded from corpus// +# (or from corpus/ itself when it has no directory for that harness), then reproduces each +# crash it found and keeps one per sanitizer SUMMARY line. Every fuzz target uses this: +# run-fuzzers, run-mock-fuzzers, fuzz-run and fuzz-deep (nix/fuzz/default.nix). +# +# outdir (default $FUZZ_WORKDIR, else a new temporary directory) gets: +# / the harness's working corpus, and the crash-* inputs it found +# .log libFuzzer's output +# crashes// crash- and repro-.txt for each distinct crash +# summary.txt per harness: coverage, crash inputs, distinct crashes +# count.txt crash inputs in all; distinct_count.txt: distinct crashes +# +# Environment: +# FUZZ_TIME seconds per harness (60) +# FUZZ_FORKS libFuzzer -fork= jobs, with -ignore_crashes=1 (0: run in-process) +# FUZZ_MAXLEN libFuzzer -max_len= +# FUZZ_DICT a libFuzzer dictionary + +bindir=$1 +corpus=$2 +out=${3:-${FUZZ_WORKDIR:-$(mktemp -d)}} +time=${FUZZ_TIME:-60} +forks=${FUZZ_FORKS:-0} + +args=(-max_total_time="$time") +if [ "$forks" -gt 0 ]; then + args+=(-fork="$forks" -ignore_crashes=1) +fi +if [ -n "${FUZZ_MAXLEN:-}" ]; then + args+=(-max_len="$FUZZ_MAXLEN") +fi +if [ -n "${FUZZ_DICT:-}" ]; then + args+=(-dict="$FUZZ_DICT") +fi +export ASAN_OPTIONS=abort_on_error=0:detect_leaks=0:symbolize=1 +export UBSAN_OPTIONS=print_stacktrace=1 + +mkdir -p "$out/crashes" +echo "=== Homa fuzzers: ${time} s per harness, ${forks} fork(s) ===" >"$out/summary.txt" +total=0 +distinct=0 +for bin in "$bindir"/fuzz_*; do + name=$(basename "$bin") + work=$out/$name + seeds=$corpus/$name + [ -d "$seeds" ] || seeds=$corpus + mkdir -p "$work" + cp -rn "$seeds"/. "$work/" 2>/dev/null || true + "$bin" "${args[@]}" -artifact_prefix="$work/" "$work" >"$out/$name.log" 2>&1 || true + + # One crash kept per signature: the SUMMARY line, without addresses. + declare -A seen=() + crashes=0 + kept=0 + while IFS= read -r crash; do + crashes=$((crashes + 1)) + sig=$("$bin" "$crash" 2>&1 | grep -m 1 -oE 'SUMMARY: .*' | sed 's/0x[0-9a-f]*//g' || true) + sig=${sig:-unknown-$crashes} + hash=$(printf '%s' "$sig" | cksum | cut -d ' ' -f 1) + if [ -z "${seen[$hash]:-}" ]; then + seen[$hash]=1 + kept=$((kept + 1)) + mkdir -p "$out/crashes/$name" + cp "$crash" "$out/crashes/$name/crash-$hash" + "$bin" "$crash" >"$out/crashes/$name/repro-$hash.txt" 2>&1 || true + fi + done < <(find "$work" -maxdepth 1 -name 'crash-*') + unset seen + + cov=$(grep -oE 'cov: [0-9]+' "$out/$name.log" | tail -n 1 || true) + printf ' %-20s %-10s %s crash input(s), %s distinct\n' \ + "$name:" "${cov:-cov: ?}" "$crashes" "$kept" | tee -a "$out/summary.txt" + total=$((total + crashes)) + distinct=$((distinct + kept)) +done +{ + echo + echo "Total crash inputs: $total Distinct crashes: $distinct (in $out)" +} | tee -a "$out/summary.txt" +echo "$total" >"$out/count.txt" +echo "$distinct" >"$out/distinct_count.txt" diff --git a/nix/parts/README.md b/nix/parts/README.md index f002d6ab..d96d48b7 100644 --- a/nix/parts/README.md +++ b/nix/parts/README.md @@ -9,6 +9,7 @@ outputs are what `nix build .#`, `nix run`, `nix develop` and `nix flake c | `scope.nix` | none: builds the package scope ([`../scope.nix`](../scope.nix)) and passes it to the others as `homa` | | `build.nix` | `homa-module*`, `homa-build-matrix`, `homa-utils*`, `homa-tests`, `homa-image*`, cross builds (x86_64 hosts only); `default` is `homa-module` | | `analysis.nix` | `analysis-*`; check `analysis-quick` | +| `fuzz.nix` | `fuzz*`; check `fuzz-selftest` | | `dev.nix` | `devShells.default` (`nix develop`) and `formatter` (`nix fmt`) | ## How a part works diff --git a/nix/parts/fuzz.nix b/nix/parts/fuzz.nix new file mode 100644 index 00000000..b4b12928 --- /dev/null +++ b/nix/parts/fuzz.nix @@ -0,0 +1,9 @@ +# nix/parts/fuzz.nix +# +# The libFuzzer harnesses (../fuzz): fuzz-selftest is a check, the rest are packages. + +{ + perSystem = { homa, ... }: { + inherit (homa.fuzzPkgs) packages checks; + }; +} diff --git a/nix/scope.nix b/nix/scope.nix index cc6a50a5..dccda0ff 100644 --- a/nix/scope.nix +++ b/nix/scope.nix @@ -41,5 +41,6 @@ pkgs.lib.makeScope pkgs.newScope (self: { inherit (self.userspacePkgs) homa-utils homa-utils-debug; }; analysisPkgs = self.callPackage ./analysis { }; + fuzzPkgs = self.callPackage ./fuzz { }; devShell = self.callPackage ./dev/shell.nix { }; }) From 1b4f1838707d986291aa68c64bbf3b335e78eca5 Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Fri, 30 Oct 2026 18:26:13 -0700 Subject: [PATCH 04/11] nix: microVM tests: single VMs, lifecycles, VM pairs and a matrix QEMU microVMs (microvm.nix) that boot a chosen kernel and load the homa.ko built for it, with the host's /nix/store shared in, so no image is built. No root, tap or bridge: the pair link is a QEMU UDP socket on 127.0.0.1. - homa-microvm-test[-]: insmod, echo RPCs over loopback and between two network namespaces (GRO + RPS), rmmod, dmesg gate; -debug runs it on a KASAN/KMEMLEAK kernel - homa-lifecycle[-]: the same steps as timed phases, driven over the serial and a virtio console, plus repeated module load/unload - homa-pair-test[--]: two VMs, echo and cp_node (Homa and TCP) across the link, for every pairing of kernels (wire compatibility between builds) - homa-pair-lifecycle[-]: the pair over its consoles, with tc netem loss and delay phases and Homa's qdisc and pacer - homa-vm-matrix: all of them in parallel, with a summary table; homa-vm-compare compares two runs phase by phase The host-side drivers are a Tcl/Expect package (nix/microvms/tcl) with table-driven tcltest suites, nagelfar and tclint, run by the `tcl-tests` check. The dev shell gets the Tcl tools. Co-Authored-By: Claude Opus 5.5 --- flake.nix | 11 + nix/dev/packages.nix | 12 +- nix/dev/shell.nix | 5 + nix/microvms/PLAN-pairs.md | 102 +++ nix/microvms/README.md | 52 ++ nix/microvms/constants.nix | 103 +++ nix/microvms/default.nix | 234 ++++++ nix/microvms/guest/README.md | 29 + nix/microvms/guest/metrics.nix | 94 +++ nix/microvms/guest/mkVm.nix | 213 +++++ nix/microvms/guest/pair.nix | 230 +++++ nix/microvms/guest/qdisc.nix | 40 + nix/microvms/guest/smoke.nix | 120 +++ nix/microvms/host/README.md | 21 + nix/microvms/host/lifecycle.nix | 41 + nix/microvms/host/matrix.nix | 28 + nix/microvms/host/pair-lifecycle.nix | 75 ++ nix/microvms/host/pair-test.nix | 100 +++ nix/microvms/host/smoke-test.nix | 52 ++ nix/microvms/host/tcl.nix | 162 ++++ nix/microvms/ports.nix | 68 ++ nix/microvms/tcl/README.md | 161 ++++ nix/microvms/tcl/bin/lifecycle.tcl | 93 +++ nix/microvms/tcl/bin/pair-lifecycle.tcl | 509 ++++++++++++ nix/microvms/tcl/bin/vm-compare.tcl | 94 +++ nix/microvms/tcl/bin/vm-matrix.tcl | 83 ++ nix/microvms/tcl/console.tcl | 180 ++++ nix/microvms/tcl/lc.tcl | 1016 +++++++++++++++++++++++ nix/microvms/tcl/parse.tcl | 519 ++++++++++++ nix/microvms/tcl/pkgIndex.tcl | 11 + nix/microvms/tcl/protocol.tcl | 110 +++ nix/microvms/tcl/report.tcl | 310 +++++++ nix/microvms/tcl/tclint.toml | 8 + nix/microvms/tcl/tests/all.tcl | 25 + nix/microvms/tcl/tests/batch.test | 181 ++++ nix/microvms/tcl/tests/console.test | 182 ++++ nix/microvms/tcl/tests/fakeqemu.tcl | 80 ++ nix/microvms/tcl/tests/files.test | 121 +++ nix/microvms/tcl/tests/helpers.tcl | 119 +++ nix/microvms/tcl/tests/lintdb.tcl | 137 +++ nix/microvms/tcl/tests/node.test | 172 ++++ nix/microvms/tcl/tests/parse.test | 282 +++++++ nix/microvms/tcl/tests/protocol.test | 67 ++ nix/microvms/tcl/tests/report.test | 168 ++++ nix/microvms/tcl/tests/style.test | 235 ++++++ nix/microvms/tcl/tests/table.test | 157 ++++ nix/microvms/tcl/tests/wait.test | 94 +++ nix/parts/README.md | 1 + nix/parts/vms.nix | 9 + nix/scope.nix | 5 + 50 files changed, 6920 insertions(+), 1 deletion(-) create mode 100644 nix/microvms/PLAN-pairs.md create mode 100644 nix/microvms/README.md create mode 100644 nix/microvms/constants.nix create mode 100644 nix/microvms/default.nix create mode 100644 nix/microvms/guest/README.md create mode 100644 nix/microvms/guest/metrics.nix create mode 100644 nix/microvms/guest/mkVm.nix create mode 100644 nix/microvms/guest/pair.nix create mode 100644 nix/microvms/guest/qdisc.nix create mode 100644 nix/microvms/guest/smoke.nix create mode 100644 nix/microvms/host/README.md create mode 100644 nix/microvms/host/lifecycle.nix create mode 100644 nix/microvms/host/matrix.nix create mode 100644 nix/microvms/host/pair-lifecycle.nix create mode 100644 nix/microvms/host/pair-test.nix create mode 100644 nix/microvms/host/smoke-test.nix create mode 100644 nix/microvms/host/tcl.nix create mode 100644 nix/microvms/ports.nix create mode 100644 nix/microvms/tcl/README.md create mode 100644 nix/microvms/tcl/bin/lifecycle.tcl create mode 100644 nix/microvms/tcl/bin/pair-lifecycle.tcl create mode 100644 nix/microvms/tcl/bin/vm-compare.tcl create mode 100644 nix/microvms/tcl/bin/vm-matrix.tcl create mode 100644 nix/microvms/tcl/console.tcl create mode 100644 nix/microvms/tcl/lc.tcl create mode 100644 nix/microvms/tcl/parse.tcl create mode 100644 nix/microvms/tcl/pkgIndex.tcl create mode 100644 nix/microvms/tcl/protocol.tcl create mode 100644 nix/microvms/tcl/report.tcl create mode 100644 nix/microvms/tcl/tclint.toml create mode 100644 nix/microvms/tcl/tests/all.tcl create mode 100644 nix/microvms/tcl/tests/batch.test create mode 100644 nix/microvms/tcl/tests/console.test create mode 100644 nix/microvms/tcl/tests/fakeqemu.tcl create mode 100644 nix/microvms/tcl/tests/files.test create mode 100644 nix/microvms/tcl/tests/helpers.tcl create mode 100644 nix/microvms/tcl/tests/lintdb.tcl create mode 100644 nix/microvms/tcl/tests/node.test create mode 100644 nix/microvms/tcl/tests/parse.test create mode 100644 nix/microvms/tcl/tests/protocol.test create mode 100644 nix/microvms/tcl/tests/report.test create mode 100644 nix/microvms/tcl/tests/style.test create mode 100644 nix/microvms/tcl/tests/table.test create mode 100644 nix/microvms/tcl/tests/wait.test create mode 100644 nix/parts/vms.nix diff --git a/flake.nix b/flake.nix index 2d1057e7..7b7cb3c5 100644 --- a/flake.nix +++ b/flake.nix @@ -17,6 +17,16 @@ # nix build .#homa-utils-debug # user-space tools with AddressSanitizer # nix build .#homa-tests # build + run the unit tests (ASan) # +# nix run .#homa-microvm-test # boot a microVM, insmod homa.ko, loopback echo RPCs +# nix run .#homa-microvm-test-6_12 # ... per VM kernel: 6_12 7_3 (kernel/config.nix vmSeries) +# nix run .#homa-microvm-test-debug # ... under a KASAN/KMEMLEAK kernel +# nix run .#homa-lifecycle[-6_12] # timed phases driven over ttyS0 + a virtio console +# nix run .#homa-pair-test # two VMs: echo + cp_node (Homa, TCP) over a link +# nix run .#homa-pair-test-6_12-7_3 # ... server kernel, client kernel (every pairing) +# nix run .#homa-pair-lifecycle[-6_12] # the pair over its consoles: netem loss, Homa qdisc +# nix run .#homa-pair-exp # latency sweep (sysctls x workloads, metric deltas) +# nix run .#homa-vm-matrix # every VM test + pair in parallel, summary table +# # nix build .#homa-image # OCI userspace image (docker load < result) # nix build .#homa-image-debug # @@ -65,6 +75,7 @@ ./nix/parts/build.nix ./nix/parts/analysis.nix ./nix/parts/fuzz.nix + ./nix/parts/vms.nix ./nix/parts/dev.nix ]; }; diff --git a/nix/dev/packages.nix b/nix/dev/packages.nix index 6acfc5e2..a5b71502 100644 --- a/nix/dev/packages.nix +++ b/nix/dev/packages.nix @@ -35,8 +35,18 @@ let pyTools = [ pkgs.ruff pkgs.mypy pythonEnv ]; shTools = [ pkgs.shellcheck pkgs.shfmt ]; docTools = [ pkgs.groff pkgs.ghostscript ]; # man/ PDF generation + + # The VM test drivers (nix/microvms/tcl): Tcl 8.6 with Expect and tcllib, and the two + # linters. Tcl's setup hook puts expect's and tcllib's lib on TCLLIBPATH. + tclTools = [ + pkgs.tcl + pkgs.expect + pkgs.tclPackages.tcllib + pkgs.nagelfar + pkgs.tclint + ]; in { inherit pythonEnv; - allPackages = buildTools ++ cTools ++ pyTools ++ shTools ++ docTools; + allPackages = buildTools ++ cTools ++ pyTools ++ shTools ++ docTools ++ tclTools; } diff --git a/nix/dev/shell.nix b/nix/dev/shell.nix index 492c67ba..0d0014a3 100644 --- a/nix/dev/shell.nix +++ b/nix/dev/shell.nix @@ -30,6 +30,11 @@ pkgs.mkShell { homa-checkpatch run kernel checkpatch (needs a Linux source tree) homa-lint the analysis-quick tier: ruff, shellcheck, clang-format + The VM test drivers (nix/microvms/tcl): tclsh with Expect and tcllib, nagelfar, tclint. + nix run .#homa-tcl-test -- -file wait.test their tests, or some of them + nix run .#homa-tcl-lint their lint + nix run .#homa-vm-compare -- old new two matrix runs' logs, phase by phase + EOF ''; } diff --git a/nix/microvms/PLAN-pairs.md b/nix/microvms/PLAN-pairs.md new file mode 100644 index 00000000..41813104 --- /dev/null +++ b/nix/microvms/PLAN-pairs.md @@ -0,0 +1,102 @@ +# Plan: Homa between pairs of microVMs + +Goal: show that the flake can build Homa for several kernels, boot each kernel, and pass +real Homa traffic between two VMs, including across different kernel versions, with a +single `nix run` command. It doubles as a demo of the flake. + +## Already in place + +| Need | Target | Notes | +|---|---|---| +| Compile against several kernels | `homa-module-`, `homa-build-matrix` | nixpkgs 6.1/6.6/6.12/6.18/latest plus kernel.org 7.0–7.2 trees (compile only) | +| One VM per kernel | `homa-microvm[-]` | interactive, root autologin | +| Automated single-VM test | `homa-microvm-test[-]`, `-debug` | insmod, loopback echo, netns/veth with GRO+RPS, rmmod; PASS/FAIL from the console | + +Gaps: +- No VM-to-VM traffic: the netns/veth test stays inside one kernel. +- No single command that runs every kernel and prints a table. +- VM tests are generated for 6.1/6.6/6.18, where the module doesn't build (below the supported + range; 6.18.57 has the stable backports), so they fail for reasons that aren't news. + +## Design: the uds-rdma-proxy pair pattern + +Two guests built by the existing `mkVm.nix`, each with a second NIC: + +``` + node0 (server, kernel A) node1 (client, kernel B) + eth0 user NAT (store/init) eth0 user NAT + eth1 10.88.0.1/24 ◄── QEMU -netdev dgram ──► eth1 10.88.0.2/24 + udp 127.0.0.1: udp 127.0.0.1:+1000 +``` + +- `-netdev dgram` (UDP between two localhost ports): a private L2 link between the two qemu + processes. No root, tap or bridge needed, so it works on any host with KVM, and in CI. The + first version used `-netdev socket,listen|connect` (a TCP stream); that stalls some packets + for milliseconds (STATUS.md F2) and is kept only as `homa-pair-exp-socket`. +- The port is derived from the pair name so different pairs can run in parallel. +- Hostnames are `node0`/`node1` with `/etc/hosts` entries, because `cp_node` addresses servers + as `node`. +- The server guest's oneshot: insmod, start `demo_server1` and `cp_node server`, print + `HOMA_PAIR_SERVER_READY`, then idle. +- The client guest's oneshot runs phases. Each prints `HOMA_PAIR__OK`, and the run ends + with `HOMA_PAIR_PASS` or `HOMA_PAIR_FAIL:`: + 1. `ping` over eth1, so a broken link isn't reported as a Homa failure; + 2. echo 100 B–1 MB across VMs (`demo_client1 10.88.0.1`), which covers multi-packet messages + and grants on a real NIC (virtio-net); + 3. a short `cp_node client --protocol homa` run (workload `w4`, a few seconds), checking that + RPCs complete and the log shows no `incomplete` warnings; + 4. the same with `--protocol tcp`, giving a baseline to print beside the Homa numbers; + 5. rmmod and a dmesg gate (`BUG|Oops|WARNING.*homa`). +- Orchestrator (`pair-test.nix`, xtcp2/uds-rdma-proxy style): + 1. start node0 and wait for READY; + 2. start node1 and wait for PASS or FAIL; + 3. power off node0, then apply the same dmesg gate to node0's console (catches oopses on the + receive side); + 4. print one verdict, with both console logs kept in `$HOMA_VM_LOGDIR`. + +## Phases + +| # | Deliverable | New targets | Done when | +|---|---|---|---| +| 1 ✅ | Pair harness, same kernel | `homa-pair-test` (default kernel) | **Done:** echo + cp_node Homa/TCP pass on 7.3 ↔ 7.3, about 35 s cached; a client with a dead link fails at `PING`. The interactive pair (`homa-pair`) moves to phase 3 | +| 2 ✅ | Cross-kernel pairs | `homa-pair-test--` for supported kernels (6.12 ↔ 7.3 both ways) | **Done:** all four pairings pass; each has its own link port, and all four run in parallel in 50 s. `vmSeries` in `kernel/config.nix` limits VM targets to 6.12 and 7.3 | +| 3 ✅ | One command for everything | `homa-vm-matrix` (`matrix.nix`): every single-VM test plus every pair, run in parallel, printing a table (kernel(s), verdict, echo, GRO calls, cp_node rates) | **Done:** 6 tests in 46 s. Its first run found a use-after-free in `homa_route_free` (STATUS.md F8); guests now boot with `init_on_free=1` | +| 4 ✅ | Lossy and slow links | `homa-pair-lifecycle[-]` (`pair-lifecycle.nix`): the pair driven over both guests' consoles as timed lifecycle phases, with `tc netem` on eth1 of both nodes (`delay 1ms`, `loss 0.1%`, `1%`, `delay 1ms loss 1%`, `5%`). Every netem qdisc has `limit 100000`, so no tail drops | **Done:** echo, cp_node Homa and TCP complete under every phase on 6.12 and 7.3; each node's netem drop rate matches the configured loss; Homa recovers by RESEND with no `rpc_timeouts`. Found STATUS.md F10 (delay alone triggers resends) | +| 5 ✅ | Homa qdisc on virtio | `homa-pair-lifecycle` phases 10–12: `homa-qdisc on` (`qdisc.nix`) on eth1 of both nodes, traffic, then `net.homa.link_mbps=1000` so the pacer must defer, traffic, then the default qdisc back | **Done:** reproduced round-4 candidate 5 in VMs. On `main` the qdisc takes the link speed as −1 and echo, Homa and TCP all stall (red). With fork #23 every phase passes and the pacer sends Homa packets (green). STATUS.md F11 | +| 6 (stretch) | Boot 7.0–7.2 | full kernels built from the kernel/src.nix tarballs (`buildLinux`, about 30 min each, cached) | The VM matrix covers every 7.x series, not just compile-only | + +Housekeeping, done with phase 1: +- Add a `vmSupported` list in `kernel/config.nix` (today 6.12 and 7.3), so VM and pair targets are + generated only for kernels where the module builds. +- `homa-build-matrix` still reports all kernels, so the 6.1/6.6/6.18 failures stay visible + there. + +## Files + +- `nix/microvms/constants.nix`: pair addresses, MACs, base port, server and client ready + timeouts. +- `nix/microvms/guest/mkVm.nix`: optional `pair = { role, ip, port }` argument adding eth1 (qemu + `extraArgs`), static IP and hosts; `test` selects which in-guest script runs (smoke, pair + server, pair client). +- `nix/microvms/guest/pair.nix` (new): server and client in-guest scripts, shellcheck-clean, using + the same phase-marker style as `smoke.nix`. +- `nix/microvms/host/pair-test.nix` (new): two-VM orchestrator. +- `nix/microvms/host/matrix.nix` (new): parallel runner and summary table. +- `nix/microvms/default.nix`: wires up the new targets. +- `nix/README.md`, `DESIGN.md`: a demo section. + +## Observations + +- On the original `socket` link, Homa's cp_node P50 RTT was about 5 ms against about 0.3 ms + for TCP. The latency sweep (`homa-pair-exp`) traced it to the link: QEMU's TCP stream + backend delays packets, and Homa recovers through its 5 × 1 ms RESEND timer. Over `dgram` + the two protocols match. See [`../STATUS.md`](../STATUS.md) F2. + +## Caveats + +- The emulated link is slow (a few hundred Mb/s to a few Gb/s), and the guests share host CPUs. cp_node numbers + show that traffic flows, not how Homa performs; the demo output says so. +- Everything stays on the fork's `nix-flake` branch. Nothing goes upstream. The only upstream + benefit is bugs it finds, for example phase 5. +- Resources: two guests of 2 vCPU and 2 GiB each; the full matrix at about 5 pairs needs + about 20 GiB, which this host (24 cores, 125 GiB) handles. diff --git a/nix/microvms/README.md b/nix/microvms/README.md new file mode 100644 index 00000000..3e66925a --- /dev/null +++ b/nix/microvms/README.md @@ -0,0 +1,52 @@ +# nix/microvms: Homa in real kernels, in QEMU + +Each test boots one or two small NixOS guests in QEMU, with the same kernel `homa.ko` was +built against, and loads the module there. The guests share the host's `/nix/store` +read-only, so nothing is copied into a disk image, and they need no root on the host: only +KVM (`/dev/kvm`) and some localhost ports. + +| Target | Guests | What it checks | +|---|---|---| +| `homa-microvm-test[-]` | 1 | the guest runs the smoke test at boot: insmod, echo RPCs over loopback and across two netns with GRO/RPS, rmmod, dmesg | +| `homa-lifecycle[-]` | 1 | the host drives the guest over its consoles, phase by phase with timeouts: boot, smoke, module load/unload cycles, dmesg, poweroff | +| `homa-pair-test[--]` | 2 | Homa between two VMs over a private link: echo, then cp_node with Homa and with TCP | +| `homa-pair-lifecycle[-]` | 2 | the pair, driven over both nodes' consoles: traffic, netem loss and delay, Homa's qdisc | +| `homa-pair-exp[-socket]` | 2 | a latency sweep over sysctl settings ([`../STATUS.md`](../STATUS.md) F2) | +| `homa-vm-matrix` | all | every per-kernel test and every pair at once, and a summary table | +| `homa-vm-compare old new` | — | two matrix runs' logs, test by test: phase ids, names and verdicts, `HOMA_*` lines, `VERDICT` | +| `homa-tcl-test`, `homa-tcl-lint` | — | the drivers' tests (all, or `-file`/`-match`) and lint ([`tcl/README.md`](tcl/README.md)) | +| `homa-microvm[-]` | 1 | an interactive guest (root, no password) | + +`` is each kernel in `vmSeries` ([`../kernel/config.nix`](../kernel/config.nix)); +without it, the default kernel. Each test prints its log directory (`HOMA_VM_LOGDIR` to +choose one) and ends with a `VERDICT: PASS` or `VERDICT: FAIL` line. + +## Layout + +| Path | Runs | What it is | +|---|---|---| +| `default.nix` | | every target above, one function call each | +| `constants.nix` | | the numbers: guest sizes, timeouts, addresses, netem phases, the oops regex | +| `ports.nix` | | every host port, from one table; evaluation fails if two tests would share one | +| [`guest/`](guest/) | in the VM | the guest's NixOS configuration and the scripts it runs | +| [`host/`](host/) | on the host | the commands that boot the guests and judge the result | +| [`tcl/`](tcl/) | on the host | the Tcl/Expect drivers that the lifecycle tests and the matrix run | +| `PLAN-pairs.md` | | the design notes for the two-VM tests | + +## Two ways to drive a guest + +The simple tests (`homa-microvm-test`, `homa-pair-test`) let the guest drive itself: a +systemd unit runs the test script at boot and prints markers such as `HOMA_SMOKE_PASS` on +the serial console, then powers off. The host waits for QEMU to exit and greps the log. + +The lifecycle tests keep the host in charge. The guest offers a root shell on a virtio +console, exposed as a localhost TCP port, and a Tcl/Expect driver sends one command at a +time and times each phase. That shows where a run failed or hung, and lets a test reach +into both nodes of a pair mid-run, to add netem loss or swap the qdisc. + +## Ports + +Tests that run at once must not share a host port. `ports.nix` hands them out from one +table: two console ports per lifecycle guest, four per pair lifecycle, and one link port +per pair (a `dgram` link also uses that port plus `pair.dgramPortOffset`). TCP and UDP +ports are separate. A clash is an evaluation error that names both owners. diff --git a/nix/microvms/constants.nix b/nix/microvms/constants.nix new file mode 100644 index 00000000..b7588e8e --- /dev/null +++ b/nix/microvms/constants.nix @@ -0,0 +1,103 @@ +# nix/microvms/constants.nix +# +# Pure data shared by the guest builder (guest/) and the host-side drivers (host/). Single +# source of truth for identity, resources, ports and timeouts (uds-rdma-proxy pattern). + +{ lib }: + +{ + hostname = "homa-vm"; + + # Guest resources. + vcpu = 2; + mem = 2048; # MiB, normal build + memDebug = 4096; # KASAN needs headroom + + # Whole-run timeouts (seconds): boot + smoke test + poweroff. The kernels are chosen in + # kernel/config.nix (default and matrix), not here. + bootTimeout = 300; + bootTimeoutDebug = 1200; # KASAN guests boot slowly + + # A kernel oops, warning or sanitizer report in dmesg: every dmesg check uses this, in the + # guests (smoke.nix, pair.nix) and on the host (tcl/parse.tcl, whose copy the tcl-tests + # check compares with this one). + oopsRegex = "BUG:|Oops|WARNING:|KASAN|UBSAN|general protection fault|list_add corruption"; + + # demo_client1 sends this many echo RPCs (100 B .. 1 MB); every echo check wants them all: + # smoke.nix, pair.nix and the Tcl drivers. + echoRpcs = 6; + + stateVersion = "24.11"; + + # Lifecycle test (lifecycle.nix, pcp pattern): per-phase timeouts in seconds, and where + # ports.nix starts the guests' console ports. + lifecycle = { + portBase = 24700; + timeouts = { + start = 10; # qemu up and its console ports listening + serial = 30; # first kernel line in serial.log + virtio = 120; # a root shell answering on hvc0 (full boot) + smoke = 180; # homa-smoke (smoke.nix), run over hvc0 + cycles = 90; # insmod / netns add+del / rmmod, several times + command = 30; # any other single command + exit = 60; # poweroff until qemu exits + echo = 60; # demo_client1's six RPCs, also over a lossy link + }; + moduleCycles = 5; + + # Pair lifecycle (pair-lifecycle.nix): four console ports per pair, from here on. + pairConsoleBase = 24800; + + # Homa qdisc phases: net.homa.link_mbps for the paced phase. The qdisc uses it when the + # NIC reports no speed (virtio-net), and at 1 Gb/s a 1 MB echo is several ms of link + # time, so the pacer must defer packets. + qdisc.pacedMbps = 1000; + + # Link impairment phases: tc netem on eth1 of both nodes (so both directions), each + # followed by echo, cp_node Homa and cp_node TCP. loss is a percentage (a string, as tc + # takes it). Every netem qdisc gets this limit (packets): deep enough that the queue + # never overflows, so the only drops are the configured random loss, not tail drops + # that would skew the results. pair-lifecycle.nix checks each node's measured drop + # rate against the configured one. + netem = { + limit = 100000; + phases = [ + { name = "delay1ms"; delay = "1ms"; loss = "0"; } + { name = "loss0.1"; delay = null; loss = "0.1"; } + { name = "loss1"; delay = null; loss = "1"; } + { name = "loss1-delay1ms"; delay = "1ms"; loss = "1"; } + { name = "loss5"; delay = null; loss = "5"; } + ]; + }; + }; + + # Two-VM pair test (pair.nix, pair-test.nix; uds-rdma-proxy pattern). The guests share a + # private L2 link on eth1: a QEMU "dgram" (UDP) netdev between two host ports, so no + # root, tap or bridge is needed ("socket", a TCP stream, is kept for comparison; see + # STATUS.md F2). Hostnames follow cp_node's "node" naming. + pair = { + nodes = { + server = { hostname = "node0"; ip = "10.88.0.1"; mac = "52:54:00:88:00:01"; role = "listen"; }; + client = { hostname = "node1"; ip = "10.88.0.2"; mac = "52:54:00:88:00:02"; role = "connect"; }; + }; + prefixLength = 24; + linkPortBase = 23700; # host 127.0.0.1 ports for pair links start here (ports.nix) + dgramPortOffset = 1000; # "dgram" link: node0 binds the pair's port, node1 this much higher + + # Guest TCP ports for coordination (not Homa traffic). + readyPort = 5998; # server listens once its Homa servers are up + donePort = 5999; # client sends its verdict here when finished + ctlPort = 5997; # sweep mode only: server shell for sysctls/metrics (private link) + cpNodeHomaPort = 4100; # demo_server1 already owns Homa port 4000 + + # cp_node load phase: a few seconds of workload w4 at a modest rate (the socket link + # runs at roughly 1-2 Gb/s; this shows traffic flows, not Homa's performance). + workload = "w4"; + gbps = "0.2"; + loadSeconds = 6; + + serverTimeout = 120; # client waits this long for the server's ready port + clientTimeout = 300; # server waits this long for the client's report + runTimeout = 420; # host: whole pair run + }; +} diff --git a/nix/microvms/default.nix b/nix/microvms/default.nix new file mode 100644 index 00000000..e794bfb2 --- /dev/null +++ b/nix/microvms/default.nix @@ -0,0 +1,234 @@ +# nix/microvms/default.nix +# +# microVM assembly (uds-rdma-proxy pattern, one VM per kernel like xtcp2). Instantiates the +# guest (guest/mkVm.nix) and the host-side drivers (host/) and exposes: +# homa-microvm[-] the QEMU runner package (nix run boots it directly) +# homa-microvm-test[-] boot, run smoke.nix in the guest (insmod, loopback echo +# RPCs, rmmod), PASS/FAIL from the console log +# homa-lifecycle[-] host-driven phases over ttyS0 + a virtio console, each +# timed: boot, smoke, module cycles, dmesg, poweroff, exit +# (lifecycle.nix, pcp pattern) +# homa-microvm-debug default kernel with KASAN/KMEMLEAK +# homa-microvm-test-debug ... the same test under sanitizers +# homa-pair-test[--] two VMs (pair-test.nix): Homa between server , client +# homa-pair-lifecycle[-] the pair driven over both consoles, with timed phases and +# tc netem loss/delay phases (pair-lifecycle.nix) +# homa-pair-exp[-socket] latency sweep over the pair link (STATUS.md F2) +# homa-vm-matrix every per-series smoke test, lifecycle, pair lifecycle and +# pair test, in parallel, with a summary table (matrix.nix) +# homa-vm-compare two matrix runs' logs, phase by phase +# homa-tcl-test, homa-tcl-lint the Tcl drivers' tests and lint (host/tcl.nix), which +# checks.tcl-tests also runs +# is every kernel/config.nix vmMatrix entry (6_12, 7_3); the unsuffixed names use the +# default kernel. Tests share nothing on the host, so they can run in parallel. +# +# Called from nix/scope.nix. Host ports come from ports.nix, which fails evaluation if two +# tests would bind the same one. + +{ pkgs +, lib +, homaLib +, microvm +, nixpkgs +, kernelConfig +, buildModule +, homaUtils +}: + +let + buildSystem = pkgs.stdenv.hostPlatform.system; + constants = import ./constants.nix { inherit lib; }; + tclApps = import ./host/tcl.nix { inherit pkgs lib; inherit (constants) oopsRegex; }; + metrics = import ./guest/metrics.nix { inherit pkgs lib; }; + + defaultKp = kernelConfig.kernelPackages; + vmSeries = lib.attrNames kernelConfig.vmMatrix; + + # forKernels prefix f: prefix = f "default" , and prefix- = f + # for every vmMatrix entry. + forKernels = prefix: f: + { ${prefix} = f "default" defaultKp; } // homaLib.perKernel prefix f kernelConfig.vmMatrix; + labelOf = name: if name == "default" then "" else "-${name}"; + + # Every ordered (server, client) pair of VM kernels: same-kernel pairs and both directions + # of each mixed pair (wire compatibility between Homa builds for different kernels). + pairList = lib.concatMap (s: map (c: { server = s; client = c; }) vmSeries) vmSeries; + pairName = p: "${p.server}-${p.client}"; + + ports = import ./ports.nix { + inherit lib constants vmSeries; + linkNames = + [ + { name = "pair-test"; link = "dgram"; } + { name = "pair-test-exp"; link = "dgram"; } + { name = "pair-test-exp-socket"; link = "socket"; } + ] + ++ map (n: { name = "pair-lifecycle-${n}"; link = "dgram"; }) ([ "default" ] ++ vmSeries) + ++ map (p: { name = "pair-test-${pairName p}"; link = "dgram"; }) pairList; + }; + + mkGuest = + { kernelPackages + , withSanitizers ? false + , autoTest ? false + , pairNode ? null + , pairPort ? null + , pairExperiments ? null + , pairLink ? "dgram" + , consolePorts ? null + }: import ./guest/mkVm.nix { + inherit pkgs lib microvm nixpkgs buildSystem constants buildModule homaUtils + kernelPackages withSanitizers autoTest pairNode pairPort pairExperiments pairLink + consolePorts; + }; + + mkTest = { kernelPackages, label, withSanitizers ? false }: import ./host/smoke-test.nix { + inherit pkgs constants label withSanitizers; + runner = mkGuest { inherit kernelPackages withSanitizers; autoTest = true; }; + }; + + # Lifecycle test of the kernel called name ("default" or a series). + mkLifecycle = name: kernelPackages: + let consolePorts = ports.console name; in + import ./host/lifecycle.nix { + inherit constants tclApps consolePorts; + label = labelOf name; + runner = mkGuest { inherit kernelPackages consolePorts; }; + }; + + # Pair lifecycle (PLAN-pairs.md phase 4): both nodes on one kernel, driven over their + # consoles, with netem phases. + mkPairLifecycle = name: kernelPackages: + let + consoles = ports.pairConsoles name; + node = pairNode: mkGuest { + inherit kernelPackages pairNode; + pairPort = ports.link "pair-lifecycle-${name}"; + consolePorts = consoles.${if pairNode == "server" then "node0" else "node1"}; + }; + in + import ./host/pair-lifecycle.nix { + inherit lib constants tclApps metrics; + label = labelOf name; + ports = consoles; + serverRunner = node "server"; + clientRunner = node "client"; + }; + + # Two-VM pair test: node0 (server) on serverKp, node1 (client) on clientKp, linked on the + # port of "pair-test