chore(bench): refresh the report, confirming x86_64 RSA-2048 signing level, and add two OVH VPS hosts

admin/bench-report

@@ -8,7 +8,7 @@ Usage:
   admin/bench-report summary DIR
 
 `collect` copies one commit to each host in turn, holds the host with
-`quest gate --host`, builds rind-bench there in the commit's Nix dev shell with
+`quest gate --host` and any lease it has, builds rind-bench there in the commit's Nix dev shell with
 its pinned toolchain, and times every operation in OPERATIONS for every
 library that offers it, then measures each library's build cost with
 admin/build-cost. It runs here for the host this is, and over ssh for the
@@ -52,12 +52,29 @@ import time
 import tomllib
 
 # The report's hosts, in the order they run. `jobs` caps the build: nixos2 is
-# a shared 15 GB machine that more jobs run out of memory.
+# a shared 15 GB machine that more jobs run out of memory. `ssh` is the
+# destination where it isn't the name, and `note` describes the machine in the
+# report. Other work on a host with a `lease` holds that lock file with
+# flock(1) for its whole run, so collect holds it too, or skips the host as
+# busy, and leaves no copy, build, or download cache behind.
+VPS_NOTE = "OVH VPS in {}, 4 shared vCPUs: other tenants add noise that a dedicated host's times lack."
 HOSTS = {
     "nixos": {"jobs": 16},
     "desktop": {"jobs": 12},
     "nixos2": {"jobs": 3},
     "kixtop": {"jobs": 4},
+    "ovh-iad": {
+        "jobs": 4,
+        "ssh": "root@vps-bcb7229a.vps.ovh.us",
+        "lease": "/run/lock/moq-perf.lock",
+        "note": VPS_NOTE.format("Virginia"),
+    },
+    "ovh-sea": {
+        "jobs": 4,
+        "ssh": "root@vps-39a0ff2a.vps.ovh.us",
+        "lease": "/run/lock/moq-perf.lock",
+        "note": VPS_NOTE.format("Oregon"),
+    },
 }
 
 # A host is busy when other work keeps more than this share of its logical
@@ -166,6 +183,15 @@ LIBRARIES = {
     "openssl": "openssl",
 }
 
+# The CPU features by architecture, as rind-cpuid names them, without which a
+# library panics on first use rather than run a slower path.
+REQUIRED = {
+    "graviola": {
+        "x86_64": ["aes", "pclmulqdq", "bmi1", "adx", "avx", "avx2"],
+        "aarch64": ["neon", "aes", "pmull", "sha2"],
+    }
+}
+
 # The crates whose versions a result records, by library.
 CRATES = {
     "rind": ["rind", "rind-rustls"],
@@ -211,9 +237,14 @@ def fail(message):
     sys.exit(f"bench-report: {message}")
 
 
-def operation_regex():
-    """A criterion filter matching exactly the report's operations."""
-    ops = "|".join(re.escape(op).replace(re.escape("{lib}"), "[^/]+") for op in OPERATIONS)
+def operation_regex(refused=()):
+    """A criterion filter matching exactly the report's operations, without
+    the libraries in `refused`."""
+    lib = "[^/]+"
+    if refused:
+        # The regex crate has no lookahead, so name the rest.
+        lib = "(" + "|".join(re.escape(row) for row, name in LIBRARIES.items() if name not in refused) + ")"
+    ops = "|".join(re.escape(op).replace(re.escape("{lib}"), lib) for op in OPERATIONS)
     return f"^({ops})$"
 
 
@@ -257,8 +288,9 @@ def cpu_times():
             fail("host_statistics failed")
         user, system, idle, nice = ticks
         return user + system + nice, user + system + idle + nice
+    # The first eight fields, since user and nice already count guest time.
     with open("/proc/stat") as f:
-        fields = [int(x) for x in f.readline().split()[1:]]
+        fields = [int(x) for x in f.readline().split()[1:9]]
     idle = fields[3] + fields[4]
     return sum(fields) - idle, sum(fields)
 
@@ -372,6 +404,13 @@ def run(args):
     subprocess.run(bench + ["--no-run"], env=env, check=True)
     # The CPU features rind reads, which pick its paths on this host.
     result["rind_cpu"] = json.loads(output(cargo + ["run", "-q", "--release", "-p", "rind-cpuid"], env))
+    # Each library this CPU lacks a required feature for, with the features.
+    cpuid = result["rind_cpu"]["cpuid"]
+    result["refused"] = {
+        lib: missing
+        for lib, features in REQUIRED.items()
+        if (missing := [f for f in features.get(host_arch(result), []) if not cpuid[f]])
+    }
 
     try:
         idle_before_run = check_idle("before the benchmarks")
@@ -381,7 +420,7 @@ def run(args):
 
     criterion_dir = os.path.join(env["CARGO_TARGET_DIR"], "criterion")
     shutil.rmtree(criterion_dir, ignore_errors=True)
-    criterion_args = CRITERION_ARGS + [operation_regex()]
+    criterion_args = CRITERION_ARGS + [operation_regex(result["refused"])]
     wall0 = time.monotonic()
     ticks0 = cpu_times()
     ours0 = resource.getrusage(resource.RUSAGE_CHILDREN)
@@ -431,17 +470,19 @@ def run(args):
         criterion_args=criterion_args,
         load=load,
         results=results,
-        build_cost=build_cost(env, args.jobs),
+        build_cost=build_cost(env, args.jobs, result["refused"]),
     )
 
 
-def build_cost(env, jobs):
+def build_cost(env, jobs, refused):
     """admin/build-cost's measurements on this host, after the benchmarks so
-    that neither skews the other's times."""
+    that neither skews the other's times, without the builds of libraries in
+    `refused`, whose binaries it runs."""
     env = dict(env, CARGO_BUILD_JOBS=str(jobs))
     started = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")
+    builds = [b for b in BUILDS if b not in refused] if refused else []
     proc = subprocess.run(
-        [sys.executable, os.path.join(ROOT, "admin", "build-cost")], env=env, capture_output=True, text=True
+        [sys.executable, os.path.join(ROOT, "admin", "build-cost"), *builds], env=env, capture_output=True, text=True
     )
     if proc.returncode != 0:
         lines = proc.stderr.strip().splitlines() or ["no output"]
@@ -457,11 +498,46 @@ def is_local(host):
     return host == socket.gethostname().split(".")[0]
 
 
+def ssh(host):
+    # ServerAlive ends a session whose link died, which would otherwise hang
+    # collect.
+    options = ["BatchMode=yes", "ConnectTimeout=10", "ServerAliveInterval=15", "ServerAliveCountMax=4"]
+    return ["ssh", *(arg for o in options for arg in ("-o", o)), HOSTS[host].get("ssh", host)]
+
+
 def on_host(host, script, **kwargs):
     """Runs a shell script on `host`, here or over ssh."""
     if is_local(host):
         return subprocess.run(["sh", "-c", script], **kwargs)
-    return subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", host, script], **kwargs)
+    return subprocess.run(ssh(host) + [script], **kwargs)
+
+
+# The host script's exit status when another run holds the host's lease, and
+# when it could not copy the commit.
+LEASE_HELD = 73
+COPY_FAILED = 74
+
+
+def host_script(host, work, cmd):
+    """The shell script that copies the commit from stdin into a directory of
+    its own under `work`, runs `cmd` there with its output on stderr, prints
+    the result, and removes the copy. On a leased host it holds the lease
+    throughout, and removes all of `work` before letting go."""
+    q = shlex.quote
+    w = q(work)
+    # `cmd` runs in a subshell, since `work` may be relative to the start.
+    script = (
+        f'mkdir -p {w} && src=$(mktemp -d {w}/src.XXXXXX) && tar -x -C "$src" || exit {COPY_FAILED}\n'
+        f'(cd "$src" && {cmd}) </dev/null >&2\n'
+        'cat "$src/result.json"; code=$?; rm -rf "$src"; exit $code\n'
+    )
+    if "lease" not in HOSTS[host]:
+        return script
+    # The script's processes inherit the descriptor holding the lock. Python
+    # closes it in its own children, but subprocess.run kills its child when
+    # Python fails, so no child outlives the lease.
+    lock = f"exec 9>{q(HOSTS[host]['lease'])} && {{ flock -n -E {LEASE_HELD} 9 || exit; }}\n"
+    return lock + f"trap 'rm -rf {w}' EXIT\n" + script
 
 
 def collect_host(host, commit, out_dir):
@@ -479,40 +555,36 @@ def collect_host(host, commit, out_dir):
     if probe.returncode != 0:
         return record("unreachable", probe.stderr.strip())
 
-    # A fresh copy of the commit for this collection alone, so another one on
-    # the same host cannot replace it mid-run; the build directory is shared.
+    # Each collection extracts its own copy of the commit, so another on the
+    # same host cannot replace it mid-run; the build directory is shared.
     archive = subprocess.run(["git", "-C", ROOT, "archive", "--format=tar", commit], capture_output=True, check=True)
-    copy = on_host(
-        host,
-        f'mkdir -p {q(work)} && src=$(mktemp -d {q(work)}/src.XXXXXX) && tar -x -C "$src" && echo "$src"',
-        input=archive.stdout,
-        capture_output=True,
+    cmd = (
+        f'nix develop "path:$PWD" --command python3 admin/bench-report run '
+        f"--host {q(host)} --commit {q(commit)} --jobs {HOSTS[host]['jobs']} "
+        f"--target-dir ../target --out result.json"
     )
-    if copy.returncode != 0:
-        return record("failed", "could not copy the commit")
-    src = q(copy.stdout.decode().strip())
-
     # Holding the host keeps every other gated command, such as another
     # collection's run, off it until the benchmarks and build costs finish.
     # The gate is this checkout's quest, so a commit pinning an older one
-    # still runs, in its own dev shell. A macOS ssh session runs no login
-    # shell, which is what puts Nix on PATH.
+    # still runs, in its own dev shell.
     with open(os.path.join(ROOT, "flake.lock")) as f:
         quest = json.load(f)["nodes"]["quest"]["locked"]
-    on_host(
-        host,
-        f'export PATH="$PATH:/nix/var/nix/profiles/default/bin" && cd {src} && '
-        f"nix shell github:{quest['owner']}/{quest['repo']}/{quest['rev']} --command quest gate --host "
-        f'nix develop "path:$PWD" --command python3 admin/bench-report run '
-        f"--host {q(host)} --commit {q(commit)} --jobs {HOSTS[host]['jobs']} "
-        f"--target-dir ../target --out result.json",
-    )
-    fetched = on_host(host, f"cat {src}/result.json; code=$?; rm -rf {src}; exit $code", capture_output=True)
-    if fetched.returncode != 0:
+    cmd = f"nix shell github:{quest['owner']}/{quest['repo']}/{quest['rev']} --command quest gate --host {cmd}"
+    if "lease" in HOSTS[host]:
+        # Crate downloads too, so that removing the work directory leaves nothing.
+        cmd = f'CARGO_HOME="$PWD/../cargo" {cmd}'
+    # A macOS ssh session runs no login shell, which is what puts Nix on PATH.
+    cmd = f'export PATH="$PATH:/nix/var/nix/profiles/default/bin" && {cmd}'
+    ran = on_host(host, host_script(host, work, cmd), input=archive.stdout, stdout=subprocess.PIPE)
+    if ran.returncode == LEASE_HELD:
+        return record("busy", "another run holds its lease")
+    if ran.returncode == COPY_FAILED:
+        return record("failed", "could not copy the commit")
+    if ran.returncode != 0:
         return record("failed", "the run wrote no result; see its output above")
     with open(out, "wb") as f:
-        f.write(fetched.stdout)
-    status = json.loads(fetched.stdout)["status"]
+        f.write(ran.stdout)
+    status = json.loads(ran.stdout)["status"]
     print(f"bench-report: {host}: {status}", file=sys.stderr)
 
 
@@ -1382,7 +1454,8 @@ def report(hosts, tiers, verified, gaps):
     files = {}
     shown = [name for name in HOSTS if name in hosts]
     missing = [f"`{name}`" for name in HOSTS if name not in hosts]
-    count = ["no", "one", "two", "three", "four"][len(shown)] if len(shown) <= 4 else str(len(shown))
+    words = ["no", "one", "two", "three", "four", "five", "six"]
+    count = words[len(shown)] if len(shown) < len(words) else str(len(shown))
     md = [
         "# Benchmarks",
         "",
@@ -1434,7 +1507,14 @@ def report(hosts, tiers, verified, gaps):
             "",
             f"Measured {measured(h)}, on {h['os']} with {h['toolchain']['rustc']}.",
             "",
+            *([HOSTS[name]["note"], ""] if "note" in HOSTS[name] else []),
             *([f"Not timed on `{name}`: {', '.join(untimed)}.", ""] if untimed else []),
+            *(
+                f"{COLUMNS[lib]} isn't timed on `{name}`: it requires {and_list([f.upper() for f in missing])}, "
+                "which this CPU lacks."
+                for lib, missing in h.get("refused", {}).items()
+            ),
+            *([""] if h.get("refused") else []),
             *picture(name, f"Times on {name}"),
             "",
             "<details>",

admin/bench-report-test

@@ -1,15 +1,19 @@
 #!/usr/bin/env python3
-"""Tests admin/bench-report's rendering on synthetic results.
+"""Tests admin/bench-report's rendering on synthetic results, and its lease.
 
 Usage: admin/bench-report-test
 """
 
+import fcntl
 import importlib.machinery
 import importlib.util
+import io
 import json
 import os
 import re
+import subprocess
 import sys
+import tarfile
 import tempfile
 import unittest
 
@@ -186,7 +190,7 @@ class Render(unittest.TestCase):
         md = files[br.REPORT]
         self.assertIn("on three machines", md)
         self.assertIn("Not timed on `kixtop`: full handshake, X25519MLKEM768, ", md)
-        self.assertIn("Not measured yet: `desktop`.", md)
+        self.assertIn("Not measured yet: `desktop`, `ovh-iad`, `ovh-sea`.", md)
         self.assertNotIn("`desktop` |", md)
         self.assertIn("No nightly results yet.", md)
         self.assertEqual(
@@ -196,6 +200,17 @@ class Render(unittest.TestCase):
         self.assertIn("| Operation | rind | aws-lc-rs | ring | OpenSSL |", md)
         self.assertIn("| X25519 | 30.00 µs ✓ | - | 33.00 µs ✓ | 300.00 µs |", md)
 
+    def test_leaves_out_a_library_the_cpu_lacks_features_for(self):
+        regex = re.compile(br.operation_regex({"graviola": ["adx"]}))
+        for row in ["x25519-ecdh/rind", "p256-ecdh/p256-rustcrypto", "aes128-gcm/openssl/32B"]:
+            self.assertTrue(regex.match(row), row)
+        for row in ["x25519-ecdh/graviola", "aes128-gcm/graviola/32B", "x25519-ecdh/rind/extra"]:
+            self.assertFalse(regex.match(row), row)
+        everyone = hosts()
+        everyone["nixos2"]["refused"] = {"graviola": ["bmi1", "adx"]}
+        md = br.report(everyone, None, VERIFIED, [])[br.REPORT]
+        self.assertIn("graviola isn't timed on `nixos2`: it requires BMI1 and ADX, which this CPU lacks.", md)
+
     def test_headline_is_the_mean_over_hosts_and_says_what_is_missing(self):
         everyone = hosts()
         everyone["nixos"]["build_cost"] = {
@@ -332,5 +347,46 @@ class Gaps(unittest.TestCase):
             br.load_gaps(br.load_hosts(br.DATA_DIR), json.load(f))
 
 
+class HostScript(unittest.TestCase):
+    def run_script(self, cmd):
+        """The host script's exit status and output on the host "vps" here."""
+        archive = io.BytesIO()
+        with tarfile.open(fileobj=archive, mode="w") as tar:
+            tar.addfile(tarfile.TarInfo("file"), io.BytesIO())
+        ran = subprocess.run(
+            ["sh", "-c", br.host_script("vps", self.work, cmd)], input=archive.getvalue(), capture_output=True
+        )
+        return ran.returncode, ran.stdout
+
+    def setUp(self):
+        d = tempfile.TemporaryDirectory()
+        self.addCleanup(d.cleanup)
+        self.work = os.path.join(d.name, "work")
+        self.lock = os.path.join(d.name, "lock")
+        self.addCleanup(setattr, br, "HOSTS", br.HOSTS)
+
+    def test_removes_its_copy_and_keeps_the_rest(self):
+        br.HOSTS = {"vps": {"jobs": 1}}
+        os.makedirs(os.path.join(self.work, "target"))
+        self.assertEqual(self.run_script("echo $(ls) > result.json"), (0, b"file\n"))
+        self.assertEqual(os.listdir(self.work), ["target"])
+
+    def test_holds_the_lease_through_the_run_and_its_cleanup(self):
+        br.HOSTS = {"vps": {"jobs": 1, "lease": self.lock}}
+        # Another run's lease: its work stays.
+        os.makedirs(os.path.join(self.work, "target"))
+        with open(self.lock, "w") as held:
+            fcntl.flock(held, fcntl.LOCK_EX)
+            self.assertEqual(self.run_script("true"), (br.LEASE_HELD, b""))
+        self.assertTrue(os.path.isdir(os.path.join(self.work, "target")))
+        # Ours: the run sees the lease held and its copy, and nothing is left
+        # once it lets go.
+        status, out = self.run_script(f"flock -n {self.lock} true; echo $? $(ls) > result.json")
+        self.assertEqual((status, out), (0, b"1 file\n"))
+        self.assertFalse(os.path.exists(self.work))
+        with open(self.lock) as other:
+            fcntl.flock(other, fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+
 if __name__ == "__main__":
     sys.exit(unittest.main())