From d52deca9162150e702f88e491c29b893a08f67ee Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 09:11:56 -0800 Subject: [PATCH 1/7] ci(e2e): cut fixed setup and teardown costs of the k3s conformance rows Cache the tmachine setup layers for ubuntu-k3s too, and leave install layers out of the disk cache: they hold the build under test and, on k3s, the gateway's credentials. Install layers now live under disks/install so the cache can take only the setup layers, keyed per environment. Kill QEMU at session teardown instead of powering the guest off, since the session overlay is thrown away. Layer builds still power off cleanly. Pull the HA PostgreSQL fixture image in the k3s setup layer, and run the prebuilt test program with a 180-second per-test limit instead of the 900-second default. Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 26 ++++++++--------- TESTING.md | 6 ++-- tests/runner/src/execute.rs | 16 +++++++++-- tests/tmachine/ansible/playbooks/k3s.yaml | 17 ++++++++++++ tests/tmachine/config.nix | 9 ++++-- tests/tmachine/src/qemu/install.rs | 3 +- tests/tmachine/src/qemu/layer.rs | 10 ++++++- tests/tmachine/src/qemu/session.rs | 14 ++++++---- tests/tmachine/src/qemu/setup.rs | 3 +- tests/tmachine/src/qemu/vm.rs | 34 ++--------------------- 10 files changed, 77 insertions(+), 61 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index ec6cc2e54b..8c72512e5a 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -147,24 +147,24 @@ jobs: with: cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} - # integration-runner.yml skips the disk cache for ubuntu-k3s and doesn't say - # why, so the k3s environment skips it here too until that is understood. - name: Cache tmachine disks - if: matrix.environment != 'ubuntu-k3s' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: ~/.cache/tmachine - # tmachine keys its disk layers on a hash of the Ansible tree, the + # Only the setup layers, which depend on the environment alone. The + # install layers under disks/install hold this commit's build (and + # on k3s, the gateway's credentials), so no later run could use them. + path: ~/.cache/tmachine/disks/*.qcow2 + # tmachine keys its setup layers on a hash of the Ansible tree, the # environments' playbook lists and setup inputs (config.nix, and - # files such as the Keycloak realm file) and the cache versions in - # setup.rs and install.rs. actions/cache doesn't save after an - # exact key hit, so a key that misses one of those would rebuild the - # changed layers on every run. Environments share no layers, and + # files such as the Keycloak realm file and the PostgreSQL fixture) + # and the cache version in setup.rs. actions/cache doesn't save after + # an exact key hit, so a key that misses one of those would rebuild + # the changed layers on every run. Environments share no layers, and # only the first job to save a key gets its disks in, so each - # environment and installer needs its own key. No restore-keys - # fallback: restoring an older entry would carry its stale disks - # into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/install.rs', 'scripts/keycloak-realm.json') }} + # environment needs its own key; its installers share it. No + # restore-keys fallback: restoring an older entry would carry its + # stale disks into the new one, which then only grows. + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'scripts/keycloak-realm.json', 'e2e/kubernetes/postgres-fixture.yaml') }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/TESTING.md b/TESTING.md index 58bb638831..9a36097f4d 100644 --- a/TESTING.md +++ b/TESTING.md @@ -254,8 +254,10 @@ combined with `--package`. With it, `--filter` (repeatable) takes parts of test names, as the program does, such as `--filter policy_advisor::`, rather than a nextest filterset. The runner runs the program's `ready` and `run` commands in the guest and still writes `junit.xml` and `nextest.log` -(the program's output) to the results directory. Branch E2E builds -`openshell-conformance` once and passes it to every conformance row this way. +(the program's output) to the results directory. It stops a test that runs +longer than two minutes under nextest, or three minutes under a program. +Branch E2E builds `openshell-conformance` once and passes it to every +conformance row this way. `openshell-test-runner build --family --out ` builds a family's test bundle for the guest without booting a VM, and `run --bundle ` runs diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 2f477134f9..4b9486fb08 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -32,9 +32,12 @@ const NEXTEST_LOG: &str = "nextest.log"; // disk can take minutes to report healthy. const READY_TIMEOUT_SECS: u64 = 300; /// How long the nextest run may take in total. Individual tests are bounded -/// by the nextest profile's slow-timeout; this catches a run that stops -/// making progress some other way. +/// by the nextest profile's slow-timeout or [`PROGRAM_TEST_TIMEOUT_SECS`]; +/// this catches a run that stops making progress some other way. const RUN_TIMEOUT: Duration = Duration::from_mins(45); +/// How long a test program gives each test before stopping it. A test that +/// stalls fails in minutes instead of at the program's 15-minute default. +const PROGRAM_TEST_TIMEOUT_SECS: &str = "180"; /// Slack beyond the readiness check's own timeout for SSH and process start. const READY_SLACK: Duration = Duration::from_mins(1); /// nextest's exit code when a test failed, as opposed to nextest itself @@ -342,7 +345,14 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { // The program runs its tests one at a time, and writes the report // where `collect_results` looks for it. Kind::Program => { - tests.extend([program.as_str(), "run", "--junit", &junit]); + tests.extend([ + program.as_str(), + "run", + "--timeout", + PROGRAM_TEST_TIMEOUT_SECS, + "--junit", + &junit, + ]); // `run` takes name filters as arguments, after `--` so that one // starting with a dash isn't read as an option. if !run.filters.is_empty() { diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index d27c6c487b..91c346841d 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -57,6 +57,23 @@ retries: 12 delay: 5 + # The k3s-ha installer starts PostgreSQL from this fixture. Its image is + # pinned by digest and pulled IfNotPresent, so pulling it here keeps it in + # the cached setup layer instead of pulling it on every install. + - name: Pull the PostgreSQL fixture image + ansible.builtin.command: + argv: [/usr/local/bin/k3s, crictl, pull, "{{ postgres_image }}"] + vars: + postgres_image: >- + {{ (lookup('ansible.builtin.file', openshell_postgres_fixture) + | ansible.builtin.from_yaml_all + | selectattr('kind', 'equalto', 'Deployment') | first) + .spec.template.spec.containers[0].image }} + register: postgres_pull + until: postgres_pull.rc == 0 + retries: 3 + delay: 5 + - name: Download Helm installer ansible.builtin.get_url: url: https://raw.githubusercontent.com/helm/helm/06468084e85c244c712834933d25ea232a4c2093/scripts/get-helm-4 diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index 7255dc1d85..4b76e3d0b4 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -25,6 +25,7 @@ let qemu = qemuPkgs.qemu.override { hostCpuOnly = true; }; # Every installer loads the sandbox workload image tests run python3 in. e2ePythonImage = "../../artifacts/images/openshell-e2e-python-dev.tar"; + postgresFixture = "../../e2e/kubernetes/postgres-fixture.yaml"; k3sInputs = { agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../../artifacts/binaries/${muslTarget}/openshell"; @@ -66,6 +67,8 @@ let "ansible/playbooks/test-tools.yaml" "ansible/playbooks/k3s.yaml" ]; + # k3s.yaml pulls the image of the k3s-ha installer's PostgreSQL fixture. + inputs.openshell_postgres_fixture = postgresFixture; }; fedora-podman-rootful = fedoraPodman "rootful"; fedora-podman-rootless = fedoraPodman "rootless"; @@ -91,7 +94,8 @@ let inherit (parent) use_galaxy; playbooks = parent.playbooks ++ setup.playbooks; inputs = - (if parent ? driver then { environment_driver = parent.driver; } else { }) + (parent.inputs or { }) + // (if parent ? driver then { environment_driver = parent.driver; } else { }) // (setup.inputs or { }); }; }; @@ -153,6 +157,7 @@ let inherit (base) machine; setup = { inherit (base) use_galaxy playbooks; + inputs = base.inputs or { }; }; }) bases ++ featureEnvironments @@ -170,7 +175,7 @@ let use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s-ha.yaml" ]; inputs = k3sInputs // { - openshell_postgres_fixture = "../../e2e/kubernetes/postgres-fixture.yaml"; + openshell_postgres_fixture = postgresFixture; }; } { diff --git a/tests/tmachine/src/qemu/install.rs b/tests/tmachine/src/qemu/install.rs index e1e5910520..25b3d6abf0 100644 --- a/tests/tmachine/src/qemu/install.rs +++ b/tests/tmachine/src/qemu/install.rs @@ -9,7 +9,7 @@ use blake3::Hasher; use crate::config::{Environment, Installer, Machine}; use super::ansible_hash::hash_sources; -use super::layer::{cached_layer, hash_file, hash_files, hash_inputs}; +use super::layer::{INSTALL_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; use super::setup::setup; const INSTALL_CACHE_VERSION: &[u8] = b"tmachine-install-blake3-v1"; @@ -26,6 +26,7 @@ pub async fn install( let hash = install_hash(&setup_disk, installer)?; cached_layer( + INSTALL_DISKS, &setup_disk, &hash, installer.use_galaxy, diff --git a/tests/tmachine/src/qemu/layer.rs b/tests/tmachine/src/qemu/layer.rs index e6ec03650d..7cc9b09aa1 100644 --- a/tests/tmachine/src/qemu/layer.rs +++ b/tests/tmachine/src/qemu/layer.rs @@ -12,7 +12,15 @@ use directories::ProjectDirs; use super::img::QemuImage; use super::vm::{LAYER_POWER_OFF_TIMEOUT, QemuVm}; +/// Where setup layers are kept, under tmachine's cache directory. They depend +/// only on the environment, so CI caches the disks here across commits. +pub(super) const SETUP_DISKS: &str = "disks"; +/// Where install layers are kept. They hold the build under test, so CI leaves +/// them out of its cache. +pub(super) const INSTALL_DISKS: &str = "disks/install"; + pub(super) async fn cached_layer( + disks: &str, base_image: &Path, hash: &str, use_galaxy: bool, @@ -20,7 +28,7 @@ pub(super) async fn cached_layer( inputs: &BTreeMap, ) -> Result { let project_dirs = ProjectDirs::from("com", "nvidia", "tmachine").unwrap(); - let disks_dir = project_dirs.cache_dir().join("disks"); + let disks_dir = project_dirs.cache_dir().join(disks); std::fs::create_dir_all(&disks_dir).unwrap(); let disk = disks_dir.join(format!("{hash}.qcow2")); diff --git a/tests/tmachine/src/qemu/session.rs b/tests/tmachine/src/qemu/session.rs index c3682d5958..576ece9dc6 100644 --- a/tests/tmachine/src/qemu/session.rs +++ b/tests/tmachine/src/qemu/session.rs @@ -60,7 +60,8 @@ pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(600); /// A machine booted from a throwaway overlay of the installed disk. /// /// The VM stops when the session is dropped, so a caller that fails or -/// panics cannot leak it. Call [`Session::down`] for a clean shutdown. +/// panics cannot leak it. Call [`Session::down`] to stop it and wait until +/// QEMU has exited. pub struct Session { // Declared before `_dir` so the VM stops before its disk is removed. vm: QemuVm, @@ -173,13 +174,14 @@ impl Session { .context("open SSH shell in tmachine VM") } - /// Power the machine off and wait for QEMU to exit. + /// Stop the machine and wait for QEMU to exit. /// - /// This never panics and never fails: teardown has no useful recovery, - /// so a guest that does not power off cleanly is killed and the problem - /// is logged to stderr. + /// QEMU is killed rather than the guest powered off: the overlay is + /// thrown away, so a clean shutdown would only cost time. Copy out + /// anything the guest holds first. This never panics and never fails. pub async fn down(self) { - self.vm.power_off_or_kill().await; + let Self { mut vm, .. } = self; + vm.kill().await; } } diff --git a/tests/tmachine/src/qemu/setup.rs b/tests/tmachine/src/qemu/setup.rs index 185d6c290b..83c846fee9 100644 --- a/tests/tmachine/src/qemu/setup.rs +++ b/tests/tmachine/src/qemu/setup.rs @@ -9,7 +9,7 @@ use blake3::Hasher; use crate::config::{Environment, Machine}; use super::ansible_hash::hash_sources; -use super::layer::{cached_layer, hash_file, hash_files, hash_inputs}; +use super::layer::{SETUP_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; const SETUP_CACHE_VERSION: &[u8] = b"tmachine-disk-blake3-v1"; @@ -20,6 +20,7 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result result, - Err(_) => Err(anyhow::anyhow!("QMP did not answer within {QMP_TIMEOUT:?}")), - }; - match shutdown { - Ok(()) => { - if let Err(error) = self.wait(POWER_OFF_TIMEOUT).await { - eprintln!("tmachine: {error:#}"); - } - } - Err(error) => { - eprintln!("tmachine: could not power off the VM cleanly, killing it: {error:#}"); - self.kill().await; - } - } - } } fn command( From 9c481adcbdabb52aa74cdb7381abe3b23c165f28 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 14:59:22 -0800 Subject: [PATCH 2/7] ci(e2e): drop the test timeout and PostgreSQL pre-pull from the k3s fixed-cost change Leave per-test timeouts to the runner work that owns them, and stop pulling the PostgreSQL fixture image in the k3s setup layer, which saved no measurable time and tied the tmachine cache key to e2e/. Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 10 +++++----- TESTING.md | 6 ++---- tests/runner/src/execute.rs | 16 +++------------- tests/tmachine/ansible/playbooks/k3s.yaml | 17 ----------------- tests/tmachine/config.nix | 9 ++------- 5 files changed, 12 insertions(+), 46 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 8c72512e5a..f4dee9ab32 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -156,15 +156,15 @@ jobs: path: ~/.cache/tmachine/disks/*.qcow2 # tmachine keys its setup layers on a hash of the Ansible tree, the # environments' playbook lists and setup inputs (config.nix, and - # files such as the Keycloak realm file and the PostgreSQL fixture) - # and the cache version in setup.rs. actions/cache doesn't save after - # an exact key hit, so a key that misses one of those would rebuild - # the changed layers on every run. Environments share no layers, and + # files such as the Keycloak realm file) and the cache version in + # setup.rs. actions/cache doesn't save after an exact key hit, so a + # key that misses one of those would rebuild the changed layers on + # every run. Environments share no layers, and # only the first job to save a key gets its disks in, so each # environment needs its own key; its installers share it. No # restore-keys fallback: restoring an older entry would carry its # stale disks into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'scripts/keycloak-realm.json', 'e2e/kubernetes/postgres-fixture.yaml') }} + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'scripts/keycloak-realm.json') }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/TESTING.md b/TESTING.md index 9a36097f4d..58bb638831 100644 --- a/TESTING.md +++ b/TESTING.md @@ -254,10 +254,8 @@ combined with `--package`. With it, `--filter` (repeatable) takes parts of test names, as the program does, such as `--filter policy_advisor::`, rather than a nextest filterset. The runner runs the program's `ready` and `run` commands in the guest and still writes `junit.xml` and `nextest.log` -(the program's output) to the results directory. It stops a test that runs -longer than two minutes under nextest, or three minutes under a program. -Branch E2E builds `openshell-conformance` once and passes it to every -conformance row this way. +(the program's output) to the results directory. Branch E2E builds +`openshell-conformance` once and passes it to every conformance row this way. `openshell-test-runner build --family --out ` builds a family's test bundle for the guest without booting a VM, and `run --bundle ` runs diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 4b9486fb08..2f477134f9 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -32,12 +32,9 @@ const NEXTEST_LOG: &str = "nextest.log"; // disk can take minutes to report healthy. const READY_TIMEOUT_SECS: u64 = 300; /// How long the nextest run may take in total. Individual tests are bounded -/// by the nextest profile's slow-timeout or [`PROGRAM_TEST_TIMEOUT_SECS`]; -/// this catches a run that stops making progress some other way. +/// by the nextest profile's slow-timeout; this catches a run that stops +/// making progress some other way. const RUN_TIMEOUT: Duration = Duration::from_mins(45); -/// How long a test program gives each test before stopping it. A test that -/// stalls fails in minutes instead of at the program's 15-minute default. -const PROGRAM_TEST_TIMEOUT_SECS: &str = "180"; /// Slack beyond the readiness check's own timeout for SSH and process start. const READY_SLACK: Duration = Duration::from_mins(1); /// nextest's exit code when a test failed, as opposed to nextest itself @@ -345,14 +342,7 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { // The program runs its tests one at a time, and writes the report // where `collect_results` looks for it. Kind::Program => { - tests.extend([ - program.as_str(), - "run", - "--timeout", - PROGRAM_TEST_TIMEOUT_SECS, - "--junit", - &junit, - ]); + tests.extend([program.as_str(), "run", "--junit", &junit]); // `run` takes name filters as arguments, after `--` so that one // starting with a dash isn't read as an option. if !run.filters.is_empty() { diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index 91c346841d..d27c6c487b 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -57,23 +57,6 @@ retries: 12 delay: 5 - # The k3s-ha installer starts PostgreSQL from this fixture. Its image is - # pinned by digest and pulled IfNotPresent, so pulling it here keeps it in - # the cached setup layer instead of pulling it on every install. - - name: Pull the PostgreSQL fixture image - ansible.builtin.command: - argv: [/usr/local/bin/k3s, crictl, pull, "{{ postgres_image }}"] - vars: - postgres_image: >- - {{ (lookup('ansible.builtin.file', openshell_postgres_fixture) - | ansible.builtin.from_yaml_all - | selectattr('kind', 'equalto', 'Deployment') | first) - .spec.template.spec.containers[0].image }} - register: postgres_pull - until: postgres_pull.rc == 0 - retries: 3 - delay: 5 - - name: Download Helm installer ansible.builtin.get_url: url: https://raw.githubusercontent.com/helm/helm/06468084e85c244c712834933d25ea232a4c2093/scripts/get-helm-4 diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index 4b76e3d0b4..7255dc1d85 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -25,7 +25,6 @@ let qemu = qemuPkgs.qemu.override { hostCpuOnly = true; }; # Every installer loads the sandbox workload image tests run python3 in. e2ePythonImage = "../../artifacts/images/openshell-e2e-python-dev.tar"; - postgresFixture = "../../e2e/kubernetes/postgres-fixture.yaml"; k3sInputs = { agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../../artifacts/binaries/${muslTarget}/openshell"; @@ -67,8 +66,6 @@ let "ansible/playbooks/test-tools.yaml" "ansible/playbooks/k3s.yaml" ]; - # k3s.yaml pulls the image of the k3s-ha installer's PostgreSQL fixture. - inputs.openshell_postgres_fixture = postgresFixture; }; fedora-podman-rootful = fedoraPodman "rootful"; fedora-podman-rootless = fedoraPodman "rootless"; @@ -94,8 +91,7 @@ let inherit (parent) use_galaxy; playbooks = parent.playbooks ++ setup.playbooks; inputs = - (parent.inputs or { }) - // (if parent ? driver then { environment_driver = parent.driver; } else { }) + (if parent ? driver then { environment_driver = parent.driver; } else { }) // (setup.inputs or { }); }; }; @@ -157,7 +153,6 @@ let inherit (base) machine; setup = { inherit (base) use_galaxy playbooks; - inputs = base.inputs or { }; }; }) bases ++ featureEnvironments @@ -175,7 +170,7 @@ let use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s-ha.yaml" ]; inputs = k3sInputs // { - openshell_postgres_fixture = postgresFixture; + openshell_postgres_fixture = "../../e2e/kubernetes/postgres-fixture.yaml"; }; } { From d8d1a0681f5da4af71081bf6e9d72a83d621148a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 20:59:23 -0800 Subject: [PATCH 3/7] ci(e2e): wipe k3s credentials from the cached setup layer The k3s setup layer boots k3s once, so it holds the cluster CA, token, node password and admin kubeconfig. Stop k3s and delete that state before the layer is saved, so each run's cluster generates its own credentials, and fail the layer build if any credential file remains. Also key the disk cache on ansible_hash.rs, which feeds the setup layer hash. Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 2 +- tests/tmachine/ansible/playbooks/k3s.yaml | 28 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index f4dee9ab32..032a5a4186 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -164,7 +164,7 @@ jobs: # environment needs its own key; its installers share it. No # restore-keys fallback: restoring an older entry would carry its # stale disks into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'scripts/keycloak-realm.json') }} + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'tests/tmachine/src/qemu/ansible_hash.rs', 'scripts/keycloak-realm.json') }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index d27c6c487b..99c9f667ff 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -71,3 +71,31 @@ creates: /usr/local/bin/helm environment: DESIRED_VERSION: "v{{ helm_version }}" + + # This layer is cached across runs, so it must not carry cluster + # credentials: every disk built on it would share one CA, token and admin + # kubeconfig. Stop K3s and delete the state it created on first start, + # keeping the containerd image store (agent/containerd) and the K3s data + # directory. The next start generates fresh credentials. + - name: Stop K3s and its containers + ansible.builtin.command: + argv: [/usr/local/bin/k3s-killall.sh] + changed_when: true + + - name: Remove the K3s credentials and cluster state + ansible.builtin.shell: | + set -eu + cd /var/lib/rancher/k3s + rm -rf server/tls server/cred server/db server/token server/agent-token server/node-token + find agent -maxdepth 1 -type f \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' \) -delete + rm -rf /etc/rancher/node/password /etc/rancher/k3s/k3s.yaml /var/lib/kubelet/pods /var/lib/cni + changed_when: true + + - name: Check no K3s credentials remain + ansible.builtin.shell: | + find /var/lib/rancher/k3s/server /var/lib/rancher/k3s/agent /etc/rancher/k3s /etc/rancher/node \ + -maxdepth 1 \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' -o -name token -o -name password \ + -o -name 'k3s.yaml' -o -name '*.db' -o -name tls -o -name cred \) -print + register: k3s_leftover_credentials + changed_when: false + failed_when: k3s_leftover_credentials.stdout | length > 0 From e03318df8cffc003001f63e58f528b8aa66a536b Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 18 Nov 2026 13:49:06 +0100 Subject: [PATCH 4/7] refactor(tmachine): keep setup layers under disks/setup Install layers already live in disks/install. Give setup layers a sibling directory so the CI cache path names the setup layers directly and a wider glob cannot pull install layers into it. Existing local setup layers in disks/ are no longer found and are rebuilt once; they can be deleted. Signed-off-by: Evan Lezar --- .github/workflows/e2e-suites.yml | 2 +- tests/tmachine/src/qemu/layer.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 032a5a4186..7a943ed29e 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -153,7 +153,7 @@ jobs: # Only the setup layers, which depend on the environment alone. The # install layers under disks/install hold this commit's build (and # on k3s, the gateway's credentials), so no later run could use them. - path: ~/.cache/tmachine/disks/*.qcow2 + path: ~/.cache/tmachine/disks/setup/*.qcow2 # tmachine keys its setup layers on a hash of the Ansible tree, the # environments' playbook lists and setup inputs (config.nix, and # files such as the Keycloak realm file) and the cache version in diff --git a/tests/tmachine/src/qemu/layer.rs b/tests/tmachine/src/qemu/layer.rs index 7cc9b09aa1..97eb1da0f2 100644 --- a/tests/tmachine/src/qemu/layer.rs +++ b/tests/tmachine/src/qemu/layer.rs @@ -14,7 +14,7 @@ use super::vm::{LAYER_POWER_OFF_TIMEOUT, QemuVm}; /// Where setup layers are kept, under tmachine's cache directory. They depend /// only on the environment, so CI caches the disks here across commits. -pub(super) const SETUP_DISKS: &str = "disks"; +pub(super) const SETUP_DISKS: &str = "disks/setup"; /// Where install layers are kept. They hold the build under test, so CI leaves /// them out of its cache. pub(super) const INSTALL_DISKS: &str = "disks/install"; From 13db26b2eec6f25d15e19b16f0df821872301ee0 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 14:21:58 -0800 Subject: [PATCH 5/7] ci(e2e): key the tmachine disk cache on the setup layer hash Add `tmachine layer-key `, which prints the hash tmachine names the environment's setup layer after, or `none` when the environment has no setup playbooks, without starting a VM. setup() now derives its layer name from the same function. Key the e2e-suites disk cache on that value instead of a hand-written hashFiles list, so a new setup input changes the key without a workflow edit. Skip the cache for environments without a setup layer. Closes #5627 Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 30 +++++--- TESTING.md | 10 +-- tests/tmachine/src/lib.rs | 5 +- tests/tmachine/src/main.rs | 12 ++- tests/tmachine/src/qemu/ansible_hash.rs | 5 +- tests/tmachine/src/qemu/mod.rs | 2 +- tests/tmachine/src/qemu/setup.rs | 99 +++++++++++++++++++++++-- 7 files changed, 135 insertions(+), 28 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 7a943ed29e..d6f4b56c72 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -147,24 +147,32 @@ jobs: with: cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + # The hash tmachine names the environment's setup layer after, or `none` + # when the environment has no setup layer. It reads the base image, which + # the run step needs anyway, so this mostly moves that download earlier. + - name: Compute the tmachine setup layer key + id: layer-key + env: + ENVIRONMENT: ${{ matrix.environment }} + run: | + set -euo pipefail + nix build --no-link .#tmachine || nix build --no-link .#tmachine + key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}") + echo "key=${key}" >> "$GITHUB_OUTPUT" + - name: Cache tmachine disks + if: steps.layer-key.outputs.key != 'none' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: # Only the setup layers, which depend on the environment alone. The # install layers under disks/install hold this commit's build (and # on k3s, the gateway's credentials), so no later run could use them. path: ~/.cache/tmachine/disks/setup/*.qcow2 - # tmachine keys its setup layers on a hash of the Ansible tree, the - # environments' playbook lists and setup inputs (config.nix, and - # files such as the Keycloak realm file) and the cache version in - # setup.rs. actions/cache doesn't save after an exact key hit, so a - # key that misses one of those would rebuild the changed layers on - # every run. Environments share no layers, and - # only the first job to save a key gets its disks in, so each - # environment needs its own key; its installers share it. No - # restore-keys fallback: restoring an older entry would carry its - # stale disks into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'tests/tmachine/src/qemu/ansible_hash.rs', 'scripts/keycloak-realm.json') }} + # Keyed on the layer's own hash, so the cache misses exactly when + # tmachine would rebuild the layer. The environment's installers + # share the key. No restore-keys fallback: restoring an older entry + # would carry its stale disks into the new one, which then only grows. + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/TESTING.md b/TESTING.md index 58bb638831..93e410cf86 100644 --- a/TESTING.md +++ b/TESTING.md @@ -305,11 +305,11 @@ service. setup playbook starts the service (with a restart policy, since setup is cached as a disk) and writes the variables as `KEY=VALUE` lines to `/etc/openshell-test/env.d/.env`. Pass files the playbook needs - through the environment's setup `inputs`, and add them to the tmachine disk - cache key in `e2e-suites.yml`. The runner rejects names without the - `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, so - values never appear in a runner command. Values are taken literally, without - quoting. + through the environment's setup `inputs`; they are part of the setup layer's + hash, which CI's disk cache keys on too. The runner rejects names without + the `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, + so values never appear in a runner command. Values are taken literally, + without quoting. - **Fixture.** Put the service's fixture in a support crate under `tests/support/`, named `-support`, so any suite can depend on it, or reuse the existing one. It defines the variable names as constants and a diff --git a/tests/tmachine/src/lib.rs b/tests/tmachine/src/lib.rs index 5376c367af..662e6d562f 100644 --- a/tests/tmachine/src/lib.rs +++ b/tests/tmachine/src/lib.rs @@ -4,7 +4,8 @@ //! Isolated QEMU test machines provisioned with Ansible. //! //! The `tmachine` binary drives one-shot `setup`, `install`, and `test` -//! commands. Callers that need to act on a running machine use [`Session`]. +//! commands, and prints the key of an environment's setup layer with +//! `layer-key`. Callers that need to act on a running machine use [`Session`]. //! A binary that starts machines should run its work in [`until_signal`], so //! a `SIGTERM` does not leave QEMU running. @@ -12,4 +13,4 @@ pub mod ansible; pub mod config; mod qemu; -pub use qemu::{Session, install, setup, ssh_port, test, until_signal}; +pub use qemu::{Session, install, setup, setup_layer_key, ssh_port, test, until_signal}; diff --git a/tests/tmachine/src/main.rs b/tests/tmachine/src/main.rs index d38325cd36..5cb644c17e 100644 --- a/tests/tmachine/src/main.rs +++ b/tests/tmachine/src/main.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use tmachine::config::{Config, Environment, Installer, Machine}; -use tmachine::{install, setup, test}; +use tmachine::{install, setup, setup_layer_key, test}; #[derive(Parser)] #[command(name = "tmachine")] @@ -23,6 +23,11 @@ enum Command { Setup { environment: String, }, + /// Print the key of the environment's setup layer, or `none` when it has + /// no setup playbooks, without starting a VM. + LayerKey { + environment: String, + }, Install { environment: String, installer: String, @@ -51,6 +56,11 @@ async fn run(cli: Cli) -> Result<()> { let (machine, environment) = find_environment(&config, &environment)?; setup(&machine, &environment).await?; } + Command::LayerKey { environment } => { + let (machine, environment) = find_environment(&config, &environment)?; + let key = setup_layer_key(&machine, &environment)?; + println!("{}", key.as_deref().unwrap_or("none")); + } Command::Install { environment, installer, diff --git a/tests/tmachine/src/qemu/ansible_hash.rs b/tests/tmachine/src/qemu/ansible_hash.rs index 3426225898..ddfc7054cc 100644 --- a/tests/tmachine/src/qemu/ansible_hash.rs +++ b/tests/tmachine/src/qemu/ansible_hash.rs @@ -12,7 +12,10 @@ use super::layer::hash_file; pub(super) fn hash_sources(hasher: &mut Hasher) -> Result<()> { let requirements = crate::ansible::requirements_path(); - let root = requirements.parent().unwrap(); + hash_sources_in(hasher, requirements.parent().unwrap()) +} + +pub(super) fn hash_sources_in(hasher: &mut Hasher, root: &Path) -> Result<()> { hash_tree(hasher, root, root) } diff --git a/tests/tmachine/src/qemu/mod.rs b/tests/tmachine/src/qemu/mod.rs index d1b7c119fa..d0f611b896 100644 --- a/tests/tmachine/src/qemu/mod.rs +++ b/tests/tmachine/src/qemu/mod.rs @@ -14,6 +14,6 @@ mod vm; pub use cleanup::until_signal; pub use install::install; pub use session::Session; -pub use setup::setup; +pub use setup::{setup, setup_layer_key}; pub use test::test; pub use vm::ssh_port; diff --git a/tests/tmachine/src/qemu/setup.rs b/tests/tmachine/src/qemu/setup.rs index 83c846fee9..d4b9a9ca91 100644 --- a/tests/tmachine/src/qemu/setup.rs +++ b/tests/tmachine/src/qemu/setup.rs @@ -1,24 +1,23 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use blake3::Hasher; use crate::config::{Environment, Machine}; -use super::ansible_hash::hash_sources; +use super::ansible_hash::hash_sources_in; use super::layer::{SETUP_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; const SETUP_CACHE_VERSION: &[u8] = b"tmachine-disk-blake3-v1"; pub async fn setup(machine: &Machine, environment: &Environment) -> Result { - if environment.setup.playbooks.is_empty() { + let Some(hash) = setup_layer_key(machine, environment)? else { return Ok(machine.base_image.clone()); - } + }; - let hash = setup_hash(machine, environment)?; cached_layer( SETUP_DISKS, &machine.base_image, @@ -30,13 +29,23 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result Result { +/// The hash that names the environment's setup layer (`disks/.qcow2`), +/// or `None` when it has no setup playbooks and so runs on the base image. +pub fn setup_layer_key(machine: &Machine, environment: &Environment) -> Result> { + if environment.setup.playbooks.is_empty() { + return Ok(None); + } + let requirements = crate::ansible::requirements_path(); + setup_hash(machine, environment, requirements.parent().unwrap()).map(Some) +} + +fn setup_hash(machine: &Machine, environment: &Environment, ansible: &Path) -> Result { let mut hasher = Hasher::new(); hasher.update(SETUP_CACHE_VERSION); hash_file(&mut hasher, &machine.base_image) .with_context(|| format!("failed to hash base image for machine {:?}", machine.name))?; hasher.update(&[u8::from(environment.setup.use_galaxy)]); - hash_sources(&mut hasher).context("failed to hash Ansible sources")?; + hash_sources_in(&mut hasher, ansible).context("failed to hash Ansible sources")?; hash_files(&mut hasher, &environment.setup.playbooks) .context("failed to hash setup playbooks")?; // Skipped when empty so environments without inputs keep their cache keys. @@ -45,3 +54,79 @@ fn setup_hash(machine: &Machine, environment: &Environment) -> Result { } Ok(hasher.finalize().to_hex().to_string()) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::fs; + + use super::*; + use crate::config::Setup; + + fn environment(playbooks: Vec, inputs: BTreeMap) -> Environment { + Environment { + name: "test".into(), + machine: "test".into(), + setup: Setup { + use_galaxy: false, + playbooks, + inputs, + }, + } + } + + #[test] + fn an_environment_without_setup_playbooks_has_no_layer() { + let machine = Machine { + name: "test".into(), + base_image: "/nonexistent/base.qcow2".into(), + }; + let key = setup_layer_key(&machine, &environment(Vec::new(), BTreeMap::new())).unwrap(); + assert_eq!(key, None); + } + + #[test] + fn the_setup_hash_follows_every_setup_input() { + let dir = tempfile::tempdir().unwrap(); + let ansible = dir.path().join("ansible"); + fs::create_dir(&ansible).unwrap(); + fs::write(ansible.join("requirements.yaml"), "[]").unwrap(); + let base_image = dir.path().join("base.qcow2"); + fs::write(&base_image, "base").unwrap(); + let playbook = dir.path().join("setup.yaml"); + fs::write(&playbook, "- hosts: all").unwrap(); + let input = dir.path().join("realm.json"); + fs::write(&input, "{}").unwrap(); + + let machine = Machine { + name: "test".into(), + base_image: base_image.clone(), + }; + let environment = environment( + vec![playbook.clone()], + BTreeMap::from([("realm".into(), input.clone())]), + ); + let hash = || setup_hash(&machine, &environment, &ansible).unwrap(); + + let original = hash(); + assert_eq!(original.len(), 64); + assert_eq!(hash(), original); + + let mut seen = vec![original]; + for (file, contents) in [ + (&input, "{\"realm\": 1}"), + (&playbook, "- hosts: guest"), + (&ansible.join("requirements.yaml"), "[role]"), + (&base_image, "rebuilt base"), + ] { + fs::write(file, contents).unwrap(); + let changed = hash(); + assert!( + !seen.contains(&changed), + "{} did not change the hash", + file.display() + ); + seen.push(changed); + } + } +} From d1c5b7d7e8688a301916483727150a9bb859919b Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 22:56:40 -0800 Subject: [PATCH 6/7] ci(integration): key the tmachine disk cache on the setup layer hash The integration runner cached all of ~/.cache/tmachine, install layers included, under one fixed key per OS and arch, and skipped ubuntu-k3s. Cache only the setup layers and key them on `tmachine layer-key`, as e2e-suites.yml does. Signed-off-by: Matthew Grossman --- .github/workflows/integration-runner.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index f84811bdd9..fc2e02cfee 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -64,12 +64,26 @@ jobs: - uses: ./.github/actions/setup-nix + # The hash tmachine names the environment's setup layer after, or `none` + # when the environment has no setup layer. See e2e-suites.yml. + - name: Compute the tmachine setup layer key + id: layer-key + env: + ENVIRONMENT: ${{ matrix.environment }} + run: | + set -euo pipefail + nix build --no-link .#tmachine || nix build --no-link .#tmachine + key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}") + echo "key=${key}" >> "$GITHUB_OUTPUT" + + # Only the setup layers: install layers under disks/install hold this + # commit's build, so no later run could use them. - name: Cache tmachine disks - if: matrix.environment != 'ubuntu-k3s' + if: steps.layer-key.outputs.key != 'none' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: ~/.cache/tmachine - key: tmachine-${{ runner.os }}-${{ runner.arch }} + path: ~/.cache/tmachine/disks/setup/*.qcow2 + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 From b3fb5ab85072d33d4d850c7b093032ab2ff90acb Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 16:13:27 -0800 Subject: [PATCH 7/7] refactor(tmachine): simplify the layer-key command Print the setup-layer hash for every environment instead of `none` for environments without setup playbooks, and drop the workflow checks for it. Expose the existing hash function directly, and drop the tests and the Ansible hashing split that only they needed. Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 7 +- .github/workflows/integration-runner.yml | 5 +- tests/tmachine/src/main.rs | 7 +- tests/tmachine/src/qemu/ansible_hash.rs | 5 +- tests/tmachine/src/qemu/setup.rs | 100 ++--------------------- 5 files changed, 17 insertions(+), 107 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index d6f4b56c72..dc39e1610c 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -147,9 +147,9 @@ jobs: with: cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} - # The hash tmachine names the environment's setup layer after, or `none` - # when the environment has no setup layer. It reads the base image, which - # the run step needs anyway, so this mostly moves that download earlier. + # The hash tmachine names the environment's setup layer after. It reads + # the base image, which the run step needs anyway, so this mostly moves + # that download earlier. - name: Compute the tmachine setup layer key id: layer-key env: @@ -161,7 +161,6 @@ jobs: echo "key=${key}" >> "$GITHUB_OUTPUT" - name: Cache tmachine disks - if: steps.layer-key.outputs.key != 'none' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: # Only the setup layers, which depend on the environment alone. The diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index fc2e02cfee..239371e4c3 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -64,8 +64,8 @@ jobs: - uses: ./.github/actions/setup-nix - # The hash tmachine names the environment's setup layer after, or `none` - # when the environment has no setup layer. See e2e-suites.yml. + # The hash tmachine names the environment's setup layer after. See + # e2e-suites.yml. - name: Compute the tmachine setup layer key id: layer-key env: @@ -79,7 +79,6 @@ jobs: # Only the setup layers: install layers under disks/install hold this # commit's build, so no later run could use them. - name: Cache tmachine disks - if: steps.layer-key.outputs.key != 'none' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/.cache/tmachine/disks/setup/*.qcow2 diff --git a/tests/tmachine/src/main.rs b/tests/tmachine/src/main.rs index 5cb644c17e..5ab7d2c260 100644 --- a/tests/tmachine/src/main.rs +++ b/tests/tmachine/src/main.rs @@ -23,8 +23,8 @@ enum Command { Setup { environment: String, }, - /// Print the key of the environment's setup layer, or `none` when it has - /// no setup playbooks, without starting a VM. + /// Print the hash that names the environment's setup layer, without + /// starting a VM. LayerKey { environment: String, }, @@ -58,8 +58,7 @@ async fn run(cli: Cli) -> Result<()> { } Command::LayerKey { environment } => { let (machine, environment) = find_environment(&config, &environment)?; - let key = setup_layer_key(&machine, &environment)?; - println!("{}", key.as_deref().unwrap_or("none")); + println!("{}", setup_layer_key(&machine, &environment)?); } Command::Install { environment, diff --git a/tests/tmachine/src/qemu/ansible_hash.rs b/tests/tmachine/src/qemu/ansible_hash.rs index ddfc7054cc..3426225898 100644 --- a/tests/tmachine/src/qemu/ansible_hash.rs +++ b/tests/tmachine/src/qemu/ansible_hash.rs @@ -12,10 +12,7 @@ use super::layer::hash_file; pub(super) fn hash_sources(hasher: &mut Hasher) -> Result<()> { let requirements = crate::ansible::requirements_path(); - hash_sources_in(hasher, requirements.parent().unwrap()) -} - -pub(super) fn hash_sources_in(hasher: &mut Hasher, root: &Path) -> Result<()> { + let root = requirements.parent().unwrap(); hash_tree(hasher, root, root) } diff --git a/tests/tmachine/src/qemu/setup.rs b/tests/tmachine/src/qemu/setup.rs index d4b9a9ca91..66fbc765ee 100644 --- a/tests/tmachine/src/qemu/setup.rs +++ b/tests/tmachine/src/qemu/setup.rs @@ -1,23 +1,24 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use anyhow::{Context, Result}; use blake3::Hasher; use crate::config::{Environment, Machine}; -use super::ansible_hash::hash_sources_in; +use super::ansible_hash::hash_sources; use super::layer::{SETUP_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; const SETUP_CACHE_VERSION: &[u8] = b"tmachine-disk-blake3-v1"; pub async fn setup(machine: &Machine, environment: &Environment) -> Result { - let Some(hash) = setup_layer_key(machine, environment)? else { + if environment.setup.playbooks.is_empty() { return Ok(machine.base_image.clone()); - }; + } + let hash = setup_layer_key(machine, environment)?; cached_layer( SETUP_DISKS, &machine.base_image, @@ -29,23 +30,14 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result.qcow2`), -/// or `None` when it has no setup playbooks and so runs on the base image. -pub fn setup_layer_key(machine: &Machine, environment: &Environment) -> Result> { - if environment.setup.playbooks.is_empty() { - return Ok(None); - } - let requirements = crate::ansible::requirements_path(); - setup_hash(machine, environment, requirements.parent().unwrap()).map(Some) -} - -fn setup_hash(machine: &Machine, environment: &Environment, ansible: &Path) -> Result { +/// The hash that names the environment's setup layer (`disks/.qcow2`). +pub fn setup_layer_key(machine: &Machine, environment: &Environment) -> Result { let mut hasher = Hasher::new(); hasher.update(SETUP_CACHE_VERSION); hash_file(&mut hasher, &machine.base_image) .with_context(|| format!("failed to hash base image for machine {:?}", machine.name))?; hasher.update(&[u8::from(environment.setup.use_galaxy)]); - hash_sources_in(&mut hasher, ansible).context("failed to hash Ansible sources")?; + hash_sources(&mut hasher).context("failed to hash Ansible sources")?; hash_files(&mut hasher, &environment.setup.playbooks) .context("failed to hash setup playbooks")?; // Skipped when empty so environments without inputs keep their cache keys. @@ -54,79 +46,3 @@ fn setup_hash(machine: &Machine, environment: &Environment, ansible: &Path) -> R } Ok(hasher.finalize().to_hex().to_string()) } - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - use std::fs; - - use super::*; - use crate::config::Setup; - - fn environment(playbooks: Vec, inputs: BTreeMap) -> Environment { - Environment { - name: "test".into(), - machine: "test".into(), - setup: Setup { - use_galaxy: false, - playbooks, - inputs, - }, - } - } - - #[test] - fn an_environment_without_setup_playbooks_has_no_layer() { - let machine = Machine { - name: "test".into(), - base_image: "/nonexistent/base.qcow2".into(), - }; - let key = setup_layer_key(&machine, &environment(Vec::new(), BTreeMap::new())).unwrap(); - assert_eq!(key, None); - } - - #[test] - fn the_setup_hash_follows_every_setup_input() { - let dir = tempfile::tempdir().unwrap(); - let ansible = dir.path().join("ansible"); - fs::create_dir(&ansible).unwrap(); - fs::write(ansible.join("requirements.yaml"), "[]").unwrap(); - let base_image = dir.path().join("base.qcow2"); - fs::write(&base_image, "base").unwrap(); - let playbook = dir.path().join("setup.yaml"); - fs::write(&playbook, "- hosts: all").unwrap(); - let input = dir.path().join("realm.json"); - fs::write(&input, "{}").unwrap(); - - let machine = Machine { - name: "test".into(), - base_image: base_image.clone(), - }; - let environment = environment( - vec![playbook.clone()], - BTreeMap::from([("realm".into(), input.clone())]), - ); - let hash = || setup_hash(&machine, &environment, &ansible).unwrap(); - - let original = hash(); - assert_eq!(original.len(), 64); - assert_eq!(hash(), original); - - let mut seen = vec![original]; - for (file, contents) in [ - (&input, "{\"realm\": 1}"), - (&playbook, "- hosts: guest"), - (&ansible.join("requirements.yaml"), "[role]"), - (&base_image, "rebuilt base"), - ] { - fs::write(file, contents).unwrap(); - let changed = hash(); - assert!( - !seen.contains(&changed), - "{} did not change the hash", - file.display() - ); - seen.push(changed); - } - } -}