diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index ec6cc2e54b..dc39e1610c 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -147,24 +147,31 @@ jobs: with: cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} - # integration-runner.yml skips the disk cache for ubuntu-k3s and doesn't say - # why, so the k3s environment skips it here too until that is understood. + # The hash tmachine names the environment's setup layer after. It reads + # the base image, which the run step needs anyway, so this mostly moves + # that download earlier. + - name: Compute the tmachine setup layer key + id: layer-key + env: + ENVIRONMENT: ${{ matrix.environment }} + run: | + set -euo pipefail + nix build --no-link .#tmachine || nix build --no-link .#tmachine + key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}") + echo "key=${key}" >> "$GITHUB_OUTPUT" + - name: Cache tmachine disks - if: matrix.environment != 'ubuntu-k3s' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: ~/.cache/tmachine - # tmachine keys its disk layers on a hash of the Ansible tree, the - # environments' playbook lists and setup inputs (config.nix, and - # files such as the Keycloak realm file) and the cache versions in - # setup.rs and install.rs. actions/cache doesn't save after an - # exact key hit, so a key that misses one of those would rebuild the - # changed layers on every run. Environments share no layers, and - # only the first job to save a key gets its disks in, so each - # environment and installer needs its own key. No restore-keys - # fallback: restoring an older entry would carry its stale disks - # into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/install.rs', 'scripts/keycloak-realm.json') }} + # Only the setup layers, which depend on the environment alone. The + # install layers under disks/install hold this commit's build (and + # on k3s, the gateway's credentials), so no later run could use them. + path: ~/.cache/tmachine/disks/setup/*.qcow2 + # Keyed on the layer's own hash, so the cache misses exactly when + # tmachine would rebuild the layer. The environment's installers + # share the key. No restore-keys fallback: restoring an older entry + # would carry its stale disks into the new one, which then only grows. + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index f84811bdd9..239371e4c3 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -64,12 +64,25 @@ jobs: - uses: ./.github/actions/setup-nix + # The hash tmachine names the environment's setup layer after. See + # e2e-suites.yml. + - name: Compute the tmachine setup layer key + id: layer-key + env: + ENVIRONMENT: ${{ matrix.environment }} + run: | + set -euo pipefail + nix build --no-link .#tmachine || nix build --no-link .#tmachine + key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}") + echo "key=${key}" >> "$GITHUB_OUTPUT" + + # Only the setup layers: install layers under disks/install hold this + # commit's build, so no later run could use them. - name: Cache tmachine disks - if: matrix.environment != 'ubuntu-k3s' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: ~/.cache/tmachine - key: tmachine-${{ runner.os }}-${{ runner.arch }} + path: ~/.cache/tmachine/disks/setup/*.qcow2 + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/TESTING.md b/TESTING.md index 58bb638831..93e410cf86 100644 --- a/TESTING.md +++ b/TESTING.md @@ -305,11 +305,11 @@ service. setup playbook starts the service (with a restart policy, since setup is cached as a disk) and writes the variables as `KEY=VALUE` lines to `/etc/openshell-test/env.d/.env`. Pass files the playbook needs - through the environment's setup `inputs`, and add them to the tmachine disk - cache key in `e2e-suites.yml`. The runner rejects names without the - `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, so - values never appear in a runner command. Values are taken literally, without - quoting. + through the environment's setup `inputs`; they are part of the setup layer's + hash, which CI's disk cache keys on too. The runner rejects names without + the `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, + so values never appear in a runner command. Values are taken literally, + without quoting. - **Fixture.** Put the service's fixture in a support crate under `tests/support/`, named `-support`, so any suite can depend on it, or reuse the existing one. It defines the variable names as constants and a diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index d27c6c487b..99c9f667ff 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -71,3 +71,31 @@ creates: /usr/local/bin/helm environment: DESIRED_VERSION: "v{{ helm_version }}" + + # This layer is cached across runs, so it must not carry cluster + # credentials: every disk built on it would share one CA, token and admin + # kubeconfig. Stop K3s and delete the state it created on first start, + # keeping the containerd image store (agent/containerd) and the K3s data + # directory. The next start generates fresh credentials. + - name: Stop K3s and its containers + ansible.builtin.command: + argv: [/usr/local/bin/k3s-killall.sh] + changed_when: true + + - name: Remove the K3s credentials and cluster state + ansible.builtin.shell: | + set -eu + cd /var/lib/rancher/k3s + rm -rf server/tls server/cred server/db server/token server/agent-token server/node-token + find agent -maxdepth 1 -type f \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' \) -delete + rm -rf /etc/rancher/node/password /etc/rancher/k3s/k3s.yaml /var/lib/kubelet/pods /var/lib/cni + changed_when: true + + - name: Check no K3s credentials remain + ansible.builtin.shell: | + find /var/lib/rancher/k3s/server /var/lib/rancher/k3s/agent /etc/rancher/k3s /etc/rancher/node \ + -maxdepth 1 \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' -o -name token -o -name password \ + -o -name 'k3s.yaml' -o -name '*.db' -o -name tls -o -name cred \) -print + register: k3s_leftover_credentials + changed_when: false + failed_when: k3s_leftover_credentials.stdout | length > 0 diff --git a/tests/tmachine/src/lib.rs b/tests/tmachine/src/lib.rs index 5376c367af..662e6d562f 100644 --- a/tests/tmachine/src/lib.rs +++ b/tests/tmachine/src/lib.rs @@ -4,7 +4,8 @@ //! Isolated QEMU test machines provisioned with Ansible. //! //! The `tmachine` binary drives one-shot `setup`, `install`, and `test` -//! commands. Callers that need to act on a running machine use [`Session`]. +//! commands, and prints the key of an environment's setup layer with +//! `layer-key`. Callers that need to act on a running machine use [`Session`]. //! A binary that starts machines should run its work in [`until_signal`], so //! a `SIGTERM` does not leave QEMU running. @@ -12,4 +13,4 @@ pub mod ansible; pub mod config; mod qemu; -pub use qemu::{Session, install, setup, ssh_port, test, until_signal}; +pub use qemu::{Session, install, setup, setup_layer_key, ssh_port, test, until_signal}; diff --git a/tests/tmachine/src/main.rs b/tests/tmachine/src/main.rs index d38325cd36..5ab7d2c260 100644 --- a/tests/tmachine/src/main.rs +++ b/tests/tmachine/src/main.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use tmachine::config::{Config, Environment, Installer, Machine}; -use tmachine::{install, setup, test}; +use tmachine::{install, setup, setup_layer_key, test}; #[derive(Parser)] #[command(name = "tmachine")] @@ -23,6 +23,11 @@ enum Command { Setup { environment: String, }, + /// Print the hash that names the environment's setup layer, without + /// starting a VM. + LayerKey { + environment: String, + }, Install { environment: String, installer: String, @@ -51,6 +56,10 @@ async fn run(cli: Cli) -> Result<()> { let (machine, environment) = find_environment(&config, &environment)?; setup(&machine, &environment).await?; } + Command::LayerKey { environment } => { + let (machine, environment) = find_environment(&config, &environment)?; + println!("{}", setup_layer_key(&machine, &environment)?); + } Command::Install { environment, installer, diff --git a/tests/tmachine/src/qemu/install.rs b/tests/tmachine/src/qemu/install.rs index e1e5910520..25b3d6abf0 100644 --- a/tests/tmachine/src/qemu/install.rs +++ b/tests/tmachine/src/qemu/install.rs @@ -9,7 +9,7 @@ use blake3::Hasher; use crate::config::{Environment, Installer, Machine}; use super::ansible_hash::hash_sources; -use super::layer::{cached_layer, hash_file, hash_files, hash_inputs}; +use super::layer::{INSTALL_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; use super::setup::setup; const INSTALL_CACHE_VERSION: &[u8] = b"tmachine-install-blake3-v1"; @@ -26,6 +26,7 @@ pub async fn install( let hash = install_hash(&setup_disk, installer)?; cached_layer( + INSTALL_DISKS, &setup_disk, &hash, installer.use_galaxy, diff --git a/tests/tmachine/src/qemu/layer.rs b/tests/tmachine/src/qemu/layer.rs index e6ec03650d..97eb1da0f2 100644 --- a/tests/tmachine/src/qemu/layer.rs +++ b/tests/tmachine/src/qemu/layer.rs @@ -12,7 +12,15 @@ use directories::ProjectDirs; use super::img::QemuImage; use super::vm::{LAYER_POWER_OFF_TIMEOUT, QemuVm}; +/// Where setup layers are kept, under tmachine's cache directory. They depend +/// only on the environment, so CI caches the disks here across commits. +pub(super) const SETUP_DISKS: &str = "disks/setup"; +/// Where install layers are kept. They hold the build under test, so CI leaves +/// them out of its cache. +pub(super) const INSTALL_DISKS: &str = "disks/install"; + pub(super) async fn cached_layer( + disks: &str, base_image: &Path, hash: &str, use_galaxy: bool, @@ -20,7 +28,7 @@ pub(super) async fn cached_layer( inputs: &BTreeMap, ) -> Result { let project_dirs = ProjectDirs::from("com", "nvidia", "tmachine").unwrap(); - let disks_dir = project_dirs.cache_dir().join("disks"); + let disks_dir = project_dirs.cache_dir().join(disks); std::fs::create_dir_all(&disks_dir).unwrap(); let disk = disks_dir.join(format!("{hash}.qcow2")); diff --git a/tests/tmachine/src/qemu/mod.rs b/tests/tmachine/src/qemu/mod.rs index d1b7c119fa..d0f611b896 100644 --- a/tests/tmachine/src/qemu/mod.rs +++ b/tests/tmachine/src/qemu/mod.rs @@ -14,6 +14,6 @@ mod vm; pub use cleanup::until_signal; pub use install::install; pub use session::Session; -pub use setup::setup; +pub use setup::{setup, setup_layer_key}; pub use test::test; pub use vm::ssh_port; diff --git a/tests/tmachine/src/qemu/session.rs b/tests/tmachine/src/qemu/session.rs index c3682d5958..576ece9dc6 100644 --- a/tests/tmachine/src/qemu/session.rs +++ b/tests/tmachine/src/qemu/session.rs @@ -60,7 +60,8 @@ pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(600); /// A machine booted from a throwaway overlay of the installed disk. /// /// The VM stops when the session is dropped, so a caller that fails or -/// panics cannot leak it. Call [`Session::down`] for a clean shutdown. +/// panics cannot leak it. Call [`Session::down`] to stop it and wait until +/// QEMU has exited. pub struct Session { // Declared before `_dir` so the VM stops before its disk is removed. vm: QemuVm, @@ -173,13 +174,14 @@ impl Session { .context("open SSH shell in tmachine VM") } - /// Power the machine off and wait for QEMU to exit. + /// Stop the machine and wait for QEMU to exit. /// - /// This never panics and never fails: teardown has no useful recovery, - /// so a guest that does not power off cleanly is killed and the problem - /// is logged to stderr. + /// QEMU is killed rather than the guest powered off: the overlay is + /// thrown away, so a clean shutdown would only cost time. Copy out + /// anything the guest holds first. This never panics and never fails. pub async fn down(self) { - self.vm.power_off_or_kill().await; + let Self { mut vm, .. } = self; + vm.kill().await; } } diff --git a/tests/tmachine/src/qemu/setup.rs b/tests/tmachine/src/qemu/setup.rs index 185d6c290b..66fbc765ee 100644 --- a/tests/tmachine/src/qemu/setup.rs +++ b/tests/tmachine/src/qemu/setup.rs @@ -9,7 +9,7 @@ use blake3::Hasher; use crate::config::{Environment, Machine}; use super::ansible_hash::hash_sources; -use super::layer::{cached_layer, hash_file, hash_files, hash_inputs}; +use super::layer::{SETUP_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; const SETUP_CACHE_VERSION: &[u8] = b"tmachine-disk-blake3-v1"; @@ -18,8 +18,9 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result Result Result { +/// The hash that names the environment's setup layer (`disks/.qcow2`). +pub fn setup_layer_key(machine: &Machine, environment: &Environment) -> Result { let mut hasher = Hasher::new(); hasher.update(SETUP_CACHE_VERSION); hash_file(&mut hasher, &machine.base_image) diff --git a/tests/tmachine/src/qemu/vm.rs b/tests/tmachine/src/qemu/vm.rs index b5952a0555..354d04c53a 100644 --- a/tests/tmachine/src/qemu/vm.rs +++ b/tests/tmachine/src/qemu/vm.rs @@ -25,18 +25,10 @@ use super::img::QemuImage; /// `TMACHINE_SSH_PORT` says otherwise. const DEFAULT_SSH_PORT: u16 = 2222; -/// How long the QMP power-off request has to be acknowledged. A guest or QEMU -/// that stops answering must not hang teardown. -const QMP_TIMEOUT: Duration = Duration::from_secs(10); - -/// How long a powered-off guest has to exit at teardown before QEMU is -/// killed. -const POWER_OFF_TIMEOUT: Duration = Duration::from_secs(120); - /// How long a guest being saved as a cached layer has to power off. A heavily /// provisioned guest (Docker, k3s) can take minutes to stop its services, and -/// a layer that is killed instead is not cached, so this is much longer than -/// [`POWER_OFF_TIMEOUT`]. It still bounds a guest that never stops. +/// a layer that is killed instead is not cached. It still bounds a guest that +/// never stops. pub(super) const LAYER_POWER_OFF_TIMEOUT: Duration = Duration::from_secs(600); /// The host port forwarded to the guest's SSH server. @@ -170,28 +162,6 @@ impl QemuVm { } } } - - /// Power the guest off and wait for QEMU to exit. Never panics: a guest - /// that can't be powered off cleanly is killed, and the problem is - /// logged. Use this when tearing down, where there is nothing left to do - /// about a failure. - pub(super) async fn power_off_or_kill(mut self) { - let shutdown = match tokio::time::timeout(QMP_TIMEOUT, self.shutdown()).await { - Ok(result) => result, - Err(_) => Err(anyhow::anyhow!("QMP did not answer within {QMP_TIMEOUT:?}")), - }; - match shutdown { - Ok(()) => { - if let Err(error) = self.wait(POWER_OFF_TIMEOUT).await { - eprintln!("tmachine: {error:#}"); - } - } - Err(error) => { - eprintln!("tmachine: could not power off the VM cleanly, killing it: {error:#}"); - self.kill().await; - } - } - } } fn command(