Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 22 additions & 15 deletions .github/workflows/e2e-suites.yml
Original file line number Diff line number Diff line change
Expand Up @@ -147,24 +147,31 @@ jobs:
with:
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}

# integration-runner.yml skips the disk cache for ubuntu-k3s and doesn't say
# why, so the k3s environment skips it here too until that is understood.
# The hash tmachine names the environment's setup layer after. It reads
# the base image, which the run step needs anyway, so this mostly moves
# that download earlier.
- name: Compute the tmachine setup layer key
id: layer-key
env:
ENVIRONMENT: ${{ matrix.environment }}
run: |
set -euo pipefail
nix build --no-link .#tmachine || nix build --no-link .#tmachine
key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}")
echo "key=${key}" >> "$GITHUB_OUTPUT"

- name: Cache tmachine disks
if: matrix.environment != 'ubuntu-k3s'
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
path: ~/.cache/tmachine
# tmachine keys its disk layers on a hash of the Ansible tree, the
# environments' playbook lists and setup inputs (config.nix, and
# files such as the Keycloak realm file) and the cache versions in
# setup.rs and install.rs. actions/cache doesn't save after an
# exact key hit, so a key that misses one of those would rebuild the
# changed layers on every run. Environments share no layers, and
# only the first job to save a key gets its disks in, so each
# environment and installer needs its own key. No restore-keys
# fallback: restoring an older entry would carry its stale disks
# into the new one, which then only grows.
key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/install.rs', 'scripts/keycloak-realm.json') }}
# Only the setup layers, which depend on the environment alone. The
# install layers under disks/install hold this commit's build (and
# on k3s, the gateway's credentials), so no later run could use them.
path: ~/.cache/tmachine/disks/setup/*.qcow2
# Keyed on the layer's own hash, so the cache misses exactly when
# tmachine would rebuild the layer. The environment's installers
# share the key. No restore-keys fallback: restoring an older entry
# would carry its stale disks into the new one, which then only grows.
key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }}

- name: Download integration inputs
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
Expand Down
19 changes: 16 additions & 3 deletions .github/workflows/integration-runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,25 @@ jobs:

- uses: ./.github/actions/setup-nix

# The hash tmachine names the environment's setup layer after. See
# e2e-suites.yml.
- name: Compute the tmachine setup layer key
id: layer-key
env:
ENVIRONMENT: ${{ matrix.environment }}
run: |
set -euo pipefail
nix build --no-link .#tmachine || nix build --no-link .#tmachine
key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}")
echo "key=${key}" >> "$GITHUB_OUTPUT"

# Only the setup layers: install layers under disks/install hold this
# commit's build, so no later run could use them.
- name: Cache tmachine disks
if: matrix.environment != 'ubuntu-k3s'
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
path: ~/.cache/tmachine
key: tmachine-${{ runner.os }}-${{ runner.arch }}
path: ~/.cache/tmachine/disks/setup/*.qcow2
key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }}

- name: Download integration inputs
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
Expand Down
10 changes: 5 additions & 5 deletions TESTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -305,11 +305,11 @@ service.
setup playbook starts the service (with a restart policy, since setup is
cached as a disk) and writes the variables as `KEY=VALUE` lines to
`/etc/openshell-test/env.d/<service>.env`. Pass files the playbook needs
through the environment's setup `inputs`, and add them to the tmachine disk
cache key in `e2e-suites.yml`. The runner rejects names without the
`OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, so
values never appear in a runner command. Values are taken literally, without
quoting.
through the environment's setup `inputs`; they are part of the setup layer's
hash, which CI's disk cache keys on too. The runner rejects names without
the `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest,
so values never appear in a runner command. Values are taken literally,
without quoting.
- **Fixture.** Put the service's fixture in a support crate under
`tests/support/<service>`, named `<service>-support`, so any suite can
depend on it, or reuse the existing one. It defines the variable names as constants and a
Expand Down
28 changes: 28 additions & 0 deletions tests/tmachine/ansible/playbooks/k3s.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,31 @@
creates: /usr/local/bin/helm
environment:
DESIRED_VERSION: "v{{ helm_version }}"

# This layer is cached across runs, so it must not carry cluster
# credentials: every disk built on it would share one CA, token and admin
# kubeconfig. Stop K3s and delete the state it created on first start,
# keeping the containerd image store (agent/containerd) and the K3s data
# directory. The next start generates fresh credentials.
- name: Stop K3s and its containers
ansible.builtin.command:
argv: [/usr/local/bin/k3s-killall.sh]
changed_when: true

- name: Remove the K3s credentials and cluster state
ansible.builtin.shell: |
set -eu
cd /var/lib/rancher/k3s
rm -rf server/tls server/cred server/db server/token server/agent-token server/node-token
find agent -maxdepth 1 -type f \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' \) -delete
rm -rf /etc/rancher/node/password /etc/rancher/k3s/k3s.yaml /var/lib/kubelet/pods /var/lib/cni
changed_when: true

- name: Check no K3s credentials remain
ansible.builtin.shell: |
find /var/lib/rancher/k3s/server /var/lib/rancher/k3s/agent /etc/rancher/k3s /etc/rancher/node \
-maxdepth 1 \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' -o -name token -o -name password \
-o -name 'k3s.yaml' -o -name '*.db' -o -name tls -o -name cred \) -print
register: k3s_leftover_credentials
changed_when: false
failed_when: k3s_leftover_credentials.stdout | length > 0
5 changes: 3 additions & 2 deletions tests/tmachine/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@
//! Isolated QEMU test machines provisioned with Ansible.
//!
//! The `tmachine` binary drives one-shot `setup`, `install`, and `test`
//! commands. Callers that need to act on a running machine use [`Session`].
//! commands, and prints the key of an environment's setup layer with
//! `layer-key`. Callers that need to act on a running machine use [`Session`].
//! A binary that starts machines should run its work in [`until_signal`], so
//! a `SIGTERM` does not leave QEMU running.

pub mod ansible;
pub mod config;
mod qemu;

pub use qemu::{Session, install, setup, ssh_port, test, until_signal};
pub use qemu::{Session, install, setup, setup_layer_key, ssh_port, test, until_signal};
11 changes: 10 additions & 1 deletion tests/tmachine/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use std::path::PathBuf;
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use tmachine::config::{Config, Environment, Installer, Machine};
use tmachine::{install, setup, test};
use tmachine::{install, setup, setup_layer_key, test};

#[derive(Parser)]
#[command(name = "tmachine")]
Expand All @@ -23,6 +23,11 @@ enum Command {
Setup {
environment: String,
},
/// Print the hash that names the environment's setup layer, without
/// starting a VM.
LayerKey {
environment: String,
},
Install {
environment: String,
installer: String,
Expand Down Expand Up @@ -51,6 +56,10 @@ async fn run(cli: Cli) -> Result<()> {
let (machine, environment) = find_environment(&config, &environment)?;
setup(&machine, &environment).await?;
}
Command::LayerKey { environment } => {
let (machine, environment) = find_environment(&config, &environment)?;
println!("{}", setup_layer_key(&machine, &environment)?);
}
Command::Install {
environment,
installer,
Expand Down
3 changes: 2 additions & 1 deletion tests/tmachine/src/qemu/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use blake3::Hasher;
use crate::config::{Environment, Installer, Machine};

use super::ansible_hash::hash_sources;
use super::layer::{cached_layer, hash_file, hash_files, hash_inputs};
use super::layer::{INSTALL_DISKS, cached_layer, hash_file, hash_files, hash_inputs};
use super::setup::setup;

const INSTALL_CACHE_VERSION: &[u8] = b"tmachine-install-blake3-v1";
Expand All @@ -26,6 +26,7 @@ pub async fn install(

let hash = install_hash(&setup_disk, installer)?;
cached_layer(
INSTALL_DISKS,
&setup_disk,
&hash,
installer.use_galaxy,
Expand Down
10 changes: 9 additions & 1 deletion tests/tmachine/src/qemu/layer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,23 @@ use directories::ProjectDirs;
use super::img::QemuImage;
use super::vm::{LAYER_POWER_OFF_TIMEOUT, QemuVm};

/// Where setup layers are kept, under tmachine's cache directory. They depend
/// only on the environment, so CI caches the disks here across commits.
pub(super) const SETUP_DISKS: &str = "disks/setup";
/// Where install layers are kept. They hold the build under test, so CI leaves
/// them out of its cache.
pub(super) const INSTALL_DISKS: &str = "disks/install";

pub(super) async fn cached_layer(
disks: &str,
base_image: &Path,
hash: &str,
use_galaxy: bool,
playbooks: &[PathBuf],
inputs: &BTreeMap<String, PathBuf>,
) -> Result<PathBuf> {
let project_dirs = ProjectDirs::from("com", "nvidia", "tmachine").unwrap();
let disks_dir = project_dirs.cache_dir().join("disks");
let disks_dir = project_dirs.cache_dir().join(disks);
std::fs::create_dir_all(&disks_dir).unwrap();

let disk = disks_dir.join(format!("{hash}.qcow2"));
Expand Down
2 changes: 1 addition & 1 deletion tests/tmachine/src/qemu/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ mod vm;
pub use cleanup::until_signal;
pub use install::install;
pub use session::Session;
pub use setup::setup;
pub use setup::{setup, setup_layer_key};
pub use test::test;
pub use vm::ssh_port;
14 changes: 8 additions & 6 deletions tests/tmachine/src/qemu/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(600);
/// A machine booted from a throwaway overlay of the installed disk.
///
/// The VM stops when the session is dropped, so a caller that fails or
/// panics cannot leak it. Call [`Session::down`] for a clean shutdown.
/// panics cannot leak it. Call [`Session::down`] to stop it and wait until
/// QEMU has exited.
pub struct Session {
// Declared before `_dir` so the VM stops before its disk is removed.
vm: QemuVm,
Expand Down Expand Up @@ -173,13 +174,14 @@ impl Session {
.context("open SSH shell in tmachine VM")
}

/// Power the machine off and wait for QEMU to exit.
/// Stop the machine and wait for QEMU to exit.
///
/// This never panics and never fails: teardown has no useful recovery,
/// so a guest that does not power off cleanly is killed and the problem
/// is logged to stderr.
/// QEMU is killed rather than the guest powered off: the overlay is
/// thrown away, so a clean shutdown would only cost time. Copy out
/// anything the guest holds first. This never panics and never fails.
pub async fn down(self) {
self.vm.power_off_or_kill().await;
let Self { mut vm, .. } = self;
vm.kill().await;
}
}

Expand Down
8 changes: 5 additions & 3 deletions tests/tmachine/src/qemu/setup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use blake3::Hasher;
use crate::config::{Environment, Machine};

use super::ansible_hash::hash_sources;
use super::layer::{cached_layer, hash_file, hash_files, hash_inputs};
use super::layer::{SETUP_DISKS, cached_layer, hash_file, hash_files, hash_inputs};

const SETUP_CACHE_VERSION: &[u8] = b"tmachine-disk-blake3-v1";

Expand All @@ -18,8 +18,9 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result<PathB
return Ok(machine.base_image.clone());
}

let hash = setup_hash(machine, environment)?;
let hash = setup_layer_key(machine, environment)?;
cached_layer(
SETUP_DISKS,
&machine.base_image,
&hash,
environment.setup.use_galaxy,
Expand All @@ -29,7 +30,8 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result<PathB
.await
}

fn setup_hash(machine: &Machine, environment: &Environment) -> Result<String> {
/// The hash that names the environment's setup layer (`disks/<hash>.qcow2`).
pub fn setup_layer_key(machine: &Machine, environment: &Environment) -> Result<String> {
let mut hasher = Hasher::new();
hasher.update(SETUP_CACHE_VERSION);
hash_file(&mut hasher, &machine.base_image)
Expand Down
34 changes: 2 additions & 32 deletions tests/tmachine/src/qemu/vm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,10 @@ use super::img::QemuImage;
/// `TMACHINE_SSH_PORT` says otherwise.
const DEFAULT_SSH_PORT: u16 = 2222;

/// How long the QMP power-off request has to be acknowledged. A guest or QEMU
/// that stops answering must not hang teardown.
const QMP_TIMEOUT: Duration = Duration::from_secs(10);

/// How long a powered-off guest has to exit at teardown before QEMU is
/// killed.
const POWER_OFF_TIMEOUT: Duration = Duration::from_secs(120);

/// How long a guest being saved as a cached layer has to power off. A heavily
/// provisioned guest (Docker, k3s) can take minutes to stop its services, and
/// a layer that is killed instead is not cached, so this is much longer than
/// [`POWER_OFF_TIMEOUT`]. It still bounds a guest that never stops.
/// a layer that is killed instead is not cached. It still bounds a guest that
/// never stops.
pub(super) const LAYER_POWER_OFF_TIMEOUT: Duration = Duration::from_secs(600);

/// The host port forwarded to the guest's SSH server.
Expand Down Expand Up @@ -170,28 +162,6 @@ impl QemuVm {
}
}
}

/// Power the guest off and wait for QEMU to exit. Never panics: a guest
/// that can't be powered off cleanly is killed, and the problem is
/// logged. Use this when tearing down, where there is nothing left to do
/// about a failure.
pub(super) async fn power_off_or_kill(mut self) {
let shutdown = match tokio::time::timeout(QMP_TIMEOUT, self.shutdown()).await {
Ok(result) => result,
Err(_) => Err(anyhow::anyhow!("QMP did not answer within {QMP_TIMEOUT:?}")),
};
match shutdown {
Ok(()) => {
if let Err(error) = self.wait(POWER_OFF_TIMEOUT).await {
eprintln!("tmachine: {error:#}");
}
}
Err(error) => {
eprintln!("tmachine: could not power off the VM cleanly, killing it: {error:#}");
self.kill().await;
}
}
}
}

fn command(
Expand Down
Loading