From e21e443d4654be21a5a5837b4a9f48103d9c36bc Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 10 Nov 2026 14:39:49 +0100 Subject: [PATCH 1/2] test(runner): pass an explicit per-test timeout and grace to program rows Program rows relied on the program's 900 s default, so one hung test could use much of the 45-minute run timeout. Derive the per-test timeout from the same constants as the nextest profile (120 s) and pass it with a 10 s grace to the program. Part of #5416. Signed-off-by: Evan Lezar --- tests/runner/src/bundle.rs | 33 ++++++++++++++++++++++++++------- tests/runner/src/execute.rs | 18 ++++++++++++++++-- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/tests/runner/src/bundle.rs b/tests/runner/src/bundle.rs index 95d0b914ea..7942412b7e 100644 --- a/tests/runner/src/bundle.rs +++ b/tests/runner/src/bundle.rs @@ -31,17 +31,29 @@ pub const PROFILE: &str = "openshell"; /// The readiness check every bundle carries, at the bundle root. pub const READY_BINARY: &str = "testkit-ready"; -/// Nextest flags a test as slow after the period and terminates it after -/// `terminate-after` periods, so a hung test fails after two minutes instead -/// of holding the lane until the job times out. -const NEXTEST_CONFIG: &str = r#"# Written by the OpenShell test runner. +/// Nextest flags a test as slow after this period and terminates it after +/// [`SLOW_TERMINATE_AFTER`] periods. +const SLOW_PERIOD_SECS: u64 = 30; +const SLOW_TERMINATE_AFTER: u64 = 4; + +/// How long one test may run before the runner stops it, on either kind of +/// bundle, so a hung test fails after two minutes instead of holding the lane +/// until the job times out. +pub const TEST_TIMEOUT_SECS: u64 = SLOW_PERIOD_SECS * SLOW_TERMINATE_AFTER; + +/// The nextest configuration written into a nextest bundle. +fn nextest_config() -> String { + format!( + r#"# Written by the OpenShell test runner. [profile.openshell] fail-fast = false -slow-timeout = { period = "30s", terminate-after = 4 } +slow-timeout = {{ period = "{SLOW_PERIOD_SECS}s", terminate-after = {SLOW_TERMINATE_AFTER} }} [profile.openshell.junit] path = "junit.xml" -"#; +"# + ) +} /// The prebuilt program's name in a program bundle. pub const PROGRAM_BINARY: &str = "test-program"; @@ -204,7 +216,7 @@ pub fn build(root: &Path, family: &str, packages: &[String], out: &Path) -> Resu run(&mut command).with_context(|| format!("build the {} test archive", workspace.display()))?; copy_manifests(&workspace, out)?; - std::fs::write(out.join(CONFIG), NEXTEST_CONFIG)?; + std::fs::write(out.join(CONFIG), nextest_config())?; std::fs::copy(&ready, out.join(READY_BINARY)) .with_context(|| format!("copy {}", ready.display()))?; let manifest = Manifest { @@ -343,6 +355,13 @@ fn run(command: &mut Command) -> Result<()> { mod tests { use super::*; + #[test] + fn nextest_profile_stops_a_test_at_the_shared_timeout() { + let config = nextest_config(); + assert!(config.contains(r#"slow-timeout = { period = "30s", terminate-after = 4 }"#)); + assert_eq!(TEST_TIMEOUT_SECS, 120); + } + #[test] fn program_bundle_holds_only_the_program() { let dir = tempfile::tempdir().unwrap(); diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 2f477134f9..7a7aa62cc2 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -20,7 +20,7 @@ use anyhow::{Context, Result, ensure}; use tmachine::Session; use tmachine::config::Config; -use crate::bundle::{Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY}; +use crate::bundle::{Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY, TEST_TIMEOUT_SECS}; use crate::test_env; const GUEST_DIR: &str = "/var/tmp/openshell-tests"; @@ -42,6 +42,9 @@ const READY_SLACK: Duration = Duration::from_mins(1); const NEXTEST_TEST_FAILED: i32 = 100; /// nextest's exit code when `--no-tests fail` found no tests to run. const NEXTEST_NO_TESTS_RUN: i32 = 4; +/// How long a stopped test has between SIGTERM and SIGKILL. Matches nextest's +/// default, so both kinds of bundle stop a hung test the same way. +const TEST_GRACE_SECS: u64 = 10; /// A testkit program's exit code when a test failed. It exits 2 when tests /// could not run. const PROGRAM_TEST_FAILED: i32 = 1; @@ -308,6 +311,8 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { let archive = format!("{guest}/tests.tar.zst"); let extract = format!("{guest}/extract"); let config = format!("{guest}/nextest.toml"); + let timeout = TEST_TIMEOUT_SECS.to_string(); + let grace = TEST_GRACE_SECS.to_string(); let mut tests = vec!["env"]; tests.extend(env.iter().map(String::as_str)); match kind { @@ -342,7 +347,16 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { // The program runs its tests one at a time, and writes the report // where `collect_results` looks for it. Kind::Program => { - tests.extend([program.as_str(), "run", "--junit", &junit]); + tests.extend([ + program.as_str(), + "run", + "--timeout", + &timeout, + "--grace", + &grace, + "--junit", + &junit, + ]); // `run` takes name filters as arguments, after `--` so that one // starting with a dash isn't read as an option. if !run.filters.is_empty() { From accad2dad68ad1c2b23eb2c56fdf379d533255a5 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 14:33:30 -0800 Subject: [PATCH 2/2] test(runner): keep program rows on the program's default timeout Passing the runner's 120 s limit to the program would stop a test before testkit's own step timeouts (up to 10 minutes) can report which step stalled, contrary to the program's documented contract. Keep program rows on the program's default until testkit learns the per-test deadline (#5333). Write nextest's grace period into the generated profile from the same constants, rather than relying on nextest's default. Signed-off-by: Matthew Grossman --- tests/runner/src/bundle.rs | 19 +++++++++---------- tests/runner/src/execute.rs | 24 +++++------------------- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/tests/runner/src/bundle.rs b/tests/runner/src/bundle.rs index 7942412b7e..4f953f6a8e 100644 --- a/tests/runner/src/bundle.rs +++ b/tests/runner/src/bundle.rs @@ -32,14 +32,12 @@ pub const PROFILE: &str = "openshell"; pub const READY_BINARY: &str = "testkit-ready"; /// Nextest flags a test as slow after this period and terminates it after -/// [`SLOW_TERMINATE_AFTER`] periods. +/// [`SLOW_TERMINATE_AFTER`] periods, so a hung test fails after two minutes +/// instead of holding the lane until the job times out. const SLOW_PERIOD_SECS: u64 = 30; const SLOW_TERMINATE_AFTER: u64 = 4; - -/// How long one test may run before the runner stops it, on either kind of -/// bundle, so a hung test fails after two minutes instead of holding the lane -/// until the job times out. -pub const TEST_TIMEOUT_SECS: u64 = SLOW_PERIOD_SECS * SLOW_TERMINATE_AFTER; +/// How long a terminated test has between SIGTERM and SIGKILL. +const GRACE_PERIOD_SECS: u64 = 10; /// The nextest configuration written into a nextest bundle. fn nextest_config() -> String { @@ -47,7 +45,7 @@ fn nextest_config() -> String { r#"# Written by the OpenShell test runner. [profile.openshell] fail-fast = false -slow-timeout = {{ period = "{SLOW_PERIOD_SECS}s", terminate-after = {SLOW_TERMINATE_AFTER} }} +slow-timeout = {{ period = "{SLOW_PERIOD_SECS}s", terminate-after = {SLOW_TERMINATE_AFTER}, grace-period = "{GRACE_PERIOD_SECS}s" }} [profile.openshell.junit] path = "junit.xml" @@ -356,10 +354,11 @@ mod tests { use super::*; #[test] - fn nextest_profile_stops_a_test_at_the_shared_timeout() { + fn nextest_profile_stops_a_hung_test_after_two_minutes() { let config = nextest_config(); - assert!(config.contains(r#"slow-timeout = { period = "30s", terminate-after = 4 }"#)); - assert_eq!(TEST_TIMEOUT_SECS, 120); + assert!(config.contains( + r#"slow-timeout = { period = "30s", terminate-after = 4, grace-period = "10s" }"# + )); } #[test] diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 7a7aa62cc2..4ac41d743b 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -20,7 +20,7 @@ use anyhow::{Context, Result, ensure}; use tmachine::Session; use tmachine::config::Config; -use crate::bundle::{Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY, TEST_TIMEOUT_SECS}; +use crate::bundle::{Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY}; use crate::test_env; const GUEST_DIR: &str = "/var/tmp/openshell-tests"; @@ -31,9 +31,9 @@ const NEXTEST_LOG: &str = "nextest.log"; // Matches the installers' own wait: an HA k3s gateway booted from a cached // disk can take minutes to report healthy. const READY_TIMEOUT_SECS: u64 = 300; -/// How long the nextest run may take in total. Individual tests are bounded -/// by the nextest profile's slow-timeout; this catches a run that stops -/// making progress some other way. +/// How long the test run may take in total. Individual tests are bounded by +/// the nextest profile's slow-timeout, or by a program's own `run --timeout` +/// default; this catches a run that stops making progress some other way. const RUN_TIMEOUT: Duration = Duration::from_mins(45); /// Slack beyond the readiness check's own timeout for SSH and process start. const READY_SLACK: Duration = Duration::from_mins(1); @@ -42,9 +42,6 @@ const READY_SLACK: Duration = Duration::from_mins(1); const NEXTEST_TEST_FAILED: i32 = 100; /// nextest's exit code when `--no-tests fail` found no tests to run. const NEXTEST_NO_TESTS_RUN: i32 = 4; -/// How long a stopped test has between SIGTERM and SIGKILL. Matches nextest's -/// default, so both kinds of bundle stop a hung test the same way. -const TEST_GRACE_SECS: u64 = 10; /// A testkit program's exit code when a test failed. It exits 2 when tests /// could not run. const PROGRAM_TEST_FAILED: i32 = 1; @@ -311,8 +308,6 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { let archive = format!("{guest}/tests.tar.zst"); let extract = format!("{guest}/extract"); let config = format!("{guest}/nextest.toml"); - let timeout = TEST_TIMEOUT_SECS.to_string(); - let grace = TEST_GRACE_SECS.to_string(); let mut tests = vec!["env"]; tests.extend(env.iter().map(String::as_str)); match kind { @@ -347,16 +342,7 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { // The program runs its tests one at a time, and writes the report // where `collect_results` looks for it. Kind::Program => { - tests.extend([ - program.as_str(), - "run", - "--timeout", - &timeout, - "--grace", - &grace, - "--junit", - &junit, - ]); + tests.extend([program.as_str(), "run", "--junit", &junit]); // `run` takes name filters as arguments, after `--` so that one // starting with a dash isn't read as an option. if !run.filters.is_empty() {