diff --git a/.config/nextest.toml b/.config/nextest.toml index fef1880a35..636c10a725 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -10,10 +10,6 @@ failure-output = "immediate-final" slow-timeout = { period = "30s", terminate-after = 4 } [profile.e2e-kubernetes] -# Temporary waiver: port-forward reconnect can attach read-only while the old -# SSH session retains stdin ownership. Keep recovery coverage on other drivers. -# Remove this exclusion after resolving https://github.com/NVIDIA/OpenShell/issues/4445. -default-filter = "not test(/^canonical_main_connect_recovers_its_ssh_transport$/)" fail-fast = false status-level = "slow" final-status-level = "fail" diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 6496d910af..361737cb7a 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -99,6 +99,7 @@ jobs: {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"connect-ssh-supervision","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-userns-default","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/TESTING.md b/TESTING.md index dc3f7a2013..4e642f338e 100644 --- a/TESTING.md +++ b/TESTING.md @@ -650,6 +650,12 @@ rootful and rootless, and the `k3s` and `k3s-ha` installers. The tests run on the K3s node, so the K3s installers set the chart's `server.hostGatewayIP` to the node's address. +The `connect-ssh-supervision` feature suite +(`tests/suites/features/connect-ssh-supervision`) checks how `sandbox connect` +supervises its `ssh` process: it reconnects after losing the SSH transport and +forwards a SIGTERM to `ssh`. The tests find those processes in `/proc`, so they +need a Linux test host. CI runs them on Docker rootful. + Run the VM-backed Rust CLI e2e suite: ```shell diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index 4782ce04bc..17eb2b5e0c 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -389,12 +389,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - [[package]] name = "chacha20" version = "0.10.2" @@ -1855,18 +1849,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "nix" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - [[package]] name = "nom" version = "7.1.3" @@ -2031,7 +2013,6 @@ dependencies = [ "hyper", "hyper-util", "jsonwebtoken", - "nix", "noyalib", "openshell-crypto", "openshell-ocsf-validate", diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index e73648fd36..1e0e822236 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -283,7 +283,6 @@ tonic-prost = "0.14" tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] } tower = "0.5" url = "2" -nix = { version = "0.29", features = ["process", "signal", "term", "user"] } [dev-dependencies] flate2 = "1" diff --git a/e2e/rust/src/harness/binary.rs b/e2e/rust/src/harness/binary.rs index 4cf87c4c87..cd853906b9 100644 --- a/e2e/rust/src/harness/binary.rs +++ b/e2e/rust/src/harness/binary.rs @@ -52,41 +52,6 @@ pub fn openshell_cmd() -> tokio::process::Command { cmd } -fn shell_escape(arg: &str) -> String { - if arg - .chars() - .all(|c| c.is_ascii_alphanumeric() || "-_./:@".contains(c)) - { - return arg.to_string(); - } - - format!("'{}'", arg.replace('\'', "'\\''")) -} - -/// Create a [`tokio::process::Command`] that runs `openshell` under a PTY. -pub fn openshell_tty_cmd(args: &[&str]) -> tokio::process::Command { - let bin = openshell_bin(); - let mut cmd = tokio::process::Command::new("script"); - - if cfg!(target_os = "macos") { - cmd.arg("-q").arg("/dev/null").arg(bin).args(args); - } else { - let mut shell_command = shell_escape(bin.to_str().expect("openshell path is utf-8")); - for arg in args { - shell_command.push(' '); - shell_command.push_str(&shell_escape(arg)); - } - cmd.arg("-q") - .arg("-e") - .arg("-c") - .arg(shell_command) - .arg("/dev/null"); - } - - cmd.kill_on_drop(true); - cmd -} - #[cfg(test)] mod tests { use super::*; diff --git a/e2e/rust/tests/oidc_pkce.rs b/e2e/rust/tests/oidc_pkce.rs index 929f21b676..409d99bf30 100644 --- a/e2e/rust/tests/oidc_pkce.rs +++ b/e2e/rust/tests/oidc_pkce.rs @@ -1457,8 +1457,8 @@ async fn delete_workspace(admin: &LoginSession, workspace: &str) { } // Keep RBAC fixtures running until cleanup so authorization checks do not also -// exercise fast canonical-process completion. That lifecycle belongs to the -// dedicated sandbox_lifecycle suite. +// exercise fast canonical-process completion. The conformance `lifecycle` +// suite covers that. async fn assert_can_create_sandbox(session: &LoginSession, workspace: &str, sandbox_name: &str) { let create = run_workspace_cli( session, diff --git a/e2e/rust/tests/sandbox_lifecycle.rs b/e2e/rust/tests/sandbox_lifecycle.rs index 748ea40137..9f9010d949 100644 --- a/e2e/rust/tests/sandbox_lifecycle.rs +++ b/e2e/rust/tests/sandbox_lifecycle.rs @@ -3,21 +3,17 @@ #![cfg(feature = "e2e")] -#[cfg(target_os = "linux")] -use std::fs; -use std::process::Stdio; +//! The other tests in this file moved to the testkit conformance and +//! `features/connect-ssh-supervision` suites. This one stays until #5507 is fixed. + use std::time::Duration; -use openshell_e2e::harness::binary::{openshell_cmd, openshell_tty_cmd}; use openshell_e2e::harness::cli::{run_cli, wait_for_sandbox_exec_contains}; use openshell_e2e::harness::output::{extract_field, strip_ansi}; use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, SandboxGuard, unique_sandbox_name}; use serial_test::serial; -use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::time::{Instant, sleep}; -const SANDBOX_PRESENCE_TIMEOUT: Duration = Duration::from_secs(30); -const SANDBOX_LIST_POLL_INTERVAL: Duration = Duration::from_millis(500); const SANDBOX_RESTART_TIMEOUT: Duration = Duration::from_secs(60); fn normalize_output(output: &str) -> String { @@ -37,1058 +33,6 @@ fn normalize_output(output: &str) -> String { cleaned } -fn extract_sandbox_name(output: &str) -> Option { - if let Some((_, rest)) = output.split_once("Created sandbox:") { - return rest.split_whitespace().next().map(ToOwned::to_owned); - } - - extract_field(output, "Created sandbox").or_else(|| extract_field(output, "Name")) -} - -async fn sandbox_list_names(deadline: Instant) -> Option> { - if Instant::now() >= deadline { - return None; - } - - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "list", "--names"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = match tokio::time::timeout_at(deadline, cmd.output()).await { - Ok(output) => output.expect("spawn openshell sandbox list"), - Err(_) => return None, - }; - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - assert!( - output.status.success(), - "sandbox list should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - Some( - combined - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .map(ToOwned::to_owned) - .collect(), - ) -} - -async fn assert_sandbox_presence_eventually( - sandbox_name: &str, - should_exist: bool, -) -> Result<(), Vec> { - let deadline = Instant::now() + SANDBOX_PRESENCE_TIMEOUT; - let mut last_sandbox_names = Vec::new(); - - loop { - let Some(sandbox_names) = sandbox_list_names(deadline).await else { - return Err(last_sandbox_names); - }; - let exists = sandbox_names.iter().any(|name| name == sandbox_name); - if exists == should_exist { - return Ok(()); - } - - let now = Instant::now(); - if now >= deadline { - return Err(sandbox_names); - } - - last_sandbox_names = sandbox_names; - sleep(SANDBOX_LIST_POLL_INTERVAL.min(deadline - now)).await; - } -} - -async fn delete_sandbox(name: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "delete", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_exec_large_output_is_complete() { - const BYTES: usize = 8 * 1024 * 1024; - let mut sandbox = SandboxGuard::create(&[]) - .await - .expect("create sandbox for large output"); - - for stderr in [false, true] { - let script = if stderr { - format!("yes A | head -c {BYTES} >&2") - } else { - format!("yes A | head -c {BYTES}") - }; - let mut command = openshell_cmd(); - command - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "sh", - "-c", - &script, - ]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let output = tokio::time::timeout(Duration::from_secs(30), command.output()) - .await - .expect("large exec output timed out") - .expect("run large exec output"); - assert!(output.status.success(), "large exec exited unsuccessfully"); - let bytes = if stderr { output.stderr } else { output.stdout }; - assert_eq!(bytes.len(), BYTES, "exec output was truncated"); - assert!(bytes.chunks_exact(2).all(|pair| pair == b"A\n")); - } - - let mut early_exit = openshell_cmd(); - let mut early_child = early_exit - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "head", - "-n1", - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .expect("start early-exit exec"); - let mut input = b"first\n".to_vec(); - input.extend(vec![b'x'; 128 * 1024]); - let write_result = early_child - .stdin - .take() - .expect("early-exit stdin") - .write_all(&input) - .await; - if let Err(error) = write_result { - assert_eq!(error.kind(), std::io::ErrorKind::BrokenPipe); - } - let early_output = - tokio::time::timeout(Duration::from_secs(30), early_child.wait_with_output()) - .await - .expect("early-exit exec timed out") - .expect("wait for early-exit exec"); - assert!(early_output.status.success(), "early-exit exec failed"); - assert_eq!(early_output.stdout, b"first\n"); - - let mut command = openshell_cmd(); - let output = tokio::time::timeout( - Duration::from_secs(10), - command - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "sh", - "-c", - "sleep 5 & echo ok", - ]) - .stdin(Stdio::null()) - .output(), - ) - .await - .expect("exec with background pipe holder timed out") - .expect("run exec with background pipe holder"); - assert!(output.status.success(), "unexpected output failure"); - assert_eq!(output.stdout, b"ok\n"); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn piped_exec_stdin_crosses_grpc_message_limit() { - let mut sandbox = SandboxGuard::create(&[]) - .await - .expect("create sandbox for streamed stdin"); - - for size in [5, 1_048_576, 4_194_304] { - let mut command = openshell_cmd(); - command - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "wc", - "-c", - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut child = command.spawn().expect("spawn sandbox exec"); - let mut input = child.stdin.take().expect("piped stdin"); - input - .write_all(&vec![b'x'; size]) - .await - .expect("write piped stdin"); - drop(input); - let output = tokio::time::timeout(Duration::from_secs(30), child.wait_with_output()) - .await - .expect("streamed exec timed out") - .expect("wait for streamed exec"); - assert!( - output.status.success(), - "streamed exec failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - assert_eq!( - String::from_utf8_lossy(&output.stdout).trim(), - size.to_string() - ); - } - - let mut oversized = openshell_cmd(); - let mut oversized_child = oversized - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "wc", - "-c", - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .expect("spawn oversized stdin exec"); - oversized_child - .stdin - .take() - .expect("oversized stdin pipe") - .write_all(&vec![b'x'; 4_194_305]) - .await - .expect("write oversized stdin"); - let oversized_output = oversized_child - .wait_with_output() - .await - .expect("wait for oversized stdin error"); - assert!(!oversized_output.status.success()); - assert!(oversized_output.stdout.is_empty()); - assert!(String::from_utf8_lossy(&oversized_output.stderr).contains("4 MiB limit")); - - // A forced PTY with a small pipe must keep using the unary RPC for - // compatibility with older gateways. - let mut tty_command = openshell_cmd(); - let mut tty_child = tty_command - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--tty", - "--no-login-shell", - "--", - "sh", - "-c", - "printf tty-ok", - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .expect("spawn forced TTY exec"); - tty_child - .stdin - .take() - .expect("piped TTY stdin") - .write_all(b"x") - .await - .expect("write TTY stdin"); - let tty_output = tokio::time::timeout(Duration::from_secs(30), tty_child.wait_with_output()) - .await - .expect("forced TTY exec timed out") - .expect("wait for forced TTY exec"); - assert!( - tty_output.status.success(), - "forced TTY exec failed: {}", - String::from_utf8_lossy(&tty_output.stderr) - ); - assert!(String::from_utf8_lossy(&tty_output.stdout).contains("tty-ok")); - - #[cfg(unix)] - { - let directory = std::fs::File::open("/").expect("open directory as stdin"); - let mut command = openshell_cmd(); - let output = command - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "wc", - "-c", - ]) - .stdin(Stdio::from(directory)) - .output() - .await - .expect("run exec with unreadable stdin"); - assert!(!output.status.success(), "stdin read error was ignored"); - assert!(output.stdout.is_empty()); - } - - sandbox.cleanup().await; -} - -async fn reconnect_with_input_ownership( - sandbox_name: &str, -) -> (tokio::process::Child, Vec) { - let reconnect_deadline = Instant::now() + Duration::from_secs(30); - let mut attempt = 0; - - loop { - attempt += 1; - let token = format!("reconnect-{attempt}"); - let mut reconnect_cmd = openshell_cmd(); - reconnect_cmd - .args(["sandbox", "connect", sandbox_name]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut candidate = reconnect_cmd.spawn().expect("spawn reconnect attachment"); - let mut candidate_stdin = candidate.stdin.take().expect("reconnect stdin"); - candidate_stdin - .write_all(format!("{token}\n").as_bytes()) - .await - .expect("write reconnect ownership probe"); - candidate_stdin - .flush() - .await - .expect("flush reconnect ownership probe"); - - let reconnect_stdout = candidate.stdout.take().expect("reconnect stdout"); - let mut reconnect_lines = BufReader::new(reconnect_stdout).lines(); - let mut candidate_replay = Vec::new(); - let remaining = reconnect_deadline.saturating_duration_since(Instant::now()); - assert!( - !remaining.is_zero(), - "reconnect did not acquire input ownership" - ); - let attempt_timeout = Duration::from_secs(2).min(remaining); - let acquired = tokio::time::timeout(attempt_timeout, async { - loop { - let Some(line) = reconnect_lines - .next_line() - .await - .expect("read reconnect output") - else { - return false; - }; - if line.contains("main_pid=") { - candidate_replay.push(line.clone()); - } - if line.contains(&format!("input={token}")) { - return true; - } - } - }) - .await - .unwrap_or(false); - - if acquired { - return (candidate, candidate_replay); - } - - let _ = candidate.kill().await; - let _ = candidate.wait().await; - assert!( - Instant::now() < reconnect_deadline, - "reconnect did not acquire input ownership" - ); - sleep(Duration::from_millis(100)).await; - } -} - -#[cfg(target_os = "linux")] -fn find_process_with_args(expected_args: &[&str]) -> Option { - for entry in fs::read_dir("/proc").ok()?.filter_map(Result::ok) { - let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { - continue; - }; - let cmdline = fs::read(entry.path().join("cmdline")).unwrap_or_default(); - let args = cmdline.split(|byte| *byte == 0).collect::>(); - if expected_args - .iter() - .all(|expected| args.contains(&expected.as_bytes())) - { - return Some(pid); - } - } - None -} - -#[cfg(target_os = "linux")] -fn find_child_process_with_args(parent_pid: u32, expected_args: &[&str]) -> Option { - for entry in fs::read_dir("/proc").ok()?.filter_map(Result::ok) { - let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { - continue; - }; - let status = fs::read_to_string(entry.path().join("status")).unwrap_or_default(); - let process_parent = status.lines().find_map(|line| { - line.strip_prefix("PPid:") - .and_then(|value| value.trim().parse::().ok()) - }); - if process_parent != Some(parent_pid) { - continue; - } - let cmdline = fs::read(entry.path().join("cmdline")).unwrap_or_default(); - let args = cmdline.split(|byte| *byte == 0).collect::>(); - if expected_args - .iter() - .all(|expected| args.contains(&expected.as_bytes())) - { - return Some(pid); - } - } - None -} - -#[cfg(target_os = "linux")] -async fn wait_for_process_with_args(expected_args: &[&str]) -> u32 { - tokio::time::timeout(Duration::from_secs(10), async { - loop { - if let Some(pid) = find_process_with_args(expected_args) { - return pid; - } - sleep(Duration::from_millis(50)).await; - } - }) - .await - .unwrap_or_else(|_| panic!("process with arguments {expected_args:?} did not start")) -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_exit_zero_completes_persistent_sandbox() { - // Armed before create so a failed create or parse still cleans up. - let sandbox_name = unique_sandbox_name(); - let _cleanup = SandboxGuard::manage_existing(sandbox_name.clone()); - - let mut cmd = openshell_tty_cmd(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--", - "echo", - "OK", - ]); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!(output.status.success(), "create failed:\n{combined}"); - assert!( - combined.contains("OK"), - "main output was not streamed:\n{combined}" - ); - - if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox_name, true).await { - panic!( - "sandbox {sandbox_name} should still exist by default after {SANDBOX_PRESENCE_TIMEOUT:?}; \ - last observed sandbox list: {last_sandbox_list:?}" - ); - } - - let mut get_cmd = openshell_cmd(); - get_cmd - .args(["sandbox", "get", &sandbox_name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let get_output = get_cmd.output().await.expect("spawn openshell sandbox get"); - let details = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&get_output.stdout), - String::from_utf8_lossy(&get_output.stderr), - )); - assert!( - get_output.status.success(), - "sandbox get failed:\n{details}" - ); - assert!( - details.contains("Phase: Completed"), - "expected terminal sandbox phase:\n{details}" - ); -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_nonzero_exit_preserves_status() { - // Armed before create so a failed create or parse still cleans up. - let sandbox_name = unique_sandbox_name(); - let _cleanup = SandboxGuard::manage_existing(sandbox_name.clone()); - - let mut cmd = openshell_tty_cmd(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--", - "sh", - "-c", - "echo failed-main; exit 7", - ]); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let combined = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - assert_eq!( - output.status.code(), - Some(7), - "unexpected result:\n{combined}" - ); - assert!( - combined.contains("failed-main"), - "main output was not streamed:\n{combined}" - ); - - let mut get_cmd = openshell_cmd(); - get_cmd - .args(["sandbox", "get", &sandbox_name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let get_output = get_cmd.output().await.expect("spawn openshell sandbox get"); - let details = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&get_output.stdout), - String::from_utf8_lossy(&get_output.stderr), - )); - assert!( - details.contains("Phase: Error"), - "unexpected phase:\n{details}" - ); - assert!( - details.contains("Exit Code: 7"), - "missing exit code:\n{details}" - ); -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_disconnect_reconnect_replays_history_for_same_process() { - const FIRST_MARKER: &str = "sequence=0001"; - let script = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'main_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'main_pid=%s input=%s\n' "$$" "$line"; done"#; - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-lc", script]) - .await - .expect("create retained canonical main process"); - - let mut owner_cmd = openshell_cmd(); - owner_cmd - .args(["sandbox", "connect", &sandbox.name]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut owner = owner_cmd.spawn().expect("spawn input-owning attachment"); - let owner_stdout = owner.stdout.take().expect("owner stdout"); - let mut owner_lines = BufReader::new(owner_stdout).lines(); - let owner_line = tokio::time::timeout(Duration::from_secs(30), owner_lines.next_line()) - .await - .expect("owner output timeout") - .expect("read owner output") - .expect("owner output should remain open"); - assert!( - owner_line.contains("main_pid="), - "unexpected owner output: {owner_line}" - ); - - let mut observer_cmd = openshell_cmd(); - observer_cmd - .args(["sandbox", "connect", &sandbox.name]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut observer = observer_cmd.spawn().expect("spawn competing attachment"); - let observer_stderr = observer.stderr.take().expect("observer stderr"); - let mut observer_errors = BufReader::new(observer_stderr).lines(); - let warning = tokio::time::timeout(Duration::from_secs(30), observer_errors.next_line()) - .await - .expect("observer warning timeout") - .expect("read observer warning") - .expect("observer warning should remain open"); - assert!( - warning.contains("already has an input owner") && warning.contains("read-only"), - "competing attachment should become read-only: {warning}" - ); - let observer_stdout = observer.stdout.take().expect("observer stdout"); - let mut observer_lines = BufReader::new(observer_stdout).lines(); - let observer_output_line = - tokio::time::timeout(Duration::from_secs(30), observer_lines.next_line()) - .await - .expect("observer output timeout") - .expect("read observer output") - .expect("observer output should remain open"); - assert!( - observer_output_line.contains("main_pid="), - "read-only attachment should observe output: {observer_output_line}" - ); - - observer - .stdin - .as_mut() - .expect("observer stdin") - .write_all(b"\x03") - .await - .expect("send Ctrl-C to viewer"); - let observer_status = tokio::time::timeout(Duration::from_secs(30), observer.wait()) - .await - .expect("Ctrl-C should exit viewer") - .expect("wait for viewer"); - assert!(observer_status.success(), "viewer should exit successfully"); - - owner - .stdin - .as_mut() - .expect("owner stdin") - .write_all(b"owner-after-viewer-exit\n") - .await - .expect("send input after viewer exits"); - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let line = owner_lines - .next_line() - .await - .expect("read owner output after viewer exits") - .expect("main process should remain running"); - if line.contains("input=owner-after-viewer-exit") { - break; - } - } - }) - .await - .expect("owner should retain working stdin after viewer exits"); - - owner.kill().await.expect("disconnect input owner"); - owner.wait().await.expect("wait for input owner disconnect"); - - let (mut reconnect, replay) = reconnect_with_input_ownership(&sandbox.name).await; - - let first_pid = owner_line - .split_whitespace() - .find_map(|field| field.strip_prefix("main_pid=")) - .expect("owner output pid"); - assert!( - replay.iter().any(|line| line.contains(FIRST_MARKER)), - "reconnect should replay the beginning of retained history: {replay:?}" - ); - assert!( - replay - .iter() - .all(|line| line.contains(&format!("main_pid={first_pid}"))), - "reconnect should target the same canonical process: {replay:?}" - ); - assert!( - replay.iter().any(|line| !line.contains(FIRST_MARKER)), - "reconnect should observe output beyond the first record: {replay:?}" - ); - - reconnect - .kill() - .await - .expect("disconnect reconnect attachment"); - reconnect - .wait() - .await - .expect("wait for reconnect disconnect"); - sandbox.cleanup().await; -} - -#[cfg(target_os = "linux")] -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_connect_recovers_its_ssh_transport() { - let script = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'transport_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'transport_pid=%s input=%s\n' "$$" "$line"; done"#; - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-lc", script]) - .await - .expect("create retained canonical main process"); - - let mut connect_cmd = openshell_cmd(); - connect_cmd - .args(["sandbox", "connect", &sandbox.name]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut connect = connect_cmd.spawn().expect("spawn supervised attachment"); - let mut connect_stdin = connect.stdin.take().expect("connect stdin"); - let connect_stdout = connect.stdout.take().expect("connect stdout"); - let mut connect_lines = BufReader::new(connect_stdout).lines(); - let connect_stderr = connect.stderr.take().expect("connect stderr"); - let mut connect_errors = BufReader::new(connect_stderr).lines(); - - let initial_line = tokio::time::timeout(Duration::from_secs(30), connect_lines.next_line()) - .await - .expect("initial attachment output timeout") - .expect("read initial attachment output") - .expect("initial attachment output should remain open"); - assert!( - initial_line.contains("transport_pid="), - "unexpected initial attachment output: {initial_line}" - ); - - // Recovery intentionally starts only for an established attachment, so - // let the initial SSH process live beyond that setup guard before killing - // only its ProxyCommand transport. - sleep(Duration::from_secs(3)).await; - let proxy_pid = wait_for_process_with_args(&["ssh-proxy", "--sandbox", &sandbox.name]).await; - let kill_status = tokio::process::Command::new("kill") - .args(["-TERM", &proxy_pid.to_string()]) - .status() - .await - .expect("terminate SSH proxy transport"); - assert!(kill_status.success(), "terminate SSH proxy transport"); - - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let line = connect_errors - .next_line() - .await - .expect("read supervised attachment diagnostics") - .expect("supervised attachment diagnostics should remain open"); - if normalize_output(&line).contains("Connection to sandbox lost; reconnecting") { - break; - } - } - }) - .await - .expect("supervised attachment did not enter recovery"); - - let input_token = format!("after-transport-recovery-{:x}", rand::random::()); - connect_stdin - .write_all(format!("{input_token}\n").as_bytes()) - .await - .expect("write input after transport recovery"); - connect_stdin - .flush() - .await - .expect("flush input after transport recovery"); - - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let line = connect_lines - .next_line() - .await - .expect("read recovered attachment output") - .expect("recovered attachment output should remain open"); - if line.contains(&format!("input={input_token}")) { - break; - } - } - }) - .await - .expect("replacement SSH session did not reattach to canonical main"); - assert!( - connect - .try_wait() - .expect("inspect supervised attachment") - .is_none(), - "sandbox connect parent should remain alive after recovery" - ); - - connect - .kill() - .await - .expect("disconnect recovered attachment"); - connect - .wait() - .await - .expect("wait for recovered attachment disconnect"); - sandbox.cleanup().await; -} - -#[cfg(target_os = "linux")] -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_connect_forwards_pid_targeted_termination_and_reaps_ssh() { - use std::os::fd::OwnedFd; - - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-lc", "exec sleep infinity"]) - .await - .expect("create retained canonical main process"); - let pty = nix::pty::openpty(None, None).expect("open pseudo-terminal"); - let controller: OwnedFd = pty.master; - let follower: OwnedFd = pty.slave; - - let mut connect_cmd = openshell_cmd(); - connect_cmd - .args(["sandbox", "connect", &sandbox.name]) - .stdin( - follower - .try_clone() - .expect("duplicate PTY follower for stdin"), - ) - .stdout( - follower - .try_clone() - .expect("duplicate PTY follower for stdout"), - ) - .stderr( - follower - .try_clone() - .expect("duplicate PTY follower for stderr"), - ); - let mut connect = connect_cmd - .spawn() - .expect("spawn supervised PTY attachment"); - let connect_pid = connect.id().expect("connect process ID"); - drop(follower); - - let ssh_pid = tokio::time::timeout(Duration::from_secs(30), async { - loop { - if let Some(pid) = - find_child_process_with_args(connect_pid, &["-s", "sandbox", "openshell-main"]) - { - return pid; - } - sleep(Duration::from_millis(50)).await; - } - }) - .await - .expect("supervised SSH child did not start"); - - nix::sys::signal::kill( - nix::unistd::Pid::from_raw(i32::try_from(connect_pid).expect("connect PID fits i32")), - nix::sys::signal::Signal::SIGTERM, - ) - .expect("send SIGTERM to only the OpenShell parent"); - - let status = tokio::time::timeout(Duration::from_secs(10), connect.wait()) - .await - .expect("OpenShell parent did not terminate") - .expect("wait for OpenShell parent"); - assert_eq!(status.code(), Some(143)); - tokio::time::timeout(Duration::from_secs(5), async { - while fs::metadata(format!("/proc/{ssh_pid}")).is_ok() { - sleep(Duration::from_millis(25)).await; - } - }) - .await - .expect("SSH child was not reaped after parent termination"); - - drop(controller); - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_exit_255_is_not_retried_as_transport_failure() { - const READY_MARKER: &str = "exit-255-ready"; - const RELEASE_PATH: &str = "/sandbox/.openshell-exit-255-release"; - let script = format!( - "echo {READY_MARKER}; while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 255" - ); - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script]) - .await - .expect("create retained canonical main process"); - - let mut connect_cmd = openshell_cmd(); - connect_cmd - .args(["sandbox", "connect", &sandbox.name]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut connect = connect_cmd.spawn().expect("spawn supervised attachment"); - let connect_stdout = connect.stdout.take().expect("connect stdout"); - let mut connect_lines = BufReader::new(connect_stdout).lines(); - - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let line = connect_lines - .next_line() - .await - .expect("read attachment output") - .expect("attachment output should remain open"); - if line.contains(READY_MARKER) { - break; - } - } - }) - .await - .expect("attachment did not observe the canonical main process"); - - // Keep the attachment alive beyond the setup guard used to distinguish - // initial SSH failures from established transport failures. - sleep(Duration::from_secs(3)).await; - sandbox - .exec(&["touch", RELEASE_PATH]) - .await - .expect("release canonical main process"); - - let status = tokio::time::timeout(Duration::from_secs(10), connect.wait()).await; - if status.is_err() { - connect.kill().await.expect("stop stuck attachment"); - } - sandbox.cleanup().await; - let status = status - .expect("exit status 255 must not enter the transport recovery loop") - .expect("wait for canonical main attachment"); - assert_eq!(status.code(), Some(255)); -} - -#[tokio::test] -async fn sandbox_create_with_no_keep_cleans_up_after_tty_command() { - let name = format!("tty-{:015x}", rand::random::() & 0x0fff_ffff_ffff_ffff); - // Capture startup diagnostics before --no-keep removes a failed container. - // This is best-effort: the lifecycle assertions also run on other drivers. - let log_name = name.clone(); - let diagnostics = tokio::spawn(async move { - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let containers = tokio::process::Command::new("docker") - .args(["ps", "--all", "--quiet", "--filter"]) - .arg(format!("label=openshell.ai/sandbox-name={log_name}")) - .kill_on_drop(true) - .output() - .await - .ok()?; - if !containers.status.success() { - return None; - } - let ids = String::from_utf8_lossy(&containers.stdout); - if let Some(id) = ids.split_whitespace().next() { - let logs = tokio::process::Command::new("docker") - .args(["logs", "--follow", id]) - .kill_on_drop(true) - .output() - .await - .ok()?; - return Some(normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&logs.stdout), - String::from_utf8_lossy(&logs.stderr) - ))); - } - sleep(Duration::from_millis(100)).await; - } - }) - .await - .ok() - .flatten() - }); - let mut cmd = openshell_tty_cmd(&[ - "sandbox", - "create", - "--name", - &name, - "--no-keep", - "--", - "echo", - "OK", - ]); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - let startup_logs = if output.status.success() { - diagnostics.abort(); - None - } else { - diagnostics.await.ok().flatten() - }; - assert!( - output.status.success(), - "create failed:\n{combined}\nsupervisor logs:\n{}", - startup_logs.as_deref().unwrap_or("unavailable") - ); - assert!( - combined.contains("OK"), - "main output was not streamed:\n{combined}" - ); - let sandbox_name = - extract_sandbox_name(&combined).expect("sandbox name should be present in output"); - - if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox_name, false).await { - delete_sandbox(&sandbox_name).await; - panic!( - "sandbox {sandbox_name} should have been deleted automatically after \ - {SANDBOX_PRESENCE_TIMEOUT:?}; last observed sandbox list: {last_sandbox_list:?}" - ); - } -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_create_with_no_keep_preserves_failure_then_cleans_up() { - let mut cmd = openshell_tty_cmd(&[ - "sandbox", - "create", - "--no-keep", - "--", - "sh", - "-c", - "echo ephemeral-failure; exit 17", - ]); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let combined = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - - assert_eq!( - output.status.code(), - Some(17), - "ephemeral main should preserve its failure code:\n{combined}" - ); - assert!( - combined.contains("ephemeral-failure"), - "ephemeral main output was not streamed:\n{combined}" - ); - let sandbox_name = - extract_sandbox_name(&combined).expect("sandbox name should be present in output"); - - if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox_name, false).await { - delete_sandbox(&sandbox_name).await; - panic!( - "failed ephemeral sandbox {sandbox_name} should have been deleted automatically after \ - {SANDBOX_PRESENCE_TIMEOUT:?}; last observed sandbox list: {last_sandbox_list:?}" - ); - } -} - async fn sandbox_details(name: &str) -> String { let (details, exit_code) = run_cli(&["sandbox", "get", name]).await; let details = normalize_output(&details); diff --git a/e2e/rust/tests/ssh_host_identity.rs b/e2e/rust/tests/ssh_host_identity.rs deleted file mode 100644 index c348c2b3f4..0000000000 --- a/e2e/rust/tests/ssh_host_identity.rs +++ /dev/null @@ -1,156 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -use std::path::Path; -use std::time::Duration; - -use openshell_e2e::harness::cli::{ - run_cli, sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains, - wait_for_sandbox_phase, -}; -use openshell_e2e::harness::gateway::ManagedGateway; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; - -async fn fingerprint(name: &str) -> String { - let (output, code) = run_cli(&["sandbox", "get", name, "--output", "json"]).await; - assert_eq!(code, 0, "sandbox get failed: {output}"); - let sandbox: serde_json::Value = serde_json::from_str(&strip_ansi(&output)).unwrap(); - let fingerprint = sandbox["host_key_fingerprint"] - .as_str() - .unwrap() - .to_string(); - assert!(fingerprint.starts_with("SHA256:")); - let (output, code) = run_cli(&["sandbox", "list", "--output", "json"]).await; - assert_eq!(code, 0, "sandbox list failed: {output}"); - let list: serde_json::Value = serde_json::from_str(&strip_ansi(&output)).unwrap(); - let listed = list["sandboxes"] - .as_array() - .unwrap() - .iter() - .find(|sandbox| sandbox["name"] == name) - .unwrap(); - assert_eq!(listed["host_key_fingerprint"], fingerprint); - fingerprint -} - -async fn pinned_ssh(config: &Path, known_hosts: &Path, name: &str, first: bool) { - let output = tokio::time::timeout( - Duration::from_secs(30), - tokio::process::Command::new("ssh") - .kill_on_drop(true) - .arg("-F") - .arg(config) - .arg("-o") - .arg(if first { - "StrictHostKeyChecking=accept-new" - } else { - "StrictHostKeyChecking=yes" - }) - .arg("-o") - .arg(format!("UserKnownHostsFile={}", known_hosts.display())) - .arg("-o") - .arg("BatchMode=yes") - .arg(format!("openshell-{name}.default")) - .args(["printf", "ssh-identity-ok"]) - .output(), - ) - .await - .unwrap() - .unwrap(); - assert!( - output.status.success(), - "pinned SSH failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - assert_eq!(String::from_utf8_lossy(&output.stdout), "ssh-identity-ok"); -} - -#[tokio::test] -async fn ssh_host_identity_survives_restarts_and_changes_after_recreation() { - let mut sandbox = SandboxGuard::create(&[]).await.unwrap(); - let name = sandbox.name.clone(); - let original = fingerprint(&name).await; - let directory = tempfile::tempdir().unwrap(); - let config = directory.path().join("ssh_config"); - let known_hosts = directory.path().join("known_hosts"); - let (output, code) = run_cli(&["sandbox", "ssh-config", &name]).await; - assert_eq!(code, 0, "SSH config failed: {output}"); - std::fs::write(&config, output).unwrap(); - pinned_ssh(&config, &known_hosts, &name, true).await; - let key = tokio::process::Command::new("ssh-keygen") - .args(["-l", "-E", "sha256", "-f"]) - .arg(&known_hosts) - .output() - .await - .unwrap(); - assert!(key.status.success()); - assert!( - String::from_utf8_lossy(&key.stdout) - .split_whitespace() - .any(|word| word == original) - ); - sandbox - .exec(&["sh", "-c", "test ! -r /.openshell/supervisor/auth.json"]) - .await - .unwrap(); - - let (output, code) = run_cli(&["sandbox", "stop", &name]).await; - assert_eq!(code, 0, "stop failed: {output}"); - wait_for_sandbox_phase(&name, "Stopped", Duration::from_secs(60)) - .await - .unwrap(); - assert_eq!(fingerprint(&name).await, original); - let (output, code) = run_cli(&["sandbox", "start", &name]).await; - assert_eq!(code, 0, "start failed: {output}"); - wait_for_sandbox_phase(&name, "Ready", Duration::from_secs(120)) - .await - .unwrap(); - assert_eq!(fingerprint(&name).await, original); - pinned_ssh(&config, &known_hosts, &name, false).await; - - if let Some(gateway) = ManagedGateway::from_env().unwrap() { - gateway.stop().unwrap(); - gateway.start().unwrap(); - wait_for_healthy(Duration::from_secs(120)).await.unwrap(); - wait_for_sandbox_phase(&name, "Ready", Duration::from_secs(120)) - .await - .unwrap(); - wait_for_sandbox_exec_contains( - &name, - &["printf", "ssh-identity-ready"], - "ssh-identity-ready", - Duration::from_secs(240), - ) - .await - .unwrap(); - assert_eq!(fingerprint(&name).await, original); - pinned_ssh(&config, &known_hosts, &name, false).await; - } else { - eprintln!("Skipping gateway restart check: this run uses an existing gateway"); - } - - let (output, code) = run_cli(&["sandbox", "delete", &name]).await; - assert_eq!(code, 0, "delete failed: {output}"); - // Delete may return before the driver finishes removing the sandbox. - // Its name remains reserved until the gateway confirms removal. - tokio::time::timeout(Duration::from_secs(120), async { - loop { - let names = sandbox_names() - .await - .expect("list sandboxes after deletion"); - if !names.iter().any(|listed| listed == &name) { - break; - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - }) - .await - .expect("original sandbox was not fully deleted before name reuse"); - sandbox.cleanup().await; - let mut replacement = SandboxGuard::create(&["--name", &name]).await.unwrap(); - assert_ne!(fingerprint(&name).await, original); - replacement.cleanup().await; -} diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index 9715526af2..f75fa3aeed 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -26,6 +26,6 @@ openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-testkit = { path = "../../testkit/testkit" } serde_json = "1" tempfile = "3" -tokio = { version = "1", features = ["time"] } +tokio = { version = "1", features = ["process", "time"] } [workspace] diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index a6edb02a7c..0639d70d52 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -10,7 +10,10 @@ nextest run the tests as the library's unit tests. The crate also builds the |---|---| | `smoke` | Create a sandbox, run `echo` in it, delete it | | `lifecycle` | SDK: long-running and run-to-completion sandboxes, prompt delete when the workload is slow to handle TERM, the default command, restart of a stopped sandbox, deleting a stopped sandbox, exec past the startup recovery deadline, the final phase when the main command exits, the main command's environment, `--no-keep` cleanup | -| `cli` | The same lifecycle through the `openshell` CLI: status, create, get, list, exec, stop, start, delete | +| `cli` | The same lifecycle through the `openshell` CLI: status, create, get, list, exec, stop, start, delete; attached `sandbox create -- ` streams the main process and reports its exit code; `--no-keep` deletes the sandbox afterwards | +| `exec-cli` | `sandbox exec` through the CLI: complete large output, piped stdin up to the 4 MiB limit, early exit, a forced TTY, unreadable stdin | +| `attach` | `sandbox connect` to the main process: one input owner, read-only viewers, history replay on reconnect, stop and start while attached, a main exit of 255 | +| `ssh` | The SSH host key: the same in get, list, and `ssh`; kept across stop and start; new for a recreated sandbox | | `persistent-workspace` | Files in `/sandbox` survive stop and start | | `policy-advisor` | `policy.local` accepts a proposed rule; a denied address or hostname yields one draft rule | | `file-transfer` | `sandbox upload` and `download`: round trips, file modes and symlinks outside the workspace, `.gitignore` filtering, workspace-escape and option-looking names, a large upload | @@ -31,7 +34,8 @@ tests, so a test environment needs it installed environments). `credential-gating`, ported from NVIDIA/OpenShell#5264, starts servers (`tests/support/host-server`) on the machine that runs the tests and reaches them from sandboxes created from `testkit::python_image()` through -`host.openshell.internal`. +`host.openshell.internal`. `ssh` runs `ssh` and `ssh-keygen` there and needs a named +gateway, which `sandbox ssh-config` writes into its `ProxyCommand`. ## Running it diff --git a/tests/suites/conformance/src/attach.rs b/tests/suites/conformance/src/attach.rs new file mode 100644 index 0000000000..06f136be2c --- /dev/null +++ b/tests/suites/conformance/src/attach.rs @@ -0,0 +1,195 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Attaching to a sandbox's main process with `sandbox connect`: one input +//! owner, read-only viewers, retained history on reconnect, and the main +//! process's exit code. +//! +//! Ported from `e2e/rust/tests/sandbox_lifecycle.rs`. The CLI-side transport +//! recovery and signal tests are in `tests/suites/features/connect-ssh-supervision`. + +use std::process::Stdio; +use std::time::Duration; + +use testkit::sdk::SandboxPhase; +use testkit::{CliChild, Context, ResultExt as _}; + +/// How long an attachment may take to show output. +const ATTACH_TIMEOUT: Duration = Duration::from_secs(30); +/// How long the CLI treats a new attachment as still connecting. A failure +/// after that is a transport failure it may recover from. +const ESTABLISHED: Duration = Duration::from_secs(3); + +/// A main process that prints a numbered line every 0.2 seconds and echoes +/// each line of its input, each with its PID. +const ECHO_MAIN: &str = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'main_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'main_pid=%s input=%s\n' "$$" "$line"; done"#; +const FIRST_RECORD: &str = "sequence=0001"; + +/// The first `sandbox connect` owns the main process's input. A second one +/// is told it is read-only, still sees output, and exits 0 on Ctrl-C without +/// taking input from the owner. After the owner disconnects, a new +/// attachment takes over input and replays the retained output from the +/// first record, all from the same process. Stopping and starting the +/// sandbox works while it is attached. +#[testkit::test] +async fn attach_reconnect_replays_history_for_the_same_main_process( + cx: &mut Context, +) -> testkit::Result { + let sandbox = cx + .sandbox("rc") + .command(["sh", "-lc", ECHO_MAIN]) + .create_ready() + .await?; + let connect = ["sandbox", "connect", sandbox.name()]; + + let mut owner = cx.cli_spawn(connect)?; + let owner_line = owner.stdout_line(ATTACH_TIMEOUT).await?; + let main_pid = owner_line + .split_whitespace() + .find_map(|field| field.strip_prefix("main_pid=")) + .ok_or_else(|| testkit::Error::msg(format!("unexpected owner output {owner_line:?}")))? + .to_string(); + + let mut viewer = cx.cli_spawn(connect)?; + let warning = viewer + .stderr_until(ATTACH_TIMEOUT, |line| { + line.contains("already has an input owner") + }) + .await? + .pop() + .unwrap_or_default(); + testkit::ensure!( + warning.contains("read-only"), + "a second attachment was not told it is read-only: {warning:?}" + ); + let viewer_line = viewer.stdout_line(ATTACH_TIMEOUT).await?; + testkit::ensure!( + viewer_line.contains("main_pid="), + "a read-only attachment did not see output: {viewer_line:?}" + ); + viewer + .write_stdin(b"\x03") + .await + .context("send Ctrl-C to the viewer")?; + let code = viewer.wait(ATTACH_TIMEOUT).await?; + testkit::ensure!(code == 0, "the viewer exited with {code} on Ctrl-C"); + + owner + .write_stdin(b"owner-after-viewer-exit\n") + .await + .context("send input after the viewer exits")?; + owner + .stdout_until(ATTACH_TIMEOUT, |line| { + line.contains("input=owner-after-viewer-exit") + }) + .await + .context("the owner keeps its input after the viewer exits")?; + owner.kill().await?; + + let (mut reconnect, replay) = reconnect_with_input(cx, sandbox.name()).await?; + testkit::ensure!( + replay.iter().any(|line| line.contains(FIRST_RECORD)), + "reconnect did not replay the retained history from {FIRST_RECORD}: {replay:?}" + ); + testkit::ensure!( + replay + .iter() + .all(|line| line.contains(&format!("main_pid={main_pid}"))), + "reconnect shows a process other than {main_pid}: {replay:?}" + ); + testkit::ensure!( + replay.iter().any(|line| !line.contains(FIRST_RECORD)), + "reconnect shows nothing beyond the first record: {replay:?}" + ); + + sandbox.stop().await?; + testkit::ensure!( + sandbox.get().await?.phase == SandboxPhase::Stopped, + "sandbox {} is not Stopped", + sandbox.name() + ); + sandbox.start().await?; + reconnect.kill().await +} + +/// A main process that exits 255 after the attachment is established ends +/// `sandbox connect` with 255. The CLI doesn't mistake it for a lost SSH +/// transport, which also exits 255, and reconnect. +#[testkit::test] +async fn attach_main_exit_255_is_not_retried_as_transport_failure( + cx: &mut Context, +) -> testkit::Result { + const READY: &str = "exit-255-ready"; + const RELEASE: &str = "/sandbox/.openshell-exit-255-release"; + let script = format!("echo {READY}; while [ ! -e '{RELEASE}' ]; do sleep 0.05; done; exit 255"); + let sandbox = cx + .sandbox("x255") + .command(["sh", "-c", &script]) + .create_ready() + .await?; + + let mut connect = + cx.cli_spawn_with_stdin(["sandbox", "connect", sandbox.name()], Stdio::null())?; + connect + .stdout_until(ATTACH_TIMEOUT, |line| line.contains(READY)) + .await?; + // Stay attached past the CLI's setup window, so a 255 would count as an + // established transport failure. + tokio::time::sleep(ESTABLISHED).await; + let release = sandbox.exec(["touch", RELEASE]).await?; + testkit::ensure!( + release.exit_code == 0, + "release the main process: {}", + release.stderr.trim() + ); + + let code = connect + .wait(Duration::from_secs(10)) + .await + .context("a main exit of 255 must not start transport recovery")?; + testkit::ensure!( + code == 255, + "`{}` exited with {code}; expected the main process's 255", + connect.command() + ); + Ok(()) +} + +/// Attach until one attachment takes over input, which it can only do once +/// the gateway has released the previous owner. Returns the attachment and +/// the output lines it showed up to its own input. +async fn reconnect_with_input( + cx: &Context, + name: &str, +) -> testkit::Result<(CliChild, Vec)> { + const DEADLINE: Duration = Duration::from_secs(30); + let deadline = tokio::time::Instant::now() + DEADLINE; + for attempt in 1.. { + let token = format!("reconnect-{attempt}"); + let mut candidate = cx.cli_spawn(["sandbox", "connect", name])?; + candidate + .write_stdin(format!("{token}\n").as_bytes()) + .await + .context("send reconnect input")?; + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + let read = candidate + .stdout_until(Duration::from_secs(2).min(remaining), |line| { + line.contains(&format!("input={token}")) + }) + .await; + if let Ok(lines) = read { + let replay = lines + .into_iter() + .filter(|line| line.contains("main_pid=")) + .collect(); + return Ok((candidate, replay)); + } + candidate.kill().await?; + testkit::ensure!( + tokio::time::Instant::now() < deadline, + "no reconnect to sandbox {name} took over input within {DEADLINE:?}" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + unreachable!("the attempts are unbounded") +} diff --git a/tests/suites/conformance/src/cli.rs b/tests/suites/conformance/src/cli.rs index f06f16d2d0..ed18f5a212 100644 --- a/tests/suites/conformance/src/cli.rs +++ b/tests/suites/conformance/src/cli.rs @@ -16,6 +16,17 @@ const PAGE_SIZE: &str = "1000"; /// looping forever. const MAX_PAGES: usize = 1000; +/// Delay before an attached main prints and exits, so the CLI attaches while +/// the sandbox is Ready. On a multi-replica gateway, attaching after the main +/// has exited fails unless the request reaches the replica holding the +/// supervisor session (#4949). +const ATTACH_BEFORE_EXIT: &str = "sleep 5"; + +/// The flag that runs the main without or with a terminal, and a name suffix +/// for it. With `--tty` the main gets a PTY and the CLI attaches with +/// `ssh -tt`, as it does when run from a terminal. +const TERMINAL_MODES: [(&str, &str); 2] = [("--no-tty", ""), ("--tty", "t")]; + /// `openshell status` succeeds, and a sandbox created with the CLI is /// retrievable and Ready, appears in `sandbox list`, runs `sandbox exec`, and is /// gone from `sandbox list` after `sandbox delete`. @@ -58,7 +69,7 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { let sandbox = cx .cli_sandbox("cl") - .command(["sh", "-c", "exec sleep infinity"]) + .command(["sleep", "infinity"]) .create_ready() .await?; let name = sandbox.name().to_string(); @@ -83,6 +94,117 @@ async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { sandbox.expect_absent().await } +/// `sandbox create -- ` without `--detach` streams the main +/// process's output and exits with its exit code, with and without a terminal +/// for the main. The sandbox is kept by default: one whose main exited 0 is +/// listed and Completed, and one whose main exited 7 is in Error with exit +/// code 7. +#[testkit::test] +async fn cli_attached_create_streams_main_and_reports_exit(cx: &mut Context) -> testkit::Result { + for (terminal, suffix) in TERMINAL_MODES { + let completed = cx.name(&format!("ac0{suffix}")); + cx.track(&completed); + let script = format!("{ATTACH_BEFORE_EXIT}; echo OK"); + let output = cx + .cli([ + "sandbox", "create", "--name", &completed, terminal, "--", "sh", "-c", &script, + ]) + .await? + .success()?; + expect_streamed(&output, "OK")?; + testkit::ensure!( + find_listed(cx, &completed).await?.is_some(), + "sandbox {completed} is not listed after its main exited; sandboxes are kept by default" + ); + // The phase can trail the CLI's exit by a status update. + let sandbox = cx.adopt(&completed).await?; + sandbox.wait_finished().await?; + let got = sandbox.get_json().await?; + testkit::ensure!( + got["phase"] == "Completed", + "sandbox {completed} is in phase {}; expected Completed", + got["phase"] + ); + + let failed = cx.name(&format!("ac7{suffix}")); + cx.track(&failed); + let script = format!("{ATTACH_BEFORE_EXIT}; echo failed-main; exit 7"); + let output = cx + .cli([ + "sandbox", "create", "--name", &failed, terminal, "--", "sh", "-c", &script, + ]) + .await?; + testkit::ensure!( + output.exit_code == 7, + "`{}` exited with {}; expected the main process's 7; stderr: {}", + output.command, + output.exit_code, + output.stderr.trim() + ); + expect_streamed(&output, "failed-main")?; + let sandbox = cx.adopt(&failed).await?; + sandbox.wait_finished().await?; + let got = sandbox.get_json().await?; + testkit::ensure!( + got["phase"] == "Error" && got["exit_code"] == 7, + "sandbox {failed} is in phase {} with exit code {}; expected Error with 7", + got["phase"], + got["exit_code"] + ); + } + Ok(()) +} + +/// `sandbox create --no-keep -- ` streams the main process's output, +/// exits with its exit code, and then deletes the sandbox, whether the main +/// exited 0 or 17, with and without a terminal for the main. +#[testkit::test] +async fn cli_no_keep_removes_sandbox_after_main_exit(cx: &mut Context) -> testkit::Result { + for (terminal, terminal_suffix) in TERMINAL_MODES { + for (suffix, code, marker) in [("nk0", 0, "OK"), ("nk17", 17, "ephemeral-failure")] { + let name = cx.name(&format!("{suffix}{terminal_suffix}")); + cx.track(&name); + let script = format!("{ATTACH_BEFORE_EXIT}; echo {marker}; exit {code}"); + let output = cx + .cli([ + "sandbox", + "create", + "--name", + &name, + "--no-keep", + terminal, + "--", + "sh", + "-c", + &script, + ]) + .await?; + testkit::ensure!( + output.exit_code == code, + "`{}` exited with {}; expected the main process's {code}; stderr: {}", + output.command, + output.exit_code, + output.stderr.trim() + ); + expect_streamed(&output, marker)?; + wait_unlisted(cx, &name).await?; + } + } + Ok(()) +} + +/// Fail unless the CLI printed `marker` from the main process. +fn expect_streamed(output: &testkit::CliOutput, marker: &str) -> testkit::Result { + testkit::ensure!( + output.stdout.contains(marker) || output.stderr.contains(marker), + "`{}` did not stream {marker:?} from the main process; stdout: {}; stderr: {}", + output.command, + output.stdout.trim(), + output.stderr.trim() + ); + Ok(()) +} + /// Wait until `sandbox list` no longer shows the sandbox. async fn wait_unlisted(cx: &Context, name: &str) -> testkit::Result { testkit::poll_until( diff --git a/tests/suites/conformance/src/exec_cli.rs b/tests/suites/conformance/src/exec_cli.rs new file mode 100644 index 0000000000..e854277abc --- /dev/null +++ b/tests/suites/conformance/src/exec_cli.rs @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `sandbox exec` through the `openshell` CLI: large output, piped stdin, and +//! early exit. +//! +//! Ported from `e2e/rust/tests/sandbox_lifecycle.rs`. + +use std::time::Duration; + +use testkit::{Context, ResultExt as _, Sandbox}; + +/// How long one exec of a few MiB may take. +const EXEC_TIMEOUT: Duration = Duration::from_secs(30); +/// The largest stdin a non-interactive exec accepts. +const STDIN_LIMIT: usize = 4 * 1024 * 1024; + +/// `sandbox exec --no-tty` returns all of a large stdout and stderr, exits 0 +/// when the command exits before reading all its stdin, and doesn't wait for +/// a background process that still holds the output pipe. +#[testkit::test] +async fn cli_exec_large_output_is_complete(cx: &mut Context) -> testkit::Result { + const BYTES: usize = 8 * 1024 * 1024; + let sandbox = cx + .sandbox("lo") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + + for stream in ["stdout", "stderr"] { + let script = match stream { + "stdout" => format!("yes A | head -c {BYTES}"), + _ => format!("yes A | head -c {BYTES} >&2"), + }; + let output = tokio::time::timeout( + EXEC_TIMEOUT, + cx.cli(exec_args(&sandbox, &["sh", "-c", &script])), + ) + .await + .map_err(|_| testkit::Error::msg(format!("exec of 8 MiB on {stream} timed out")))?? + .success()?; + let bytes = match stream { + "stdout" => output.stdout.as_bytes(), + _ => output.stderr.as_bytes(), + }; + testkit::ensure!( + bytes.len() == BYTES, + "exec {stream} has {} bytes; expected {BYTES}", + bytes.len() + ); + testkit::ensure!( + bytes.chunks_exact(2).all(|pair| pair == b"A\n"), + "exec {stream} differs from the `yes A` output" + ); + } + + let mut early_exit = cx.cli_spawn(exec_args(&sandbox, &["head", "-n1"]))?; + let mut input = b"first\n".to_vec(); + input.extend(vec![b'x'; 128 * 1024]); + if let Err(error) = early_exit.write_stdin(&input).await { + testkit::ensure!( + error.kind() == std::io::ErrorKind::BrokenPipe, + "write stdin of `{}`: {error}", + early_exit.command() + ); + } + let output = early_exit.output(EXEC_TIMEOUT).await?.success()?; + testkit::ensure!( + output.stdout == "first\n", + "`head -n1` printed {:?}; expected \"first\\n\"", + output.stdout + ); + + let output = tokio::time::timeout( + Duration::from_secs(10), + cx.cli(exec_args(&sandbox, &["sh", "-c", "sleep 5 & echo ok"])), + ) + .await + .map_err(|_| testkit::Error::msg("exec waited for a background process holding its output"))?? + .success()?; + testkit::ensure!( + output.stdout == "ok\n", + "exec printed {:?}; expected \"ok\\n\"", + output.stdout + ); + Ok(()) +} + +/// Piped stdin larger than one gRPC message reaches the command intact, up to +/// the 4 MiB limit. A larger stdin is rejected with a message naming the +/// limit, a forced TTY with piped stdin still works, and a stdin that can't be +/// read fails the exec. +#[testkit::test] +async fn cli_exec_piped_stdin_crosses_grpc_message_limit(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("si") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let wc = exec_args(&sandbox, &["wc", "-c"]); + + for size in [5, 1024 * 1024, STDIN_LIMIT] { + let mut exec = cx.cli_spawn(&wc)?; + exec.write_stdin(&vec![b'x'; size]) + .await + .context(format!("write {size} bytes to `{}`", exec.command()))?; + let output = exec.output(EXEC_TIMEOUT).await?.success()?; + testkit::ensure!( + output.stdout.trim() == size.to_string(), + "`wc -c` counted {:?} of {size} piped bytes", + output.stdout.trim() + ); + } + + let mut oversized = cx.cli_spawn(&wc)?; + oversized + .write_stdin(&vec![b'x'; STDIN_LIMIT + 1]) + .await + .context(format!( + "write oversized stdin to `{}`", + oversized.command() + ))?; + let output = oversized.output(EXEC_TIMEOUT).await?.failure()?; + testkit::ensure!( + output.stdout.is_empty(), + "an exec with oversized stdin printed {:?}", + output.stdout + ); + testkit::ensure!( + output.stderr.contains("4 MiB limit"), + "an exec with oversized stdin did not name the limit; stderr: {}", + output.stderr.trim() + ); + + // A forced PTY with a small pipe must keep using the unary RPC for + // compatibility with older gateways. + let mut tty = cx.cli_spawn([ + "sandbox", + "exec", + "--name", + sandbox.name(), + "--tty", + "--no-login-shell", + "--", + "sh", + "-c", + "printf tty-ok", + ])?; + tty.write_stdin(b"x") + .await + .context(format!("write stdin to `{}`", tty.command()))?; + let output = tty.output(EXEC_TIMEOUT).await?.success()?; + testkit::ensure!( + output.stdout.contains("tty-ok"), + "a forced TTY exec printed {:?}", + output.stdout + ); + + #[cfg(unix)] + { + let directory = std::fs::File::open("/").context("open / as stdin")?; + let output = cx + .cli_spawn_with_stdin(&wc, directory)? + .output(EXEC_TIMEOUT) + .await? + .failure()?; + testkit::ensure!( + output.stdout.is_empty(), + "an exec whose stdin can't be read printed {:?}", + output.stdout + ); + } + Ok(()) +} + +/// `sandbox exec` of `command` without a TTY or login shell. +fn exec_args(sandbox: &Sandbox, command: &[&str]) -> Vec { + [ + "sandbox", + "exec", + "--name", + sandbox.name(), + "--no-tty", + "--no-login-shell", + "--", + ] + .iter() + .chain(command) + .map(ToString::to_string) + .collect() +} diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index cabd7f3a3b..f0a568679c 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -8,8 +8,10 @@ //! `#[testkit::test]` also registers them, so the `openshell-conformance` //! program in `src/main.rs` runs them by linking this library. +mod attach; mod cli; mod credential_gating; +mod exec_cli; mod file_transfer; mod filesystem_policy; mod image; @@ -20,4 +22,5 @@ mod policy_advisor; mod process; mod settings; mod smoke; +mod ssh; mod templates; diff --git a/tests/suites/conformance/src/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs index 82ff54cc09..31acc4fa04 100644 --- a/tests/suites/conformance/src/policy_advisor.rs +++ b/tests/suites/conformance/src/policy_advisor.rs @@ -224,9 +224,7 @@ async fn create( suffix: &str, policy: Option, ) -> testkit::Result { - let mut builder = cx - .sandbox(suffix) - .command(["sh", "-c", "exec sleep infinity"]); + let mut builder = cx.sandbox(suffix).command(["sleep", "infinity"]); if let Some(policy) = policy { builder = builder.policy(policy); } diff --git a/tests/suites/conformance/src/ssh.rs b/tests/suites/conformance/src/ssh.rs new file mode 100644 index 0000000000..729ef3b51d --- /dev/null +++ b/tests/suites/conformance/src/ssh.rs @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A sandbox's SSH host identity, through `sandbox ssh-config` and a plain +//! `ssh` client that pins the host key. +//! +//! Ported from `e2e/rust/tests/ssh_host_identity.rs`. The check across a +//! gateway restart needs gateway control, which testkit doesn't have yet +//! (#5652). + +use std::path::Path; +use std::time::Duration; + +use testkit::sdk::{ListOptions, SandboxPhase}; +use testkit::{Context, ResultExt as _, Sandbox}; + +/// How long one `ssh` command may take. +const SSH_TIMEOUT: Duration = Duration::from_secs(30); +const SSH_OUTPUT: &str = "ssh-identity-ok"; + +/// A sandbox reports one SSH host key fingerprint in get and list, and that +/// key is the one `ssh` sees. The key stays the same across stop and start, +/// so a pinned `known_hosts` keeps working, and a new sandbox with the same +/// name gets a new key. The workload can't read the supervisor's +/// credentials. +/// +/// Needs a named gateway, which `sandbox ssh-config` puts in its +/// `ProxyCommand`, and an `ssh` client on the test host. +#[testkit::test] +async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate( + cx: &mut Context, +) -> testkit::Result { + let cli = cx.cli_handle(); + testkit::ensure!( + cli.gateway().name.is_some(), + "this test needs a named gateway for `sandbox ssh-config`; set OPENSHELL_GATEWAY" + ); + let sandbox = cx + .sandbox("hk") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let name = sandbox.name().to_string(); + let original = fingerprint(cx, &sandbox).await?; + + let directory = tempfile::tempdir().context("create a directory for the SSH config")?; + let config = directory.path().join("ssh_config"); + let known_hosts = directory.path().join("known_hosts"); + let rendered = cx + .cli(["sandbox", "ssh-config", &name]) + .await? + .success()? + .stdout; + let alias = rendered + .lines() + .find_map(|line| line.trim().strip_prefix("Host ")) + .ok_or_else(|| testkit::Error::msg(format!("no Host line in ssh-config: {rendered}")))? + .trim() + .to_string(); + std::fs::write(&config, &rendered).context("write the SSH config")?; + + let ssh = Ssh { + cli: &cli, + config: &config, + known_hosts: &known_hosts, + alias: &alias, + }; + ssh.run("accept-new").await?; + let key = tokio::process::Command::new("ssh-keygen") + .args(["-l", "-E", "sha256", "-f"]) + .arg(&known_hosts) + .output() + .await + .context("run ssh-keygen")?; + let listed = String::from_utf8_lossy(&key.stdout); + testkit::ensure!( + key.status.success() && listed.split_whitespace().any(|word| word == original), + "the host key ssh pinned is not {original}; ssh-keygen printed {:?}; stderr: {}", + listed.trim(), + String::from_utf8_lossy(&key.stderr).trim() + ); + + let read = sandbox + .exec(["sh", "-c", "test ! -r /.openshell/supervisor/auth.json"]) + .await?; + testkit::ensure!( + read.exit_code == 0, + "the workload can read the supervisor's auth.json" + ); + + cx.cli(["sandbox", "stop", &name]).await?.success()?; + sandbox + .wait_phase(SandboxPhase::Stopped, testkit::STOP_TIMEOUT) + .await?; + let stopped = fingerprint(cx, &sandbox).await?; + testkit::ensure!( + stopped == original, + "the fingerprint changed from {original} to {stopped} on stop" + ); + cx.cli(["sandbox", "start", &name]).await?.success()?; + sandbox.wait_ready().await?; + let started = fingerprint(cx, &sandbox).await?; + testkit::ensure!( + started == original, + "the fingerprint changed from {original} to {started} on start" + ); + ssh.run("yes") + .await + .context("ssh with the pinned host key after a restart")?; + + cx.cli(["sandbox", "delete", &name]).await?.success()?; + // The name stays reserved until the gateway has removed the sandbox. + cx.client() + .wait_deleted(&name, testkit::DELETE_TIMEOUT, Some(sandbox.id())) + .await + .context(format!("wait for sandbox {name} to be deleted"))?; + let replacement = cx + .sandbox("hk") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let replaced = fingerprint(cx, &replacement).await?; + testkit::ensure!( + replaced != original, + "a new sandbox named {name} has the deleted one's host key {original}" + ); + Ok(()) +} + +/// The sandbox's host key fingerprint, the same in get and list. +async fn fingerprint(cx: &Context, sandbox: &Sandbox) -> testkit::Result { + let name = sandbox.name(); + let fingerprint = sandbox.get().await?.host_key_fingerprint.ok_or_else(|| { + testkit::Error::msg(format!("sandbox {name} has no host key fingerprint")) + })?; + testkit::ensure!( + fingerprint.starts_with("SHA256:"), + "sandbox {name} has fingerprint {fingerprint:?}; expected SHA256:..." + ); + let listed = cx + .client() + .list_all_sandboxes(ListOptions::default()) + .await + .context("list sandboxes")? + .into_iter() + .find(|listed| listed.name == name) + .ok_or_else(|| testkit::Error::msg(format!("sandbox {name} is not listed")))?; + testkit::ensure!( + listed.host_key_fingerprint.as_deref() == Some(fingerprint.as_str()), + "sandbox {name} is listed with fingerprint {:?}; get says {fingerprint}", + listed.host_key_fingerprint + ); + Ok(fingerprint) +} + +/// A plain `ssh` client using the config from `sandbox ssh-config` and its +/// own `known_hosts`. +struct Ssh<'a> { + cli: &'a testkit::Cli, + config: &'a Path, + known_hosts: &'a Path, + alias: &'a str, +} + +impl Ssh<'_> { + /// Run a command over SSH with `StrictHostKeyChecking` set to `strict`. + async fn run(&self, strict: &str) -> testkit::Result { + let mut ssh = tokio::process::Command::new("ssh"); + ssh.kill_on_drop(true) + .arg("-F") + .arg(self.config) + .arg("-o") + .arg(format!("StrictHostKeyChecking={strict}")) + .arg("-o") + .arg(format!("UserKnownHostsFile={}", self.known_hosts.display())) + .args(["-o", "BatchMode=yes", self.alias, "printf", SSH_OUTPUT]); + // The ProxyCommand runs the CLI, which must reach the same gateway. + self.cli.apply_env(&mut ssh); + let output = tokio::time::timeout(SSH_TIMEOUT, ssh.output()) + .await + .map_err(|_| testkit::Error::msg(format!("ssh timed out after {SSH_TIMEOUT:?}")))? + .context("run ssh; the test host needs an ssh client")?; + let stdout = String::from_utf8_lossy(&output.stdout); + testkit::ensure!( + output.status.success() && stdout == SSH_OUTPUT, + "ssh to {} exited with {:?} and printed {stdout:?}; stderr: {}", + self.alias, + output.status.code(), + String::from_utf8_lossy(&output.stderr).trim() + ); + Ok(()) + } +} diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 0d91b3a2a9..0b9bd02a90 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -520,6 +520,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "connect-ssh-supervision" +version = "0.0.0" +dependencies = [ + "nix", + "openshell-testkit", + "tokio", +] + [[package]] name = "container-engine-support" version = "0.0.0" @@ -1213,6 +1222,7 @@ version = "0.0.0" dependencies = [ "host-server-support", "openshell-testkit", + "sandbox-clients-support", ] [[package]] @@ -3029,6 +3039,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sandbox-clients-support" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "serde", + "serde_json", +] + [[package]] name = "schannel" version = "0.1.29" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index a592b6f028..78f4dd6277 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["cdi", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] +members = ["cdi", "connect-ssh-supervision", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] diff --git a/tests/suites/features/cdi/src/lib.rs b/tests/suites/features/cdi/src/lib.rs index 46a94f3c86..ddadd385ef 100644 --- a/tests/suites/features/cdi/src/lib.rs +++ b/tests/suites/features/cdi/src/lib.rs @@ -374,26 +374,13 @@ pub async fn scenario(cx: &Context, name: &str) -> Result { let Scenario { device, checks } = load(name)?; let driver = compute_driver(cx).await?; - let sandbox = cx.name("cdi"); - cx.track(&sandbox); let config = json!({driver: {"cdi_devices": [device]}}).to_string(); let probe = probe_script(&checks); let output = cx - .cli([ - "sandbox", - "create", - "--name", - &sandbox, - "--no-tty", - "--gpu", - "1", - "--driver-config-json", - &config, - "--", - "sh", - "-c", - &probe, - ]) + .cli_sandbox("cdi") + .args(["--gpu", "1", "--driver-config-json", &config]) + .command(["sh", "-c", &probe]) + .run() .await?; let output = output diff --git a/tests/suites/features/connect-ssh-supervision/Cargo.toml b/tests/suites/features/connect-ssh-supervision/Cargo.toml new file mode 100644 index 0000000000..ad7c9fa6f1 --- /dev/null +++ b/tests/suites/features/connect-ssh-supervision/Cargo.toml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "connect-ssh-supervision" +description = "Features suite: `sandbox connect` recovers its SSH transport and forwards termination" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../../testkit/testkit" } +tokio = { version = "1.43", features = ["rt", "time"] } + +[target.'cfg(target_os = "linux")'.dependencies] +nix = { version = "0.29", features = ["process", "signal", "term"] } diff --git a/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs b/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs new file mode 100644 index 0000000000..f911f7819d --- /dev/null +++ b/tests/suites/features/connect-ssh-supervision/tests/connect_ssh_supervision.rs @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! How `sandbox connect` supervises the `ssh` process it attaches through: +//! it reconnects when the SSH transport is lost, and a SIGTERM sent to the +//! CLI alone ends the `ssh` child too. +//! +//! The tests find the CLI's `ssh` and `ssh-proxy` processes in the test +//! host's `/proc`, so they need Linux. Attach behavior every driver must +//! provide is in the conformance `attach` suite. +//! +//! CI runs them on Docker only. Through a Kubernetes port-forward, the +//! reconnect can attach read-only while the old SSH session still owns input +//! (#4445). +//! +//! Ported from `e2e/rust/tests/sandbox_lifecycle.rs`. + +use std::time::Duration; + +use testkit::Context; + +/// How long an attachment may take to show output or recover. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +const ATTACH_TIMEOUT: Duration = Duration::from_secs(30); + +/// Killing the `ssh-proxy` transport of an established `sandbox connect` +/// makes the CLI report the lost connection and reconnect to the same main +/// process, which then receives new input. The CLI keeps running. +#[testkit::test] +async fn connect_recovers_its_ssh_transport(cx: &mut Context) -> testkit::Result { + #[cfg(target_os = "linux")] + return linux::connect_recovers_its_ssh_transport(cx).await; + #[cfg(not(target_os = "linux"))] + return needs_linux(cx); +} + +/// A SIGTERM sent to only the `sandbox connect` process, with a terminal as +/// its stdin, is forwarded to its `ssh` child. The CLI exits 143 and the +/// `ssh` child is gone. +#[testkit::test] +async fn connect_forwards_pid_targeted_termination_and_reaps_ssh( + cx: &mut Context, +) -> testkit::Result { + #[cfg(target_os = "linux")] + return linux::connect_forwards_pid_targeted_termination_and_reaps_ssh(cx).await; + #[cfg(not(target_os = "linux"))] + return needs_linux(cx); +} + +#[cfg(not(target_os = "linux"))] +fn needs_linux(_cx: &Context) -> testkit::Result { + Err(testkit::Error::msg( + "this test reads the CLI's processes from /proc and needs a Linux test host", + )) +} + +#[cfg(target_os = "linux")] +mod linux { + use std::os::fd::OwnedFd; + use std::process::Stdio; + use std::time::Duration; + + use nix::sys::signal::{Signal, kill}; + use nix::unistd::Pid; + use testkit::{Context, ResultExt as _}; + + use super::ATTACH_TIMEOUT; + + /// How long the CLI treats a new attachment as still connecting. A lost + /// transport after that is one it recovers from. + const ESTABLISHED: Duration = Duration::from_secs(3); + + /// A main process that prints a numbered line every 0.2 seconds and + /// echoes each line of its input. + const ECHO_MAIN: &str = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'transport_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'transport_pid=%s input=%s\n' "$$" "$line"; done"#; + + pub(super) async fn connect_recovers_its_ssh_transport(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("tr") + .command(["sh", "-lc", ECHO_MAIN]) + .create_ready() + .await?; + let mut connect = cx.cli_spawn(["sandbox", "connect", sandbox.name()])?; + let first = connect.stdout_line(ATTACH_TIMEOUT).await?; + testkit::ensure!( + first.contains("transport_pid="), + "unexpected attachment output {first:?}" + ); + + // Recovery starts only for an established attachment. + tokio::time::sleep(ESTABLISHED).await; + let proxy = wait_for_process( + &format!("the ssh-proxy of sandbox {}", sandbox.name()), + || find_process(connect.pid(), &["ssh-proxy", "--sandbox", sandbox.name()]), + ) + .await?; + send(proxy, Signal::SIGTERM).context("terminate the ssh-proxy transport")?; + connect + .stderr_until(ATTACH_TIMEOUT, |line| { + line.contains("Connection to sandbox lost; reconnecting") + }) + .await + .context("the CLI reports the lost transport")?; + + let token = format!("after-transport-recovery-{}", cx.run_id()); + connect + .write_stdin(format!("{token}\n").as_bytes()) + .await + .context("send input after recovery")?; + connect + .stdout_until(ATTACH_TIMEOUT, |line| { + line.contains(&format!("input={token}")) + }) + .await + .context("the reconnected attachment reaches the main process")?; + testkit::ensure!( + connect.try_wait()?.is_none(), + "`{}` exited after recovering", + connect.command() + ); + connect.kill().await + } + + pub(super) async fn connect_forwards_pid_targeted_termination_and_reaps_ssh( + cx: &mut Context, + ) -> testkit::Result { + let sandbox = cx + .sandbox("term") + .command(["sh", "-lc", "exec sleep infinity"]) + .create_ready() + .await?; + let pty = nix::pty::openpty(None, None) + .map_err(std::io::Error::from) + .context("open a pseudo-terminal")?; + // Held until the end, so the terminal stays open. + let _controller: OwnedFd = pty.master; + let mut connect = cx.cli_spawn_with_stdin( + ["sandbox", "connect", sandbox.name()], + Stdio::from(pty.slave), + )?; + let connect_pid = connect.pid(); + + let ssh = wait_for_process(&format!("the ssh child of `{}`", connect.command()), || { + find_process(connect_pid, &["-s", "sandbox", "openshell-main"]) + }) + .await?; + send(connect_pid, Signal::SIGTERM).context("send SIGTERM to only the CLI")?; + let code = connect.wait(Duration::from_secs(10)).await?; + testkit::ensure!( + code == 143, + "`{}` exited with {code} on SIGTERM; expected 143", + connect.command() + ); + testkit::poll_until( + &format!("ssh process {ssh} to be gone"), + Duration::from_secs(5), + Duration::from_millis(25), + async || { + Ok(if std::fs::metadata(format!("/proc/{ssh}")).is_ok() { + testkit::Poll::Pending(format!("/proc/{ssh} still exists")) + } else { + testkit::Poll::Ready(()) + }) + }, + ) + .await + } + + fn send(pid: u32, signal: Signal) -> testkit::Result { + let pid = i32::try_from(pid) + .map_err(|error| testkit::Error::msg(format!("pid {pid}: {error}")))?; + kill(Pid::from_raw(pid), signal) + .map_err(std::io::Error::from) + .context(format!("send {signal} to {pid}")) + } + + async fn wait_for_process( + what: &str, + mut find: impl FnMut() -> Option, + ) -> testkit::Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + if let Some(pid) = find() { + return Ok(pid); + } + testkit::ensure!( + tokio::time::Instant::now() < deadline, + "{what} did not start" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + /// A descendant of `ancestor` whose arguments include all of `args`. + fn find_process(ancestor: u32, args: &[&str]) -> Option { + for entry in std::fs::read_dir("/proc").ok()?.filter_map(Result::ok) { + let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { + continue; + }; + if !descends_from(pid, ancestor) { + continue; + } + let cmdline = std::fs::read(entry.path().join("cmdline")).unwrap_or_default(); + let argv: Vec<&[u8]> = cmdline.split(|byte| *byte == 0).collect(); + if args.iter().all(|arg| argv.contains(&arg.as_bytes())) { + return Some(pid); + } + } + None + } + + fn descends_from(pid: u32, ancestor: u32) -> bool { + let mut current = pid; + while current > 1 { + let Some(parent) = parent_pid(current) else { + return false; + }; + if parent == ancestor { + return true; + } + current = parent; + } + false + } + + fn parent_pid(pid: u32) -> Option { + std::fs::read_to_string(format!("/proc/{pid}/status")) + .ok()? + .lines() + .find_map(|line| line.strip_prefix("PPid:")?.trim().parse().ok()) + } +} diff --git a/tests/suites/features/host-gateway/Cargo.toml b/tests/suites/features/host-gateway/Cargo.toml index 0dfef50bc5..e5c1cd8cc0 100644 --- a/tests/suites/features/host-gateway/Cargo.toml +++ b/tests/suites/features/host-gateway/Cargo.toml @@ -14,3 +14,4 @@ publish = false [dependencies] host-server-support = { path = "../../../support/host-server" } openshell-testkit = { path = "../../../testkit/testkit" } +sandbox-clients-support = { path = "../../../support/sandbox-clients" } diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 7d285116e4..f77a3e378a 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -5,36 +5,16 @@ //! compute driver. //! //! The test starts an HTTP server where the tests run, which is the host the -//! gateway runs its sandboxes on, and runs a client in a sandbox from the -//! Python test image ([`testkit::python_image`]) that requests a page from that -//! server through the name. +//! gateway runs its sandboxes on, and requests a page from it through the name +//! with [`sandbox_clients_support::http`], from a sandbox created from the +//! Python test image ([`testkit::python_image`]). -use host_server_support::HttpRecorder; +use host_server_support::{HttpRecorder, TEST_HOST}; +use sandbox_clients_support::{Request, http}; use testkit::policy; use testkit::sdk::raw::proto; use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; -/// The image's Python, which needs nothing beyond its standard library. The -/// policy allows [`PYTHON_BINARIES`], the real paths behind `python3` in -/// `testkit::python_image()` images, because a rule matches the executable -/// behind a symlink. -const PYTHON: &str = "python3"; - -/// Requests `/` from `host.openshell.internal` on the port in `argv[1]` and -/// prints the response. The server listens before the sandbox is created, -/// and a Ready sandbox's network is up, so the client doesn't retry. -const CLIENT: &str = r#" -import socket, sys - -host, port = "host.openshell.internal", int(sys.argv[1]) -with socket.create_connection((host, port), timeout=10) as conn: - conn.sendall(f"GET / HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n".encode()) - response = b"" - while chunk := conn.recv(4096): - response += chunk -print(response.decode(errors="replace")) -"#; - /// A sandbox from the Python test image resolves `host.openshell.internal` /// and gets a response from a server the test runs on the host. The default /// policy denies all network access, so the test passes one that lets only the @@ -53,38 +33,35 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result let sandbox = cx .sandbox("hg") .image(image.clone()) + .command(["sleep", "infinity"]) .policy(policy(server.port)) - .command(["sh", "-c", "exec sleep infinity"]) .create_ready() .await .context(format!("create a sandbox from {image}"))?; - let client = sandbox - .exec([PYTHON, "-c", CLIENT, &server.port.to_string()]) - .await? - .success() - .context("run the client in the sandbox")?; + // The server listens before the sandbox is created, and a Ready + // sandbox's network is up, so the request isn't retried. + let reply = http(&sandbox, TEST_HOST, server.port, &Request::get("/")) + .await + .context("request a page from the host server")?; testkit::ensure!( - client.stdout.contains(&marker), - "the client did not get the host server's response:\n{}{}", - client.stdout, - client.stderr + reply + .response + .as_ref() + .is_some_and(|response| response.body_text().contains(&marker)), + "the client did not get the host server's response: {reply:?}" ); Ok(()) } -/// A sandbox policy that allows the image's Python ([`PYTHON`]) to -/// connect to `host.openshell.internal` on `port`. The endpoint names only the +/// A sandbox policy that allows the image's Python to connect to +/// `host.openshell.internal` on `port`. It allows [`PYTHON_BINARIES`], the +/// real paths behind `python3` in `testkit::python_image()` images, because a +/// rule matches the executable behind a symlink. The endpoint names only the /// host and port, as the policy docs describe, so the driver's address for the /// name is what makes it reachable, not an `allowed_ips` range. fn policy(port: u16) -> proto::SandboxPolicy { let mut policy = policy::base(); - policy::allow( - &mut policy, - "host_server", - "host.openshell.internal", - port, - PYTHON_BINARIES, - ); + policy::allow(&mut policy, "host_server", TEST_HOST, port, PYTHON_BINARIES); policy } diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 6525cfc070..a27343f10e 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -2991,6 +2991,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sandbox-clients-support" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "serde", + "serde_json", +] + [[package]] name = "schannel" version = "0.1.29" diff --git a/tests/support/Cargo.toml b/tests/support/Cargo.toml index 4c7c495f97..e3dae4234d 100644 --- a/tests/support/Cargo.toml +++ b/tests/support/Cargo.toml @@ -5,4 +5,4 @@ # environment runs. Suites depend on them by path. [workspace] resolver = "2" -members = ["container-engine", "host-server", "keycloak", "podman"] +members = ["container-engine", "host-server", "keycloak", "podman", "sandbox-clients"] diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 8ac8528570..f10a0e0632 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -21,6 +21,9 @@ use tokio::net::TcpListener; pub use http::{HttpRecorder, Request}; pub use websocket::WebSocketRecorder; +/// The name a sandbox reaches these servers by. +pub const TEST_HOST: &str = "host.openshell.internal"; + const WAIT_TIMEOUT: Duration = Duration::from_secs(10); const WAIT_INTERVAL: Duration = Duration::from_millis(100); const READ_TIMEOUT: Duration = Duration::from_secs(10); diff --git a/tests/support/sandbox-clients/Cargo.toml b/tests/support/sandbox-clients/Cargo.toml new file mode 100644 index 0000000000..9a7455e7c0 --- /dev/null +++ b/tests/support/sandbox-clients/Cargo.toml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "sandbox-clients-support" +description = "Clients that tests run in a sandbox, and typed probes for their results" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../testkit/testkit" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/tests/support/sandbox-clients/scripts/tcp_probe.py b/tests/support/sandbox-clients/scripts/tcp_probe.py new file mode 100644 index 0000000000..e1209729cd --- /dev/null +++ b/tests/support/sandbox-clients/scripts/tcp_probe.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Connects to argv[1]:argv[2], sends the bytes hex-encoded in argv[3] ("-" +# to only connect), and reads until the connection ends. Prints how each step +# went as one JSON line; failures to connect or read are results, not errors. +# +# argv[4]: connect timeout in seconds. +# argv[5]: read timeout in seconds, for all of the reading. +# argv[6]: idle timeout in seconds: stop reading after this long without +# bytes. 0 waits for the read timeout instead. +import json +import socket +import sys +import time + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +PAYLOAD = None if sys.argv[3] == "-" else bytes.fromhex(sys.argv[3]) +CONNECT_TIMEOUT = float(sys.argv[4]) +READ_TIMEOUT = float(sys.argv[5]) +IDLE_TIMEOUT = float(sys.argv[6]) + +result = { + "connect": None, + "errno": None, + "error": None, + "connect_ms": None, + "read_end": None, + "read_errno": None, + "read_error": None, + "received_hex": "", +} + + +def millis(since): + return int((time.monotonic() - since) * 1000) + + +def exchange(sock): + """Send PAYLOAD and read until the connection ends. Returns the bytes + read and how reading ended.""" + received = bytearray() + try: + sock.sendall(PAYLOAD) + deadline = time.monotonic() + READ_TIMEOUT + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return received, "timeout" + idle = 0 < IDLE_TIMEOUT < remaining + sock.settimeout(IDLE_TIMEOUT if idle else remaining) + try: + chunk = sock.recv(65536) + except socket.timeout: + return received, "idle" if idle else "timeout" + if not chunk: + return received, "closed" + received += chunk + except ConnectionResetError: + return received, "reset" + except OSError as error: + result["read_errno"] = error.errno + result["read_error"] = repr(error) + return received, "failed" + + +start = time.monotonic() +try: + sock = socket.create_connection((HOST, PORT), timeout=CONNECT_TIMEOUT) +except socket.timeout: + result["connect"] = "timeout" +except OSError as error: + result["connect"] = "failed" + result["errno"] = error.errno + result["error"] = repr(error) +else: + result["connect"] = "ok" +result["connect_ms"] = millis(start) + +if result["connect"] == "ok": + with sock: + if PAYLOAD is not None: + received, read_end = exchange(sock) + result["received_hex"] = received.hex() + result["read_end"] = read_end + +print(json.dumps(result, sort_keys=True), flush=True) diff --git a/tests/support/sandbox-clients/src/lib.rs b/tests/support/sandbox-clients/src/lib.rs new file mode 100644 index 0000000000..392507aa43 --- /dev/null +++ b/tests/support/sandbox-clients/src/lib.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Clients that tests run inside a sandbox, and typed results for them. +//! +//! The clients run in a sandbox created from [`testkit::python_image`], with a +//! policy that lets [`testkit::PYTHON_BINARIES`] reach the target: +//! +//! ```ignore +//! use host_server_support::{HttpRecorder, TEST_HOST}; +//! use sandbox_clients_support::{Request, http}; +//! use testkit::{PYTHON_BINARIES, policy}; +//! +//! let server = HttpRecorder::start("hello").await?; +//! let mut allowed = policy::base(); +//! policy::allow(&mut allowed, "server", TEST_HOST, server.port, PYTHON_BINARIES); +//! let sandbox = cx +//! .sandbox("probe") +//! .image(testkit::python_image()) +//! .command(["sleep", "infinity"]) +//! .policy(allowed) +//! .create_ready() +//! .await?; +//! let reply = http(&sandbox, TEST_HOST, server.port, &Request::get("/")).await?; +//! testkit::ensure!(reply.status() == Some(200), "{reply:?}"); +//! ``` +//! +//! - [`connect`] opens a TCP connection and reports whether it connected, +//! the errno it failed with, or a timeout, and how long it took. +//! - [`http`] sends a [`Request`], and [`http_raw`] sends bytes as they are. +//! Both report how the connection went, the bytes that came back, and the +//! parsed [`Response`]. +//! - [`scripts`] holds the client scripts, for tests that run one with +//! [`testkit::Sandbox::run_python`] and read its result with +//! [`testkit::ExecOutput::last_json`]. Protocol clients, such as `GraphQL` or +//! JSON-RPC ones, belong there too, so every suite can use them. +//! +//! Both probes use a raw socket. Sandboxes set no proxy variables, so this is +//! the same path a library such as `urllib` takes. + +mod message; +mod probe; + +pub use message::{Http, Request, Response}; +pub use probe::{Connect, ConnectOutcome, ReadEnd, connect, http, http_raw}; + +/// Client scripts, as text to pass to [`testkit::Sandbox::run_python`]. +pub mod scripts { + /// Connects to a host and port, optionally sends bytes and reads the + /// reply, and prints the outcome as one JSON line. [`crate::connect`] and + /// [`crate::http`] run it and parse its output. + pub const TCP_PROBE: &str = include_str!("../scripts/tcp_probe.py"); +} diff --git a/tests/support/sandbox-clients/src/message.rs b/tests/support/sandbox-clients/src/message.rs new file mode 100644 index 0000000000..e74b6fca1a --- /dev/null +++ b/tests/support/sandbox-clients/src/message.rs @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP requests for [`crate::http`] and the replies it reads. + +use std::fmt; +use std::time::Duration; + +use crate::probe::{ConnectOutcome, Probe, ReadEnd}; + +/// An HTTP/1.1 request for [`crate::http`]. +/// +/// It sends `Host` for the target host and port, `Content-Length` for a body +/// or a method that takes one, and `Connection: close`, unless the request +/// sets those headers itself. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Request { + method: String, + target: String, + headers: Vec<(String, String)>, + body: Vec, +} + +impl Request { + /// `GET target`. + pub fn get(target: impl Into) -> Self { + Self::new("GET", target) + } + + /// `method target`. The target is usually a path, such as `/allowed`, + /// but may be absolute, such as `http://host:port/allowed`. + pub fn new(method: impl Into, target: impl Into) -> Self { + Self { + method: method.into(), + target: target.into(), + headers: Vec::new(), + body: Vec::new(), + } + } + + /// Add a header. A header added more than once is sent more than once. + #[must_use] + pub fn header(mut self, name: impl Into, value: impl Into) -> Self { + self.headers.push((name.into(), value.into())); + self + } + + /// Send `body` with the request. + #[must_use] + pub fn body(mut self, body: impl Into>) -> Self { + self.body = body.into(); + self + } + + pub(crate) fn is_head(&self) -> bool { + self.method.eq_ignore_ascii_case("HEAD") + } + + /// The request as bytes on the wire to `host:port`. + pub(crate) fn encode(&self, host: &str, port: u16) -> Vec { + let has = |name: &str| { + self.headers + .iter() + .any(|(header, _)| header.eq_ignore_ascii_case(name)) + }; + let mut head = format!("{} {} HTTP/1.1\r\n", self.method, self.target); + if !has("Host") { + if host.contains(':') { + head.push_str(&format!("Host: [{host}]:{port}\r\n")); + } else { + head.push_str(&format!("Host: {host}:{port}\r\n")); + } + } + for (name, value) in &self.headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + let takes_body = ["POST", "PUT", "PATCH"] + .iter() + .any(|method| self.method.eq_ignore_ascii_case(method)); + if (takes_body || !self.body.is_empty()) + && !has("Content-Length") + && !has("Transfer-Encoding") + { + head.push_str(&format!("Content-Length: {}\r\n", self.body.len())); + } + if !has("Connection") { + head.push_str("Connection: close\r\n"); + } + head.push_str("\r\n"); + let mut bytes = head.into_bytes(); + bytes.extend_from_slice(&self.body); + bytes + } +} + +/// The result of [`crate::http`] or [`crate::http_raw`]. +#[derive(Clone, PartialEq, Eq)] +pub struct Http { + pub connect: ConnectOutcome, + /// How long the connect took. + pub elapsed: Duration, + /// How reading the reply ended, or none if the connect failed. + pub read_end: Option, + /// Every byte that came back. + pub received: Vec, + /// The reply parsed as an HTTP response, or none if nothing came back or + /// it isn't one. Only the first response is parsed. + pub response: Option, +} + +impl Http { + pub(crate) fn from_probe(probe: Probe, head: bool) -> Self { + let response = parse_response(&probe.received, head); + Self { + connect: probe.connect, + elapsed: probe.connect_elapsed, + read_end: probe.read_end, + received: probe.received, + response, + } + } + + /// The response's status, if one came back. + pub fn status(&self) -> Option { + self.response.as_ref().map(|response| response.status) + } + + /// [`Http::received`] as text, with invalid UTF-8 replaced. + pub fn received_text(&self) -> String { + String::from_utf8_lossy(&self.received).into_owned() + } +} + +impl fmt::Debug for Http { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Http") + .field("connect", &self.connect) + .field("elapsed", &self.elapsed) + .field("read_end", &self.read_end) + .field("received", &self.received_text()) + .field("response", &self.response) + .finish() + } +} + +/// An HTTP response. +#[derive(Clone, PartialEq, Eq)] +pub struct Response { + pub status: u16, + /// The headers, in order, as they arrived. + pub headers: Vec<(String, String)>, + /// The body, with chunked transfer encoding removed. + pub body: Vec, +} + +impl Response { + /// The first header named `name`, ignoring case. + pub fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(header, _)| header.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + /// The body as text, with invalid UTF-8 replaced. + pub fn body_text(&self) -> String { + String::from_utf8_lossy(&self.body).into_owned() + } +} + +impl fmt::Debug for Response { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Response") + .field("status", &self.status) + .field("headers", &self.headers) + .field("body", &self.body_text()) + .finish() + } +} + +/// The first HTTP response in `bytes`, or none if they don't start with a +/// complete status line and headers. `head` says the request was `HEAD`, so +/// the response has no body. +/// +/// The body ends where chunked encoding or `Content-Length` says, or at the +/// end of `bytes` without either. A truncated body is returned as far as it +/// arrived. +fn parse_response(bytes: &[u8], head: bool) -> Option { + let end = find(bytes, b"\r\n\r\n")?; + let text = std::str::from_utf8(&bytes[..end]).ok()?; + let mut lines = text.split("\r\n"); + let status = parse_status_line(lines.next()?)?; + let headers: Vec<(String, String)> = lines + .filter_map(|line| { + let (name, value) = line.split_once(':')?; + Some((name.trim().to_string(), value.trim().to_string())) + }) + .collect(); + let rest = &bytes[end + 4..]; + let mut response = Response { + status, + headers, + body: Vec::new(), + }; + let bodiless = head || (100..200).contains(&status) || status == 204 || status == 304; + if !bodiless { + response.body = if response + .header("Transfer-Encoding") + .is_some_and(|encoding| encoding.to_ascii_lowercase().contains("chunked")) + { + dechunk(rest) + } else if let Some(length) = response + .header("Content-Length") + .and_then(|length| length.parse::().ok()) + { + rest[..length.min(rest.len())].to_vec() + } else { + rest.to_vec() + }; + } + Some(response) +} + +/// The status code in `HTTP/1.x NNN reason`. +fn parse_status_line(line: &str) -> Option { + let mut parts = line.splitn(3, ' '); + let version = parts.next()?; + let code = parts.next()?; + if !version.starts_with("HTTP/1.") || code.len() != 3 { + return None; + } + code.parse().ok() +} + +/// The data in a chunked body, up to the last complete chunk if it is cut +/// short. +fn dechunk(mut rest: &[u8]) -> Vec { + let mut body = Vec::new(); + while let Some(line_end) = find(rest, b"\r\n") { + let size_line = String::from_utf8_lossy(&rest[..line_end]); + let size = size_line.split(';').next().unwrap_or_default().trim(); + let Ok(size) = usize::from_str_radix(size, 16) else { + break; + }; + rest = &rest[line_end + 2..]; + if size == 0 { + break; + } + let available = size.min(rest.len()); + body.extend_from_slice(&rest[..available]); + if available < size { + break; + } + rest = rest.get(size + 2..).unwrap_or_default(); + } + body +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_chunked_response_is_decoded() { + let response = parse_response( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5;ext=1\r\nhello\r\n6\r\n world\r\n0\r\n\r\n", + false, + ) + .expect("a response"); + assert_eq!(response.body, b"hello world"); + } + + #[test] + fn non_http_or_incomplete_bytes_are_not_a_response() { + assert_eq!(parse_response(b"", false), None); + assert_eq!(parse_response(b"HTTP/1.1 200 OK\r\nContent-", false), None); + assert_eq!( + parse_response(b"\x00\xff\x13\x37not-http\r\n\r\n", false), + None + ); + assert_eq!(parse_response(b"SSH-2.0-OpenSSH\r\n\r\n", false), None); + assert_eq!(parse_response(b"HTTP/1.1 20 OK\r\n\r\n", false), None); + assert_eq!(parse_response(b"HTTP/2 200\r\n\r\n", false), None); + } +} diff --git a/tests/support/sandbox-clients/src/probe.rs b/tests/support/sandbox-clients/src/probe.rs new file mode 100644 index 0000000000..c967d24b00 --- /dev/null +++ b/tests/support/sandbox-clients/src/probe.rs @@ -0,0 +1,228 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Runs [`scripts::TCP_PROBE`] in a sandbox and turns its JSON line into +//! typed results. + +use std::time::Duration; + +use serde::Deserialize; +use testkit::{Result, ResultExt as _, Sandbox}; + +use crate::message::{Http, Request}; +use crate::scripts; + +/// How long a connect may take, as the clients the probes replace allowed. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// How long reading a reply may take in all. +const READ_TIMEOUT: Duration = Duration::from_secs(15); +/// How long [`http_raw`] waits for more bytes before it stops reading. Raw +/// bytes may leave the server waiting for more, so the connection may never +/// close. +const RAW_IDLE_TIMEOUT: Duration = Duration::from_secs(2); + +const EPERM: i32 = 1; +const EACCES: i32 = 13; + +/// How a connect ended. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ConnectOutcome { + Connected, + /// The connect failed with `errno`, or none for an error without one. + /// `error` is Python's description of it. + Failed { + errno: Option, + error: String, + }, + TimedOut, +} + +impl ConnectOutcome { + pub fn is_connected(&self) -> bool { + matches!(self, Self::Connected) + } + + /// The errno a failed connect reported. + pub fn errno(&self) -> Option { + match self { + Self::Failed { errno, .. } => *errno, + Self::Connected | Self::TimedOut => None, + } + } + + /// Whether the connect failed with `EACCES` or `EPERM`, as the sandbox + /// denies a connection. + pub fn is_permission_denied(&self) -> bool { + matches!(self.errno(), Some(EACCES | EPERM)) + } +} + +/// The result of [`connect`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Connect { + pub outcome: ConnectOutcome, + /// How long the connect took, until it connected, failed, or timed out. + pub elapsed: Duration, +} + +/// How reading a reply ended. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReadEnd { + /// The peer closed the connection. + Closed, + /// The peer reset the connection (`ECONNRESET`). + Reset, + /// No bytes arrived for the idle timeout of [`http_raw`]. + Idle, + /// The read timeout ran out before the connection ended. + TimedOut, + /// Sending or reading failed some other way. + Failed { errno: Option, error: String }, +} + +/// Open a TCP connection from `sandbox` to `host:port`, then close it. +/// A connect that fails or times out is a result, not an error. +pub async fn connect(sandbox: &Sandbox, host: &str, port: u16) -> Result { + let probe = run(sandbox, host, port, None, Duration::ZERO).await?; + Ok(Connect { + outcome: probe.connect, + elapsed: probe.connect_elapsed, + }) +} + +/// Send `request` from `sandbox` to `host:port` and read the reply until the +/// server closes the connection. The request asks it to, with +/// `Connection: close`, unless it sets its own `Connection` header. +pub async fn http(sandbox: &Sandbox, host: &str, port: u16, request: &Request) -> Result { + let bytes = request.encode(host, port); + let probe = run(sandbox, host, port, Some(&bytes), Duration::ZERO).await?; + Ok(Http::from_probe(probe, request.is_head())) +} + +/// Send `bytes` as they are from `sandbox` to `host:port`, and read the reply +/// until the connection ends or goes quiet for two seconds. Use it for +/// requests [`Request`] can't make, or bytes that aren't HTTP at all. +pub async fn http_raw(sandbox: &Sandbox, host: &str, port: u16, bytes: &[u8]) -> Result { + let probe = run(sandbox, host, port, Some(bytes), RAW_IDLE_TIMEOUT).await?; + Ok(Http::from_probe(probe, false)) +} + +/// What the probe script reported. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Probe { + pub(crate) connect: ConnectOutcome, + pub(crate) connect_elapsed: Duration, + pub(crate) read_end: Option, + pub(crate) received: Vec, +} + +async fn run( + sandbox: &Sandbox, + host: &str, + port: u16, + payload: Option<&[u8]>, + idle_timeout: Duration, +) -> Result { + let payload = payload.map_or_else(|| "-".to_string(), to_hex); + let args = [ + host.to_string(), + port.to_string(), + payload, + CONNECT_TIMEOUT.as_secs_f64().to_string(), + READ_TIMEOUT.as_secs_f64().to_string(), + idle_timeout.as_secs_f64().to_string(), + ]; + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + let output = sandbox + .run_python(scripts::TCP_PROBE, &args) + .await + .context(format!( + "probe {host}:{port} from sandbox {}", + sandbox.name() + ))?; + parse_probe(output.last_json()?) +} + +#[derive(Deserialize)] +struct ProbeJson { + connect: ConnectJson, + errno: Option, + error: Option, + connect_ms: u64, + read_end: Option, + read_errno: Option, + read_error: Option, + received_hex: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +enum ConnectJson { + Ok, + Failed, + Timeout, +} + +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +enum ReadEndJson { + Closed, + Reset, + Idle, + Timeout, + Failed, +} + +fn parse_probe(value: serde_json::Value) -> Result { + let json: ProbeJson = serde_json::from_value(value.clone()) + .context(format!("read the probe's result {value}"))?; + let connect = match json.connect { + ConnectJson::Ok => ConnectOutcome::Connected, + ConnectJson::Failed => ConnectOutcome::Failed { + errno: json.errno, + error: json.error.unwrap_or_default(), + }, + ConnectJson::Timeout => ConnectOutcome::TimedOut, + }; + let read_end = json.read_end.map(|end| match end { + ReadEndJson::Closed => ReadEnd::Closed, + ReadEndJson::Reset => ReadEnd::Reset, + ReadEndJson::Idle => ReadEnd::Idle, + ReadEndJson::Timeout => ReadEnd::TimedOut, + ReadEndJson::Failed => ReadEnd::Failed { + errno: json.read_errno, + error: json.read_error.clone().unwrap_or_default(), + }, + }); + Ok(Probe { + connect, + connect_elapsed: Duration::from_millis(json.connect_ms), + read_end, + received: from_hex(&json.received_hex)?, + }) +} + +fn to_hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + bytes.iter().fold(String::new(), |mut hex, byte| { + let _ = write!(hex, "{byte:02x}"); + hex + }) +} + +fn from_hex(hex: &str) -> Result> { + testkit::ensure!( + hex.len().is_multiple_of(2), + "the probe's received_hex has odd length" + ); + (0..hex.len()) + .step_by(2) + .map(|at| { + hex.get(at..at + 2) + .and_then(|pair| u8::from_str_radix(pair, 16).ok()) + .ok_or_else(|| { + testkit::Error::msg(format!("the probe's received_hex {hex:?} isn't hex")) + }) + }) + .collect() +} diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 3ce6f58013..f3f1f3c66e 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -63,18 +63,29 @@ fails. A killed test process skips cleanup, so names are unique to the run. `Context` currently provides: -- `cx.sandbox(suffix)` to build a sandbox with a run-unique name, then +- `cx.sandbox(suffix)` to build a sandbox with a run-unique name + (`test--`), then `create()` it or `create_ready()` to also wait for Ready. Set its image - with `image(..)` and its policy with `policy(..)`. The returned + with `image(..)`, its policy with `policy(..)` and its providers with + `providers(..)`. A long-running sandbox is + `.command(["sleep", "infinity"])`; add `.image(testkit::python_image())` + when a test runs Python clients in it. The returned `Sandbox` can wait for ready or finished, stop, start, exec or `poll_exec` until the output matches, read its config and draft policy, set - a setting, read `sandbox get` JSON with `get_json()`, and delete. Its - `expect_listed`, `expect_absent` and `wait_absent` check list and get. - Exec output has `success()`, `failure()` and `expect_stdout()`. + a setting, read `sandbox get` JSON with `get_json()`, read its policy + revisions and active policy with `policy_revisions()` and + `active_policy()`, and delete. Its `expect_listed`, `expect_absent` and + `wait_absent` check list and get. `run_python(script, args)` runs a script + with `python3 -c` and fails unless it exits 0. Exec output has `success()`, + `failure()`, `expect_stdout()`, and `last_json()` for the last JSON object + or array line of stdout, such as a client script's result. - `cx.cli_sandbox(suffix)` for a sandbox only `openshell sandbox create` can make, such as one with `policy_yaml(...)` or `--no-keep`. `create()` detaches and returns the `Sandbox`, `create_ready()` also waits for Ready, and `run()` returns the attached CLI output. +- Sandbox names are limited to `testkit::MAX_SANDBOX_NAME_LEN` (19) + characters, so a sandbox suffix can have at most 5. `cx.sandbox`, + `cx.cli_sandbox` and `cx.track` panic on a longer name. - `cx.track(name)` to register a sandbox created some other way, such as through the CLI, for cleanup. Call it before the create, so a create that fails after the gateway accepted it is still cleaned @@ -93,6 +104,14 @@ fails. A killed test process skips cleanup, so names are unique to the run. plus `json()`, and `report_text()` for matching a wrapped error report. `cx.cli_handle()` returns that CLI for code that can't borrow `cx`, such as a cleanup hook. +- `cx.cli_spawn([...])` to start the CLI and keep it running, for commands + that stream or read input, such as `sandbox connect`. The `CliChild` it + returns writes stdin, reads stdout and stderr line by line or all at once, + sends signals, and waits for the exit code. `cx.cli_spawn_with_stdin` takes + another stdin, such as a file or a pseudo-terminal. +- `Cli::apply_env` to point another program at the same gateway, such as `ssh` + with the `ProxyCommand` from `sandbox ssh-config`. `Cli::gateway` says which + gateway that is. - `cx.on_cleanup(name, hook)` to remove something when the test ends. - `testkit::policy` for sandbox policies: `policy::base()` reads `testkit/src/base-policy.yaml`, which allows no network access, and @@ -100,7 +119,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. (binaries apply to every endpoint under `name`) that `policy::Endpoint` configures (`.rest(Access::Full)`, `.credential_binding(provider)`, ...). Pass the result to `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` - for the CLI. + for the CLI. `policy::binaries_yaml(binaries)` writes a `binaries:` list + into policy YAML a test writes by hand. `testkit::PYTHON_BINARIES` names `python3` in `python_image()` images. - `cx.import_profile(profile)` to import a provider profile and return its `id`, and `cx.create_provider(name, type, credentials)` to create a @@ -108,6 +128,11 @@ fails. A killed test process skips cleanup, so names are unique to the run. credentials)` does both. - `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.client()` for SDK calls testkit does not wrap. +- In `tests/support`: `host-server` starts servers on the machine that runs + the tests, which sandboxes reach as `host_server_support::TEST_HOST`, and + `sandbox-clients` runs clients in a sandbox: `connect`, `http` and + `http_raw` return typed results, and `scripts` holds client scripts for + `run_python`. - `?` and `.context(..)` on I/O, JSON, YAML, SDK and gRPC results, which convert into `testkit::Error`. - `error.grpc_code()` for the gRPC status code (`testkit::GrpcCode`) of a diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index abfa8cf97b..7b8ebd90ab 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -30,7 +30,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } tempfile = "3" -tokio = { version = "1.43", features = ["macros", "process", "rt", "time"] } +tokio = { version = "1.43", features = ["io-util", "macros", "process", "rt", "sync", "time"] } tonic = "0.14" [target.'cfg(unix)'.dependencies] diff --git a/tests/testkit/testkit/src/cli.rs b/tests/testkit/testkit/src/cli.rs index ad597482db..2973ea045b 100644 --- a/tests/testkit/testkit/src/cli.rs +++ b/tests/testkit/testkit/src/cli.rs @@ -15,6 +15,7 @@ use std::time::Duration; use serde::de::DeserializeOwned; +use crate::cli_child::CliChild; use crate::error::{Error, Result, ResultExt}; use crate::gateway::{ENDPOINT_ENV, GATEWAY_ENV, Gateway, WORKSPACE_ENV}; use crate::output::Outcome; @@ -52,6 +53,18 @@ impl Cli { } } + /// `binary` in place of the CLI, pointed at an unregistered gateway. + #[cfg(test)] + pub(crate) fn with_binary(binary: &str) -> Self { + Self { + binary: PathBuf::from(binary), + gateway: Gateway::from_endpoint( + "http://127.0.0.1:1".to_string(), + "default".to_string(), + ), + } + } + /// Check that the CLI reaches the gateway: `openshell status` must report /// it connected. pub async fn check_connected(&self) -> Result { @@ -99,36 +112,11 @@ impl Cli { I: IntoIterator, S: AsRef, { - let args: Vec<_> = args - .into_iter() - .map(|arg| arg.as_ref().to_os_string()) - .collect(); - let command = std::iter::once(self.binary.as_os_str()) - .chain(args.iter().map(std::ffi::OsString::as_os_str)) - .map(|part| part.to_string_lossy()) - .collect::>() - .join(" "); - - let mut process = tokio::process::Command::new(&self.binary); - // Set first, so the gateway and workspace selection below wins over + let (command, mut process) = self.command(args); + // Set first, so the gateway and workspace selection wins over // anything the caller passes. - process - .args(&args) - .envs(env.iter().copied()) - .env("NO_COLOR", "1") - .env(WORKSPACE_ENV, &self.gateway.workspace) - .stdin(Stdio::null()) - .kill_on_drop(true); - // Select exactly the gateway the SDK connected to. - match &self.gateway.name { - Some(name) => process.env(GATEWAY_ENV, name), - None => process.env_remove(GATEWAY_ENV), - }; - if self.gateway.endpoint_from_env { - process.env(ENDPOINT_ENV, &self.gateway.endpoint); - } else { - process.env_remove(ENDPOINT_ENV); - } + process.envs(env.iter().copied()).stdin(Stdio::null()); + self.apply_env(&mut process); if let Some(dir) = dir { process.current_dir(dir); } @@ -140,12 +128,90 @@ impl Cli { .context(format!("run {command}"))?; Ok(CliOutput { command, - // A signal-terminated CLI has no code; report it as a failure. - exit_code: output.status.code().unwrap_or(-1), + exit_code: exit_code(output.status), stdout: String::from_utf8_lossy(&output.stdout).into_owned(), stderr: String::from_utf8_lossy(&output.stderr).into_owned(), }) } + + /// Start the CLI with `args` and keep it running, with stdin, stdout, + /// and stderr piped to the test. Use it for commands that stream or + /// read input, such as `sandbox connect`. + pub fn spawn(&self, args: I) -> Result + where + I: IntoIterator, + S: AsRef, + { + self.spawn_with_stdin(args, Stdio::piped()) + } + + /// Like [`Cli::spawn`], with `stdin` as the CLI's stdin, such as + /// [`Stdio::null`], a file, or a pseudo-terminal. Stdout and stderr are + /// still piped. + pub fn spawn_with_stdin(&self, args: I, stdin: impl Into) -> Result + where + I: IntoIterator, + S: AsRef, + { + let (command, mut process) = self.command(args); + process + .stdin(stdin) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + self.apply_env(&mut process); + CliChild::spawn(command, &mut process) + } + + /// The gateway and workspace the CLI is pointed at. + pub fn gateway(&self) -> &Gateway { + &self.gateway + } + + /// Point `process` at the CLI's gateway and workspace, as every CLI + /// invocation is. Use it for a program that runs the CLI itself, such as + /// `ssh` with the `ProxyCommand` from `sandbox ssh-config`. + pub fn apply_env(&self, process: &mut tokio::process::Command) { + process + .env("NO_COLOR", "1") + .env(WORKSPACE_ENV, &self.gateway.workspace); + // Select exactly the gateway the SDK connected to. + match &self.gateway.name { + Some(name) => process.env(GATEWAY_ENV, name), + None => process.env_remove(GATEWAY_ENV), + }; + if self.gateway.endpoint_from_env { + process.env(ENDPOINT_ENV, &self.gateway.endpoint); + } else { + process.env_remove(ENDPOINT_ENV); + } + } + + /// The command line for reports, and a process for it that is killed + /// when dropped. + fn command(&self, args: I) -> (String, tokio::process::Command) + where + I: IntoIterator, + S: AsRef, + { + let args: Vec<_> = args + .into_iter() + .map(|arg| arg.as_ref().to_os_string()) + .collect(); + let command = std::iter::once(self.binary.as_os_str()) + .chain(args.iter().map(std::ffi::OsString::as_os_str)) + .map(|part| part.to_string_lossy()) + .collect::>() + .join(" "); + let mut process = tokio::process::Command::new(&self.binary); + process.args(&args).kill_on_drop(true); + (command, process) + } +} + +/// The exit code of a finished CLI. A signal-terminated CLI has no code; +/// report it as a failure. +pub(crate) fn exit_code(status: std::process::ExitStatus) -> i32 { + status.code().unwrap_or(-1) } /// The characters the CLI's error reports draw around wrapped lines: the diff --git a/tests/testkit/testkit/src/cli_child.rs b/tests/testkit/testkit/src/cli_child.rs new file mode 100644 index 0000000000..be3bb3f630 --- /dev/null +++ b/tests/testkit/testkit/src/cli_child.rs @@ -0,0 +1,407 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A CLI process the test keeps running, for commands that stream output or +//! read input, such as `sandbox connect` or a piped `sandbox exec`. + +use std::process::ExitStatus; +use std::time::Duration; + +use tokio::io::{AsyncBufReadExt as _, AsyncRead, AsyncWriteExt as _, BufReader}; +use tokio::process::{Child, ChildStdin, Command}; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; + +use crate::cli::{CliOutput, exit_code}; +use crate::error::{Error, Result, ResultExt}; + +/// A running CLI started by [`crate::Cli::spawn`]. +/// +/// Stdout and stderr are read in the background as they arrive, so a CLI that +/// writes a lot never blocks on a full pipe while the test does something +/// else. Read them line by line with [`CliChild::stdout_line`] and +/// [`CliChild::stdout_until`] (and the `stderr` forms), or all at once with +/// [`CliChild::output`]. +/// +/// The process is killed when the handle is dropped, so a failing test +/// doesn't leave it running. +#[derive(Debug)] +pub struct CliChild { + command: String, + child: Child, + pid: u32, + stdin: Option, + stdout: Stream, + stderr: Stream, +} + +/// One output stream: the lines read so far and not yet taken. +#[derive(Debug)] +struct Stream { + name: &'static str, + lines: mpsc::UnboundedReceiver>, + reader: JoinHandle<()>, +} + +impl CliChild { + pub(crate) fn spawn(command: String, process: &mut Command) -> Result { + let mut child = process + .spawn() + .map_err(|error| Error::from_error(&error)) + .context(format!("start {command}"))?; + let pid = child + .id() + .ok_or_else(|| Error::msg(format!("{command} exited before it was watched")))?; + let stdout = Stream::read("stdout", child.stdout.take().expect("stdout is piped")); + let stderr = Stream::read("stderr", child.stderr.take().expect("stderr is piped")); + Ok(Self { + command, + stdin: child.stdin.take(), + child, + pid, + stdout, + stderr, + }) + } + + /// The command line, for messages. + pub fn command(&self) -> &str { + &self.command + } + + /// The process ID of the CLI. + pub fn pid(&self) -> u32 { + self.pid + } + + /// Write `bytes` to the CLI's stdin and flush it. The error is the + /// I/O error itself, so a test can accept `BrokenPipe` from a command that + /// exits before it reads all its input. + pub async fn write_stdin(&mut self, bytes: &[u8]) -> std::io::Result<()> { + let stdin = self.stdin.as_mut().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotConnected, + "stdin is closed or not piped", + ) + })?; + stdin.write_all(bytes).await?; + stdin.flush().await + } + + /// Close the CLI's stdin, so it reads end of file. + pub fn close_stdin(&mut self) { + self.stdin = None; + } + + /// The next line of stdout, without its line ending. Fails if none + /// arrives within `timeout` or stdout closes. + pub async fn stdout_line(&mut self, timeout: Duration) -> Result { + let command = &self.command; + self.stdout.next_line(command, timeout).await + } + + /// The next line of stderr, like [`CliChild::stdout_line`]. + pub async fn stderr_line(&mut self, timeout: Duration) -> Result { + let command = &self.command; + self.stderr.next_line(command, timeout).await + } + + /// Read stdout until a line matches `found`, and return the lines read, + /// the matching one last. Fails if no line matches within `timeout` or + /// stdout closes first. + pub async fn stdout_until( + &mut self, + timeout: Duration, + found: impl FnMut(&str) -> bool, + ) -> Result> { + let command = &self.command; + self.stdout.until(command, timeout, found).await + } + + /// Read stderr until a line matches `found`, like + /// [`CliChild::stdout_until`]. + pub async fn stderr_until( + &mut self, + timeout: Duration, + found: impl FnMut(&str) -> bool, + ) -> Result> { + let command = &self.command; + self.stderr.until(command, timeout, found).await + } + + /// Send a Unix signal, such as `libc::SIGTERM`, to the CLI process only. + #[cfg(unix)] + pub fn signal(&self, signal: i32) -> Result { + let pid = libc::pid_t::try_from(self.pid).map_err(Error::from_display)?; + // SAFETY: kill(2) takes plain integers and has no memory effects. + if unsafe { libc::kill(pid, signal) } != 0 { + let error = std::io::Error::last_os_error(); + return Err(Error::from_error(&error) + .context(format!("send signal {signal} to {}", self.command))); + } + Ok(()) + } + + /// Close stdin, kill the CLI, and wait for it to exit. Closing stdin ends + /// processes that inherited it, such as the `ssh` of `sandbox connect`. + pub async fn kill(&mut self) -> Result { + self.close_stdin(); + self.child + .kill() + .await + .map_err(|error| Error::from_error(&error)) + .context(format!("kill {}", self.command)) + } + + /// The CLI's exit code if it has exited, without waiting. + pub fn try_wait(&mut self) -> Result> { + Ok(self + .child + .try_wait() + .map_err(|error| Error::from_error(&error)) + .context(format!("check {}", self.command))? + .map(exit_code)) + } + + /// Wait for the CLI to exit and return its exit code. Fails if it is + /// still running after `timeout`; it then keeps running until killed or + /// dropped. + pub async fn wait(&mut self, timeout: Duration) -> Result { + Ok(exit_code(self.wait_status(timeout).await?)) + } + + /// Close stdin, wait for the CLI to exit, and return its exit code with + /// the output not yet read as lines. Fails if the CLI, or a process that + /// inherited its stdout or stderr, is still running after `timeout`. + pub async fn output(mut self, timeout: Duration) -> Result { + self.close_stdin(); + let deadline = tokio::time::Instant::now() + timeout; + let status = self.wait_status(timeout).await?; + let stdout = self.stdout.rest(&self.command, deadline).await?; + let stderr = self.stderr.rest(&self.command, deadline).await?; + Ok(CliOutput { + command: self.command, + exit_code: exit_code(status), + stdout: String::from_utf8_lossy(&stdout).into_owned(), + stderr: String::from_utf8_lossy(&stderr).into_owned(), + }) + } + + async fn wait_status(&mut self, timeout: Duration) -> Result { + tokio::time::timeout(timeout, self.child.wait()) + .await + .map_err(|_| Error::msg(format!("still running after {timeout:?}"))) + .context(format!("wait for {}", self.command))? + .map_err(|error| Error::from_error(&error)) + .context(format!("wait for {}", self.command)) + } +} + +impl Stream { + fn read(name: &'static str, pipe: impl AsyncRead + Unpin + Send + 'static) -> Self { + let (sender, lines) = mpsc::unbounded_channel(); + let reader = tokio::spawn(async move { + let mut pipe = BufReader::new(pipe); + loop { + let mut line = Vec::new(); + match pipe.read_until(b'\n', &mut line).await { + Ok(0) | Err(_) => break, + Ok(_) => { + if sender.send(line).is_err() { + break; + } + } + } + } + }); + Self { + name, + lines, + reader, + } + } + + async fn next_line(&mut self, command: &str, timeout: Duration) -> Result { + match tokio::time::timeout(timeout, self.lines.recv()).await { + Ok(Some(line)) => Ok(trim_line_ending(&line)), + Ok(None) => Err(Error::msg(format!("{} of {command} closed", self.name))), + Err(_) => Err(Error::msg(format!( + "no line on {} of {command} within {timeout:?}", + self.name + ))), + } + } + + async fn until( + &mut self, + command: &str, + timeout: Duration, + mut found: impl FnMut(&str) -> bool, + ) -> Result> { + let mut read = Vec::new(); + let outcome = tokio::time::timeout(timeout, async { + while let Some(line) = self.lines.recv().await { + let line = trim_line_ending(&line); + let matched = found(&line); + read.push(line); + if matched { + return true; + } + } + false + }) + .await; + match outcome { + Ok(true) => Ok(read), + Ok(false) => Err(Error::msg(format!( + "{} of {command} closed before the expected line; read {read:?}", + self.name + ))), + Err(_) => Err(Error::msg(format!( + "no expected line on {} of {command} within {timeout:?}; read {read:?}", + self.name + ))), + } + } + + /// Everything not yet taken, once the pipe has closed. + async fn rest(&mut self, command: &str, deadline: tokio::time::Instant) -> Result> { + tokio::time::timeout_at(deadline, &mut self.reader) + .await + .map_err(|_| { + Error::msg(format!( + "{} of {command} is still open after the CLI exited", + self.name + )) + })? + .map_err(|error| Error::from_error(&error))?; + let mut bytes = Vec::new(); + while let Ok(line) = self.lines.try_recv() { + bytes.extend(line); + } + Ok(bytes) + } +} + +impl Drop for Stream { + fn drop(&mut self) { + self.reader.abort(); + } +} + +fn trim_line_ending(line: &[u8]) -> String { + let line = line.strip_suffix(b"\n").unwrap_or(line); + let line = line.strip_suffix(b"\r").unwrap_or(line); + String::from_utf8_lossy(line).into_owned() +} + +#[cfg(all(test, unix))] +mod tests { + use std::time::Duration; + + use crate::cli::Cli; + + const TIMEOUT: Duration = Duration::from_secs(10); + + /// `sh` in place of the CLI, so the tests need no gateway. + fn sh() -> Cli { + Cli::with_binary("/bin/sh") + } + + #[tokio::test] + async fn reads_lines_and_writes_stdin() { + let mut child = sh() + .spawn([ + "-c", + "echo ready; echo warning >&2; read line; echo \"got=$line\"; exit 3", + ]) + .unwrap(); + assert_eq!(child.stdout_line(TIMEOUT).await.unwrap(), "ready"); + assert_eq!(child.stderr_line(TIMEOUT).await.unwrap(), "warning"); + child.write_stdin(b"hello\n").await.unwrap(); + let read = child + .stdout_until(TIMEOUT, |line| line.starts_with("got=")) + .await + .unwrap(); + assert_eq!(read, ["got=hello"]); + assert_eq!(child.wait(TIMEOUT).await.unwrap(), 3); + } + + #[tokio::test] + async fn output_keeps_unread_bytes_exactly() { + let mut child = sh() + .spawn(["-c", "echo first; printf 'a\\r\\nb'; cat >&2"]) + .unwrap(); + assert_eq!(child.stdout_line(TIMEOUT).await.unwrap(), "first"); + child.write_stdin(b"input").await.unwrap(); + let output = child.output(TIMEOUT).await.unwrap(); + assert_eq!(output.exit_code, 0); + assert_eq!(output.stdout, "a\r\nb"); + assert_eq!(output.stderr, "input"); + } + + #[tokio::test] + async fn until_reports_what_it_read_when_the_stream_closes() { + let mut child = sh().spawn(["-c", "echo one; echo two"]).unwrap(); + let error = child + .stdout_until(TIMEOUT, |line| line == "three") + .await + .unwrap_err() + .to_string(); + assert!(error.contains("closed before the expected line"), "{error}"); + assert!(error.contains(r#"["one", "two"]"#), "{error}"); + } + + #[tokio::test] + async fn signal_reaches_the_process() { + let mut child = sh() + .spawn([ + "-c", + "trap 'exit 42' TERM; echo ready; while :; do sleep 0.1; done", + ]) + .unwrap(); + assert_eq!(child.stdout_line(TIMEOUT).await.unwrap(), "ready"); + assert_eq!(child.try_wait().unwrap(), None); + child.signal(libc::SIGTERM).unwrap(); + assert_eq!(child.wait(TIMEOUT).await.unwrap(), 42); + } + + /// `sandbox connect` hands its stdin to an `ssh` child, so killing the CLI + /// must close stdin for that child to see end of file and exit. A + /// background job reads `/dev/null` unless given stdin, hence fd 3. + #[tokio::test] + async fn kill_closes_stdin_for_processes_that_inherited_it() { + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("eof"); + let mut child = sh() + .spawn([ + "-c", + "exec 3<&0; (cat <&3 >/dev/null; touch \"$1\") & echo ready; wait", + "sh", + marker.to_str().unwrap(), + ]) + .unwrap(); + assert_eq!(child.stdout_line(TIMEOUT).await.unwrap(), "ready"); + child.kill().await.unwrap(); + let deadline = tokio::time::Instant::now() + TIMEOUT; + while !marker.exists() { + assert!( + tokio::time::Instant::now() < deadline, + "the process that inherited stdin did not read end of file" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + #[tokio::test] + async fn wait_times_out_on_a_running_process() { + let mut child = sh().spawn(["-c", "sleep 30"]).unwrap(); + let error = child + .wait(Duration::from_millis(100)) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("still running"), "{error}"); + child.kill().await.unwrap(); + } +} diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 47221f68ec..a937ed069d 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -14,6 +14,7 @@ use rand::Rng; use crate::cleanup::{CLEANUP_TIMEOUT, Cleanup, Key}; use crate::cli::{Cli, CliOutput}; +use crate::cli_child::CliChild; use crate::cli_sandbox::CliSandboxBuilder; use crate::error::{Error, Result, ResultExt}; use crate::output::Outcome; @@ -79,6 +80,30 @@ impl Context { self.handles.cli.run_with_env(None, env, args).await } + /// Start the CLI with `args` and keep it running, with stdin, stdout, + /// and stderr piped to the test. See [`CliChild`]. + pub fn cli_spawn(&self, args: I) -> Result + where + I: IntoIterator, + S: AsRef, + { + self.handles.cli.spawn(args) + } + + /// Like [`Context::cli_spawn`], with `stdin` as the CLI's stdin, such + /// as [`std::process::Stdio::null`], a file, or a pseudo-terminal. + pub fn cli_spawn_with_stdin( + &self, + args: I, + stdin: impl Into, + ) -> Result + where + I: IntoIterator, + S: AsRef, + { + self.handles.cli.spawn_with_stdin(args, stdin) + } + /// The CLI that [`Context::cli`] runs, for code that can't borrow the /// context, such as a cleanup hook. pub fn cli_handle(&self) -> Cli { @@ -101,12 +126,18 @@ impl Context { } /// Start describing a sandbox named `cx.name(suffix)`. + /// + /// # Panics + /// + /// If the name is longer than the gateway allows; see + /// [`MAX_SANDBOX_NAME_LEN`]. + #[track_caller] pub fn sandbox(&self, suffix: &str) -> SandboxBuilder { SandboxBuilder { handles: self.handles.clone(), policy: None, spec: SandboxSpec { - name: Some(self.name(suffix)), + name: Some(self.sandbox_name(suffix)), ..SandboxSpec::default() }, } @@ -116,8 +147,20 @@ impl Context { /// `openshell` CLI creates. Use it for what only `sandbox create` /// offers, such as a `--policy` file or `--no-keep`; prefer /// [`Context::sandbox`] otherwise. + /// + /// # Panics + /// + /// As [`Context::sandbox`]. + #[track_caller] pub fn cli_sandbox(&self, suffix: &str) -> CliSandboxBuilder { - CliSandboxBuilder::new(self.handles.clone(), self.name(suffix)) + CliSandboxBuilder::new(self.handles.clone(), self.sandbox_name(suffix)) + } + + #[track_caller] + fn sandbox_name(&self, suffix: &str) -> String { + let name = self.name(suffix); + check_sandbox_name(&name); + name } /// Track a sandbox the test is about to create through the CLI, so cleanup @@ -127,7 +170,13 @@ impl Context { /// Cleanup deletes the sandbox and waits until it is gone, at the point /// in the cleanup order where it was first tracked. Tracking a sandbox /// again, for example by adopting it, doesn't move it. + /// + /// # Panics + /// + /// If `name` is longer than [`MAX_SANDBOX_NAME_LEN`]. + #[track_caller] pub fn track(&self, name: &str) { + check_sandbox_name(name); self.handles.track(name); } @@ -321,6 +370,16 @@ impl SandboxBuilder { self } + /// Attach these providers to the sandbox. + pub fn providers(mut self, providers: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.spec.providers = providers.into_iter().map(Into::into).collect(); + self + } + /// Create the sandbox with this policy instead of the gateway's default. pub fn policy(mut self, policy: proto::SandboxPolicy) -> Self { self.policy = Some(policy); @@ -466,6 +525,19 @@ impl ExecOutput { Ok(self) } + /// The last line of stdout that parses as JSON, such as the result a + /// client script prints after its other output. + pub fn last_json(&self) -> Result { + last_json_line(&self.stdout).ok_or_else(|| { + Error::msg(format!( + "{} printed no JSON line; stdout: {}; stderr: {}", + self.description, + self.stdout.trim(), + self.stderr.trim() + )) + }) + } + fn outcome(&self) -> Outcome<'_> { Outcome { what: &self.description, @@ -476,6 +548,26 @@ impl ExecOutput { } } +/// Only objects and arrays count, so a line that happens to parse, such as +/// a number, isn't taken for the result. +fn last_json_line(stdout: &str) -> Option { + stdout.lines().rev().find_map(|line| { + serde_json::from_str(line.trim()) + .ok() + .filter(|value: &serde_json::Value| value.is_object() || value.is_array()) + }) +} + +/// A script's first line that isn't blank or a license header, which names +/// it in reports instead of its full text. +fn script_summary(script: &str) -> &str { + script + .lines() + .map(str::trim) + .find(|line| !line.is_empty() && !line.starts_with("# SPDX-")) + .unwrap_or_default() +} + impl Sandbox { pub fn name(&self) -> &str { &self.name @@ -649,6 +741,41 @@ impl Sandbox { }) } + /// Run `script` with `python3 -c` and `args`, as in a sandbox from + /// [`crate::python_image`]. Fails unless it exits 0. Read a JSON result + /// with [`ExecOutput::last_json`]. + pub async fn run_python(&self, script: &str, args: &[&str]) -> Result { + let mut command = vec!["python3", "-c", script]; + command.extend_from_slice(args); + let mut output = self.exec(command).await?; + output.description = format!( + "python3 script {:?} with {args:?} in sandbox {}", + script_summary(script), + self.name + ); + output.success() + } + + /// The sandbox's policy revisions, from `policy list --output json`. + pub async fn policy_revisions(&self) -> Result { + self.handles + .cli + .run(["policy", "list", &self.name, "--output", "json"]) + .await? + .success()? + .json() + } + + /// The sandbox's active policy, from `policy get --full --output json`. + pub async fn active_policy(&self) -> Result { + self.handles + .cli + .run(["policy", "get", &self.name, "--full", "--output", "json"]) + .await? + .success()? + .json() + } + /// Run `command` every 250 ms until `ready` accepts its output, and return /// that output. An exec error fails at once; a timeout reports the last /// output. @@ -784,6 +911,21 @@ impl Context { } } +/// The longest sandbox name the gateway accepts. With `test--`, a +/// sandbox suffix can have at most 5 characters. +pub const MAX_SANDBOX_NAME_LEN: usize = 19; + +/// Panic, at the test's call, on a sandbox name the gateway would reject, so +/// the test fails with this message instead of the gateway's. +#[track_caller] +fn check_sandbox_name(name: &str) { + assert!( + name.len() <= MAX_SANDBOX_NAME_LEN, + "sandbox name {name:?} has {} characters; the gateway allows {MAX_SANDBOX_NAME_LEN}, so a suffix after `test--` can have at most 5", + name.len() + ); +} + /// Short, lowercase, and DNS-label safe, so names work on every driver. fn generate_run_id() -> String { const ALPHABET: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 86f28e2599..e86b7c3b14 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -28,6 +28,7 @@ mod cleanup; mod cli; +mod cli_child; mod cli_sandbox; mod context; mod error; @@ -45,10 +46,11 @@ mod yaml_file; pub use cleanup::CLEANUP_TIMEOUT; pub use cli::{CLI_TIMEOUT, Cli, CliOutput}; +pub use cli_child::CliChild; pub use cli_sandbox::CliSandboxBuilder; pub use context::{ - Context, DELETE_TIMEOUT, EXEC_TIMEOUT, ExecOutput, READY_TIMEOUT, STOP_TIMEOUT, Sandbox, - SandboxBuilder, + Context, DELETE_TIMEOUT, EXEC_TIMEOUT, ExecOutput, MAX_SANDBOX_NAME_LEN, READY_TIMEOUT, + STOP_TIMEOUT, Sandbox, SandboxBuilder, }; pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index ab7be7cae3..c62c930528 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -75,6 +75,18 @@ pub fn to_yaml(policy: &proto::SandboxPolicy) -> Result { .context("serialize the policy") } +/// `binaries` as the flow-style list a network policy's `binaries:` key +/// takes in YAML, such as `[{path: "/usr/bin/python*"}]`. Use it in policy +/// YAML a test writes by hand, for example with +/// `binaries_yaml(PYTHON_BINARIES)`. +pub fn binaries_yaml(binaries: &[&str]) -> String { + let paths: Vec = binaries + .iter() + .map(|path| format!("{{path: {}}}", serde_json::Value::from(*path))) + .collect(); + format!("[{}]", paths.join(", ")) +} + /// Settings for an endpoint from [`allow`], named as in policy YAML. pub trait Endpoint { /// `protocol: rest` with `access`. diff --git a/tests/tmachine/artifacts.nix b/tests/tmachine/artifacts.nix index ef1be7ed98..5b605169b3 100644 --- a/tests/tmachine/artifacts.nix +++ b/tests/tmachine/artifacts.nix @@ -106,10 +106,9 @@ let # rootless guest; the other tests in the binary pass. Nextest archive # filters cannot select individual tests, so keep the complete binary here. "proxy_egress_pipeline" - # Conformance covers stop/start workspace preservation and deletion while - # stopped. The remaining canonical-main, TTY, attachment replay, and - # no-keep cases still need migration. Nextest archive filters cannot select - # individual tests, so keep the complete binary in the follow-up bucket. + # Holds only the on-failure restart test until #5507; the rest moved to + # testkit. Nextest archive filters cannot select individual tests, so keep + # the complete binary in the follow-up bucket. "sandbox_lifecycle" # Needs a prebuilt musl DNS probe in guest artifact mode; tracked in #3009. "transparent_tcp"