diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index cb9433d08e..e35368641b 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -14,6 +14,47 @@ binaries. The driver embeds libkrun, libkrunfw, the guest OCI unpacker, the portable guest sandbox, and the custom kernel runtime. Each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir. +Guest init creates missing `/dev/fd` and standard-stream links after mounting +procfs and devtmpfs, before using Bash process substitution. Existing links to +the expected procfs paths, or entries that resolve to the same files, stay +intact. Conflicting entries and unrelated dangling links fail boot with a +diagnostic rather than being replaced. + +## Lifecycle extension requirements + +Lifecycle extensions declare requirements through `LaunchPlan::require_backend` +and `require_backend_feature`. An explicit backend stays selected; incompatible +requirements fail before the workload launcher starts. Without an explicit +requirement, the driver chooses a backend that supports the requested features. + +The VM driver resolves the plan after its lifecycle extensions contribute +requirements, then checks the selected backend. The compute driver's +`GetCapabilities` RPC supplies the gateway with protocol and resource request +support at startup. + +| Feature | libkrun | QEMU | +|---|---|---| +| Guest init drop-ins | Supported | Supported | +| External kernel image | Unsupported | Supported for CPU-only and GPU VMs | +| PCI passthrough | Unsupported | Requires an assigned GPU device | +| Host directory mounts | Supported through virtiofs | Unsupported | + +For example, a VM driver with a registered lifecycle extension can set +`plan.kernel_image` for a CPU-only sandbox. The +resolver selects QEMU with a vsock CID and ordinary CPU sizing, without +allocating a GPU. Requiring libkrun for that same plan fails with an +external-kernel diagnostic. Kernel paths remain extension-owned; sandbox +requests cannot supply them. + +The QEMU launcher requires a Linux x86_64 host with KVM and vhost-vsock. +Supply a kernel through a lifecycle extension or place `vmlinux` in the +directory configured by `OPENSHELL_VM_RUNTIME_DIR`. The driver checks the +selected kernel's x86_64 ELF or Linux bzImage header before starting the +workload launcher. QEMU validates the complete image and requires a PVH entry +note for an uncompressed Linux ELF kernel. Kernel security and boot-driver +requirements still apply. Authenticated runtime admission confirms +enforcement before workload code runs. Image preparation uses libkrun. + ## How it fits together Caller driver config is disabled by default, including upload workflows diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 37950675b0..97a99e1078 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -31,6 +31,31 @@ mount_initial_fs() { mount -t tmpfs tmpfs /tmp 2>/dev/null || true mount -t tmpfs tmpfs /run 2>/dev/null || true mount -t devtmpfs devtmpfs /dev 2>/dev/null || true + setup_fd_links /dev +} + +setup_fd_links() { + local dev_dir="$1" + local name target + for name in fd stdin stdout stderr; do + case "$name" in + fd) target=/proc/self/fd ;; + stdin) target=/proc/self/fd/0 ;; + stdout) target=/proc/self/fd/1 ;; + stderr) target=/proc/self/fd/2 ;; + esac + if [[ -L "$dev_dir/$name" ]] && [[ "$(readlink "$dev_dir/$name")" == "$target" ]]; then + continue + fi + if [[ -e "$dev_dir/$name" || -L "$dev_dir/$name" ]]; then + if [[ ! "$dev_dir/$name" -ef "$target" ]]; then + printf 'guest init: %s must refer to %s\n' "$dev_dir/$name" "$target" >&2 + return 1 + fi + else + ln -s "$target" "$dev_dir/$name" + fi + done } bind_mount_into_newroot() { diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index f7946c34c2..7cd7914a63 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1936,16 +1936,7 @@ impl VmDriver { None }; - let needs_qemu = is_gpu; - - let mut plan = - match self.build_vm_launch_plan(&sandbox.id, needs_qemu, is_gpu, gpu_bdf.clone()) { - Ok(plan) => plan, - Err(err) => { - self.release_gpu(&sandbox.id); - return Err(err); - } - }; + let mut plan = self.build_vm_launch_plan(gpu_bdf.clone()); if let Err(err) = self .lifecycle_extensions @@ -1971,48 +1962,38 @@ impl VmDriver { }); } - // Resolve and validate the backend from the requirements that - // `configure_launch` extensions contributed. After this point the - // plan's backend, sizing, and host allocations are final; the - // `before_launch` hook below may still mutate - // `plan.env` and `plan.guest_init_dropins` and may abort the launch, - // but it MUST NOT change `plan.backend`, `plan.required_backends`, - // or `plan.required_backend_features` -- those are enforced as a - // documented trait contract, not a runtime check. - if let Err(err) = - self.resolve_launch_plan_backend(&sandbox.id, is_gpu, gpu_bdf.clone(), &mut plan) - { - self.lifecycle_extensions - .after_launch_failed( - &sandbox, - &state_dir, - LaunchAbortReason::BeforeLaunchHookFailed, - ) - .await; - self.release_gpu(&sandbox.id); - return Err(err); - } - - if let Err(err) = Self::validate_launch_plan_backend(is_gpu, &plan).and_then(|()| { - // Cannot fail today: `validate_sandbox` already ran and non-GPU - // QEMU is blocked above. Keeps virtiofs mounts off QEMU once that - // guard is lifted. - validate_vm_driver_mounts( - &driver_config.mounts, - &self.config, - plan.backend == VmBackend::Qemu, - ) - }) { - self.lifecycle_extensions - .after_launch_failed( - &sandbox, - &state_dir, - LaunchAbortReason::BeforeLaunchHookFailed, - ) - .await; - self.release_gpu(&sandbox.id); - return Err(err); - } + // Resolve extension requirements before before_launch prepares the host. + let kernel_image = self + .resolve_launch_plan_backend(is_gpu, &mut plan) + .and_then(|()| Self::validate_launch_plan_backend(gpu_bdf.as_deref(), &plan)) + .and_then(|()| { + plan.backend + .validate_host() + .map_err(Status::failed_precondition)?; + // An extension can select QEMU after request validation. + validate_vm_driver_mounts( + &driver_config.mounts, + &self.config, + plan.backend == VmBackend::Qemu, + )?; + plan.backend + .resolve_kernel_image(plan.kernel_image.as_deref()) + .map_err(Status::failed_precondition) + }); + let kernel_image = match kernel_image { + Ok(path) => path, + Err(err) => { + self.lifecycle_extensions + .after_launch_failed( + &sandbox, + &state_dir, + LaunchAbortReason::BeforeLaunchHookFailed, + ) + .await; + self.release_gpu(&sandbox.id); + return Err(err); + } + }; if let Err(err) = self .lifecycle_extensions @@ -2154,7 +2135,7 @@ impl VmDriver { command.arg("--vm-console-output").arg(&console_output); command.arg("--vm-vcpus").arg(plan.vcpus.to_string()); command.arg("--vm-mem-mib").arg(plan.mem_mib.to_string()); - if let Some(kernel_image) = &plan.kernel_image { + if let Some(kernel_image) = &kernel_image { command.arg("--vm-kernel-image").arg(kernel_image); } @@ -2991,9 +2972,7 @@ impl VmDriver { #[allow(clippy::result_large_err)] fn resolve_launch_plan_backend( &self, - sandbox_id: &str, is_gpu: bool, - gpu_bdf: Option, plan: &mut LaunchPlan, ) -> Result<(), Status> { if plan.kernel_image.is_some() { @@ -3003,154 +2982,97 @@ impl VmDriver { plan.require_backend_feature(BackendFeature::GuestInitDropins); } - if plan.required_backends.contains(&VmBackend::Qemu) - || plan.backend == VmBackend::Qemu - || plan + if let Some(required) = plan.required_backends.first() { + if plan + .required_backends + .iter() + .any(|backend| backend != required) + { + return Err(Status::failed_precondition( + "VM lifecycle extensions requested conflicting VM backends", + )); + } + plan.backend = required.clone(); + } else if plan + .required_backend_features + .iter() + .any(|feature| !feature.is_supported_by(&plan.backend)) + && plan .required_backend_features .iter() - .any(|feature| feature.requires_qemu()) + .all(|feature| feature.is_supported_by(&VmBackend::Qemu)) { - self.configure_qemu_launch_plan(sandbox_id, is_gpu, gpu_bdf, plan)?; + plan.backend = VmBackend::Qemu; } - Ok(()) - } - - // Keep the fallible shape used by launch-plan resolution: driver-local - // backends may add allocation failures here without changing callers. - #[allow(clippy::result_large_err, clippy::unnecessary_wraps)] - fn configure_qemu_launch_plan( - &self, - _sandbox_id: &str, - is_gpu: bool, - gpu_bdf: Option, - plan: &mut LaunchPlan, - ) -> Result<(), Status> { - plan.backend = VmBackend::Qemu; - if is_gpu { - plan.vcpus = self.config.gpu_vcpus; - plan.mem_mib = self.config.gpu_mem_mib; - } - if plan.gpu_bdf.is_none() { - plan.gpu_bdf = gpu_bdf; - } - if plan.vsock_cid.is_some() { - return Ok(()); + if plan.backend == VmBackend::Qemu { + if is_gpu { + plan.vcpus = self.config.gpu_vcpus; + plan.mem_mib = self.config.gpu_mem_mib; + } + plan.vsock_cid.get_or_insert_with(allocate_vsock_cid); } - plan.vsock_cid = Some(allocate_vsock_cid()); + Ok(()) } #[allow(clippy::result_large_err)] - fn validate_launch_plan_backend(is_gpu: bool, plan: &LaunchPlan) -> Result<(), Status> { - // NOTE: this guard exists because the non-GPU QEMU launch path - // (PCI device transport, VFIO root port wiring) has not landed - // yet. Until then, even though the resolver will happily promote - // a plan to QEMU when an extension requires `PciPassthrough` or - // `ExternalKernelImage`, the launch itself is blocked here so we - // don't spawn a QEMU instance with no concrete device backing. - // Remove this guard once the non-GPU QEMU launch path supports - // emitting `pcie-root-port` + `vfio-pci` for arbitrary device - // descriptors. - if plan.backend == VmBackend::Qemu && !is_gpu { - let offending_feature = plan - .required_backend_features - .iter() - .find(|feature| feature.requires_qemu()) - .map_or("(explicit QEMU backend requirement)", |feature| { - feature.as_str() - }); - return Err(Status::failed_precondition(format!( - "vm lifecycle extension required '{offending_feature}', which resolves to the QEMU backend, \ - but non-GPU QEMU launch is not yet supported (pending PCI device transport)" - ))); - } - if plan.backend != VmBackend::Qemu && is_gpu { - return Err(Status::failed_precondition( - "GPU sandbox launch requires the QEMU backend", - )); - } - if plan.required_backends.contains(&VmBackend::Libkrun) - && plan.required_backends.contains(&VmBackend::Qemu) - { - return Err(Status::failed_precondition( - "VM lifecycle extensions requested conflicting VM backends", - )); - } - if plan.required_backends.contains(&VmBackend::Libkrun) - && plan.backend != VmBackend::Libkrun - { + fn validate_launch_plan_backend( + assigned_gpu: Option<&str>, + plan: &LaunchPlan, + ) -> Result<(), Status> { + if plan.gpu_bdf.as_deref() != assigned_gpu { return Err(Status::failed_precondition( - "VM lifecycle extension requires the libkrun backend", + "VM lifecycle extensions must preserve the driver's GPU assignment", )); } - if plan.required_backends.contains(&VmBackend::Qemu) && plan.backend != VmBackend::Qemu { + if assigned_gpu.is_some() && plan.backend != VmBackend::Qemu { return Err(Status::failed_precondition( - "VM lifecycle extension requires the QEMU backend", + "GPU sandbox launch requires the QEMU backend", )); } - if plan.backend != VmBackend::Qemu - && let Some(feature) = plan - .required_backend_features - .iter() - .find(|feature| feature.requires_qemu()) + if let Some(feature) = plan + .required_backend_features + .iter() + .find(|feature| !feature.is_supported_by(&plan.backend)) { return Err(Status::failed_precondition(format!( - "VM backend feature '{}' requires a VM backend with PCI-style launch support", + "VM backend {:?} does not support feature '{}'", + plan.backend, feature.as_str() ))); } - if plan.kernel_image.is_some() && plan.backend != VmBackend::Qemu { + if plan + .required_backend_features + .contains(&BackendFeature::PciPassthrough) + && assigned_gpu.is_none() + { return Err(Status::failed_precondition( - "selected kernel image requires a VM backend that supports external kernel images", + "PCI passthrough requires an assigned GPU device; arbitrary PCI devices are not supported", )); } - if let Some(kernel_image) = &plan.kernel_image - && !kernel_image.is_file() - { - return Err(Status::failed_precondition(format!( - "selected kernel image does not exist: {}", - kernel_image.display() - ))); - } Ok(()) } - // Keep the fallible shape used by provisioning and lifecycle tests even - // though NIC/subnet allocation no longer introduces a failure today. - #[allow(clippy::result_large_err, clippy::unnecessary_wraps)] - fn build_vm_launch_plan( - &self, - _sandbox_id: &str, - needs_qemu: bool, - is_gpu: bool, - gpu_bdf: Option, - ) -> Result { - if !needs_qemu { - return Ok(LaunchPlan { - backend: VmBackend::Libkrun, - vcpus: self.config.vcpus, - mem_mib: self.config.mem_mib, - required_backends: Vec::new(), - required_backend_features: Vec::new(), - kernel_profile: None, - kernel_image: None, - gpu_bdf: None, - vsock_cid: None, - guest_init_dropins: Vec::new(), - env: Vec::new(), - }); - } - - let vsock_cid = allocate_vsock_cid(); - let (vcpus, mem_mib) = if is_gpu { - (self.config.gpu_vcpus, self.config.gpu_mem_mib) + fn build_vm_launch_plan(&self, gpu_bdf: Option) -> LaunchPlan { + let (backend, vcpus, mem_mib, vsock_cid) = if gpu_bdf.is_some() { + ( + VmBackend::Qemu, + self.config.gpu_vcpus, + self.config.gpu_mem_mib, + Some(allocate_vsock_cid()), + ) } else { - (self.config.vcpus, self.config.mem_mib) + ( + VmBackend::Libkrun, + self.config.vcpus, + self.config.mem_mib, + None, + ) }; - Ok(LaunchPlan { - backend: VmBackend::Qemu, + LaunchPlan { + backend, vcpus, mem_mib, required_backends: Vec::new(), @@ -3158,10 +3080,10 @@ impl VmDriver { kernel_profile: None, kernel_image: None, gpu_bdf, - vsock_cid: Some(vsock_cid), + vsock_cid, guest_init_dropins: Vec::new(), env: Vec::new(), - }) + } } async fn ensure_provisioning_active(&self, sandbox_id: &str) -> Result<(), Status> { @@ -9449,9 +9371,7 @@ mod tests { id: "sb-post-overlay".to_string(), ..Default::default() }; - let mut plan = driver - .build_vm_launch_plan(&sandbox.id, false, false, None) - .unwrap(); + let mut plan = driver.build_vm_launch_plan(None); let provisioning = tracing::info_span!("vm.provision"); async { @@ -13636,9 +13556,11 @@ while :; do sleep 0.05; done"#, fn empty_registry_keeps_non_gpu_sandbox_on_libkrun() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let plan = driver - .build_vm_launch_plan("sandbox-x", false, false, None) - .expect("plan should build"); + let mut plan = driver.build_vm_launch_plan(None); + driver + .resolve_launch_plan_backend(false, &mut plan) + .unwrap(); + VmDriver::validate_launch_plan_backend(None, &plan).unwrap(); assert_eq!(plan.backend, VmBackend::Libkrun); assert_eq!(plan.vcpus, 2); @@ -13658,90 +13580,121 @@ while :; do sleep 0.05; done"#, fn gpu_sandbox_uses_qemu_backend_and_gpu_sizing() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let plan = driver - .build_vm_launch_plan("sandbox-gpu", true, true, Some("0000:01:00.0".to_string())) - .expect("gpu plan should build"); + let assigned = "0000:01:00.0"; + let mut plan = driver.build_vm_launch_plan(Some(assigned.to_string())); + driver.resolve_launch_plan_backend(true, &mut plan).unwrap(); + VmDriver::validate_launch_plan_backend(Some(assigned), &plan).unwrap(); assert_eq!(plan.backend, VmBackend::Qemu); assert_eq!(plan.vcpus, 8); assert_eq!(plan.mem_mib, 16384); - assert_eq!(plan.gpu_bdf.as_deref(), Some("0000:01:00.0")); + assert_eq!(plan.gpu_bdf.as_deref(), Some(assigned)); assert!(plan.vsock_cid.is_some()); + + for replacement in [None, Some("0000:02:00.0".to_string())] { + plan.gpu_bdf = replacement; + driver.resolve_launch_plan_backend(true, &mut plan).unwrap(); + let error = VmDriver::validate_launch_plan_backend(Some(assigned), &plan) + .expect_err("the launch plan must keep the device reserved by the driver"); + assert!(error.message().contains("GPU assignment")); + } } #[test] - fn launch_plan_rejects_external_kernel_on_unsupported_backend() { - let mut plan = LaunchPlan { - backend: VmBackend::Libkrun, - vcpus: 2, - mem_mib: 2048, - required_backends: Vec::new(), - required_backend_features: Vec::new(), - kernel_profile: None, - kernel_image: Some(PathBuf::from("/tmp/openshell-test-kernel")), - gpu_bdf: None, - vsock_cid: None, - guest_init_dropins: Vec::new(), - env: Vec::new(), - }; - - let err = VmDriver::validate_launch_plan_backend(false, &plan) - .expect_err("external kernels require a compatible backend"); - assert_eq!(err.code(), Code::FailedPrecondition); - assert!(err.message().contains("external kernel images")); + fn external_kernel_selects_cpu_only_qemu_unless_libkrun_is_required() { + let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + let mut plan = driver.build_vm_launch_plan(None); + let kernel = tempfile::NamedTempFile::new().unwrap(); + let mut header = [0u8; 64]; + header[..6].copy_from_slice(b"\x7fELF\x02\x01"); + header[18] = 62; // EM_X86_64; a header fixture, not a bootable guest. + std::fs::write(kernel.path(), header).unwrap(); + plan.kernel_image = Some(kernel.path().to_path_buf()); + plan.require_backend(VmBackend::Libkrun); + driver + .resolve_launch_plan_backend(false, &mut plan) + .unwrap(); + assert_eq!(plan.backend, VmBackend::Libkrun); + let error = VmDriver::validate_launch_plan_backend(None, &plan) + .expect_err("an explicit libkrun requirement rules out external kernels"); + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("external-kernel-image")); - let base = unique_temp_dir(); - std::fs::create_dir_all(&base).unwrap(); - let kernel = base.join("vmlinux"); - std::fs::write(&kernel, b"kernel").unwrap(); - plan.backend = VmBackend::Qemu; - plan.kernel_image = Some(kernel); - VmDriver::validate_launch_plan_backend(true, &plan).expect("existing kernel is accepted"); - let _ = std::fs::remove_dir_all(base); + plan.required_backends.clear(); + driver + .resolve_launch_plan_backend(false, &mut plan) + .unwrap(); + assert_eq!(plan.backend, VmBackend::Qemu); + assert_eq!(plan.vcpus, driver.config.vcpus); + assert_eq!(plan.mem_mib, driver.config.mem_mib); + assert!(plan.gpu_bdf.is_none()); + assert!(plan.vsock_cid.is_some()); + VmDriver::validate_launch_plan_backend(None, &plan) + .expect("CPU-only external kernel is accepted without PCI passthrough"); + assert_eq!( + plan.backend + .resolve_kernel_image(plan.kernel_image.as_deref()) + .unwrap() + .as_deref(), + Some(kernel.path()) + ); } #[test] fn backend_feature_requirements_select_qemu_launch_plan() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let mut plan = driver - .build_vm_launch_plan("sandbox-vfio", false, false, None) - .expect("base plan should build"); + let mut plan = driver.build_vm_launch_plan(None); plan.require_backend_feature(BackendFeature::PciPassthrough); driver - .resolve_launch_plan_backend("sandbox-vfio", false, None, &mut plan) + .resolve_launch_plan_backend(false, &mut plan) .expect("backend feature should resolve"); assert_eq!(plan.backend, VmBackend::Qemu); assert!(plan.vsock_cid.is_some()); + let error = VmDriver::validate_launch_plan_backend(None, &plan) + .expect_err("a PCI feature needs an assigned device"); + assert!(error.message().contains("assigned GPU device")); } #[test] fn explicit_backend_requirement_selects_qemu_launch_plan() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let mut plan = driver - .build_vm_launch_plan("sandbox-qemu", false, false, None) - .expect("base plan should build"); + let mut plan = driver.build_vm_launch_plan(None); plan.require_backend(VmBackend::Qemu); driver - .resolve_launch_plan_backend("sandbox-qemu", false, None, &mut plan) + .resolve_launch_plan_backend(false, &mut plan) .expect("backend requirement should resolve"); assert_eq!(plan.backend, VmBackend::Qemu); assert!(plan.vsock_cid.is_some()); + VmDriver::validate_launch_plan_backend(None, &plan) + .expect("CPU-only QEMU can satisfy the backend requirements"); + } + + #[test] + fn conflicting_backend_requirements_fail_before_allocating_a_cid() { + let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + let mut plan = driver.build_vm_launch_plan(None); + plan.require_backend(VmBackend::Libkrun); + plan.require_backend(VmBackend::Qemu); + let error = driver + .resolve_launch_plan_backend(false, &mut plan) + .expect_err("conflicting runtimes must fail"); + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("conflicting VM backends")); + assert!(plan.vsock_cid.is_none()); } #[test] fn guest_init_dropin_feature_does_not_force_qemu() { let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let mut plan = driver - .build_vm_launch_plan("sandbox-init", false, false, None) - .expect("base plan should build"); + let mut plan = driver.build_vm_launch_plan(None); plan.require_backend_feature(BackendFeature::GuestInitDropins); driver - .resolve_launch_plan_backend("sandbox-init", false, None, &mut plan) + .resolve_launch_plan_backend(false, &mut plan) .expect("guest init feature should resolve"); assert_eq!(plan.backend, VmBackend::Libkrun); @@ -14041,11 +13994,9 @@ while :; do sleep 0.05; done"#, #[test] fn qemu_launch_plan_uses_vsock_only_with_host_proxy() { let driver = test_driver_with_proxy("http://127.0.0.1:8080"); - let mut plan = driver - .build_vm_launch_plan("sandbox-proxy-vsock", true, true, None) - .expect("gpu plan should build"); + let mut plan = driver.build_vm_launch_plan(Some("0000:01:00.0".to_string())); driver - .resolve_launch_plan_backend("sandbox-proxy-vsock", true, None, &mut plan) + .resolve_launch_plan_backend(true, &mut plan) .expect("host control can reach a host-loopback proxy"); assert!(plan.vsock_cid.is_some()); } diff --git a/crates/openshell-driver-vm/src/lifecycle.rs b/crates/openshell-driver-vm/src/lifecycle.rs index f92082d99e..b99891ebb7 100644 --- a/crates/openshell-driver-vm/src/lifecycle.rs +++ b/crates/openshell-driver-vm/src/lifecycle.rs @@ -82,20 +82,9 @@ pub type LifecycleResult = Result; /// /// Extensions declare features they need (e.g. PCI passthrough or an /// external kernel image) and the VM driver resolves a concrete -/// [`VmBackend`] that can satisfy them. The mapping from feature to -/// backend lives in [`BackendFeature::requires_qemu`] for now; once a -/// third backend exists this should evolve into a per-backend capability -/// table that the resolver intersects against feature requirements. -/// -/// # Current limitations -/// -/// Until the non-GPU QEMU launch path (PCI device transport / VFIO root -/// port wiring) lands, the driver still rejects launches where the -/// resolved backend is QEMU but the sandbox has no GPU. As a result, -/// declaring [`Self::PciPassthrough`] or [`Self::ExternalKernelImage`] on -/// a non-GPU sandbox is accepted by [`LifecycleExtensionRegistry::validate`] -/// at registration time but will fail provisioning with a -/// `FailedPrecondition` status. +/// [`VmBackend`] that can satisfy them. [`Self::is_supported_by`] describes +/// each backend independently. Explicit backend requirements are preserved; +/// an unsupported feature fails before the launcher starts. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BackendFeature { /// Extension supplies its own kernel image via @@ -105,8 +94,8 @@ pub enum BackendFeature { /// [`LaunchPlan::guest_init_dropins`]. Supported by all backends. GuestInitDropins, /// Extension needs PCI device passthrough on the guest. Currently - /// QEMU-only and currently rejected for non-GPU sandboxes pending the - /// non-GPU QEMU launch path landing. + /// QEMU-only. The current device model requires an assigned GPU BDF; + /// arbitrary PCI device descriptors are not supported. PciPassthrough, } @@ -120,13 +109,15 @@ impl BackendFeature { } } - /// Returns true when satisfying this feature requires the QEMU backend - /// today. This is the simplest possible resolver and is expected to be - /// replaced with a per-backend capability table once a third backend - /// exists. + /// Whether the backend implements this feature. Device assignment and + /// kernel compatibility are validated separately for the concrete plan. #[must_use] - pub fn requires_qemu(self) -> bool { - matches!(self, Self::ExternalKernelImage | Self::PciPassthrough) + pub fn is_supported_by(self, backend: &VmBackend) -> bool { + match (backend, self) { + (VmBackend::Libkrun, Self::ExternalKernelImage | Self::PciPassthrough) => false, + (VmBackend::Libkrun | VmBackend::Qemu, Self::GuestInitDropins) + | (VmBackend::Qemu, Self::ExternalKernelImage | Self::PciPassthrough) => true, + } } } @@ -287,7 +278,7 @@ pub enum ExtensionActivation { /// and allocates backend-specific host resources such as a vsock CID. /// 3. [`before_launch`](Self::before_launch) — perform host-side /// side effects with the resolved plan in hand, optionally append -/// additional guest env via [`LaunchPlan::env`]. +/// guest environment entries or init drop-ins. /// 4. The driver spawns the VM launcher process. /// /// On launch failure or sandbox deletion, the driver invokes @@ -338,19 +329,13 @@ pub trait LifecycleExtension: std::fmt::Debug + Send + Sync { /// Perform host-side preparation with the resolved launch plan. /// - /// At this point [`LaunchPlan::backend`], - /// [`LaunchPlan::required_backends`], and - /// [`LaunchPlan::required_backend_features`] are finalized and any - /// backend-specific host resources have been - /// allocated. This hook is the right place to bind PCI devices, set - /// up filesystem state, or otherwise prepare the host. + /// The driver has finalized the backend, requirements, kernel, sizing, + /// GPU assignment, and vsock CID. Use this hook to bind a reserved PCI + /// device, set up filesystem state, or otherwise prepare the host. /// - /// Implementations MAY append entries to [`LaunchPlan::env`] to - /// inject additional guest environment variables, and MAY return an - /// error to abort the launch. Implementations MUST NOT change - /// [`LaunchPlan::backend`], [`LaunchPlan::required_backends`], or - /// [`LaunchPlan::required_backend_features`]; those changes are - /// ignored by the driver once `before_launch` is reached. + /// Implementations can append entries to [`LaunchPlan::env`] and + /// [`LaunchPlan::guest_init_dropins`] or return an error to abort the + /// launch. The registry rejects changes to finalized launch fields. /// /// If this hook performs allocations that must be released on failure /// or delete, implement @@ -604,8 +589,37 @@ impl LifecycleExtensionRegistry { plan: &mut LaunchPlan, ) -> LifecycleResult<()> { let span_status = openshell_otel::ErrorStatusGuard::current(); + let assigned_gpu = plan.gpu_bdf.clone(); + let backend = plan.backend.clone(); + let required_backends = plan.required_backends.clone(); + let required_backend_features = plan.required_backend_features.clone(); + let kernel_profile = plan.kernel_profile.clone(); + let kernel_image = plan.kernel_image.clone(); + let vsock_cid = plan.vsock_cid; + let vcpus = plan.vcpus; + let mem_mib = plan.mem_mib; for ext in self.active_for(sandbox) { ext.before_launch(sandbox, state_dir, plan).await?; + if plan.gpu_bdf != assigned_gpu { + return Err(LifecycleError::new(format!( + "extension '{}' changed the assigned GPU device in before_launch", + ext.name() + ))); + } + if plan.backend != backend + || plan.required_backends != required_backends + || plan.required_backend_features != required_backend_features + || plan.kernel_profile != kernel_profile + || plan.kernel_image != kernel_image + || plan.vsock_cid != vsock_cid + || plan.vcpus != vcpus + || plan.mem_mib != mem_mib + { + return Err(LifecycleError::new(format!( + "extension '{}' changed finalized launch fields in before_launch", + ext.name() + ))); + } } span_status.finish(Ok(())) } @@ -906,6 +920,10 @@ mod tests { ))); } plan.env.push(format!("RECORDING_{}=1", self.name)); + plan.guest_init_dropins.push(GuestInitDropin::new( + format!("90-{}.sh", self.name), + b"#!/bin/sh\n".to_vec(), + )); Ok(()) } @@ -978,6 +996,67 @@ mod tests { extension.clone() } + type PlanMutation = fn(&mut LaunchPlan); + + #[derive(Debug)] + struct PlanMutatingExtension(PlanMutation); + + #[tonic::async_trait] + impl LifecycleExtension for PlanMutatingExtension { + fn name(&self) -> &'static str { + "plan-mutator" + } + + async fn before_launch( + &self, + _sandbox: &Sandbox, + _state_dir: &Path, + plan: &mut LaunchPlan, + ) -> LifecycleResult<()> { + (self.0)(plan); + Ok(()) + } + } + + #[tokio::test] + async fn before_launch_rejects_changes_to_finalized_fields() { + let mutations: [(&str, PlanMutation); 10] = [ + ("assigned GPU device", |plan| plan.gpu_bdf = None), + ("assigned GPU device", |plan| { + plan.gpu_bdf = Some("0000:02:00.0".to_string()); + }), + ("finalized launch fields", |plan| { + plan.backend = VmBackend::Libkrun; + }), + ("finalized launch fields", |plan| { + plan.require_backend(VmBackend::Libkrun); + }), + ("finalized launch fields", |plan| { + plan.require_backend_feature(BackendFeature::PciPassthrough); + }), + ("finalized launch fields", |plan| { + plan.kernel_profile = Some("other".into()); + }), + ("finalized launch fields", |plan| { + plan.kernel_image = Some("other".into()); + }), + ("finalized launch fields", |plan| plan.vsock_cid = Some(8)), + ("finalized launch fields", |plan| plan.vcpus = 4), + ("finalized launch fields", |plan| plan.mem_mib = 4096), + ]; + for (message, mutate) in mutations { + let registry = + LifecycleExtensionRegistry::with(vec![Arc::new(PlanMutatingExtension(mutate))]); + let mut plan = sample_plan(VmBackend::Qemu); + plan.gpu_bdf = Some("0000:01:00.0".to_string()); + let error = registry + .before_launch(&sample_sandbox(), Path::new("/tmp/state"), &mut plan) + .await + .expect_err("finalized launch fields must survive before_launch"); + assert!(error.message().contains(message)); + } + } + #[tokio::test] async fn configure_launch_runs_each_extension_in_order() { let ext_a = RecordingExtension::new("a"); @@ -1096,7 +1175,7 @@ mod tests { } #[tokio::test] - async fn before_launch_runs_each_extension_in_order_and_collects_env() { + async fn before_launch_runs_each_extension_in_order_and_collects_guest_inputs() { let ext_a = RecordingExtension::new("a"); let ext_b = RecordingExtension::new("b"); let registry = @@ -1110,6 +1189,13 @@ mod tests { .expect("before_launch succeeds"); assert_eq!(plan.env, vec!["RECORDING_a=1", "RECORDING_b=1"]); + assert_eq!( + plan.guest_init_dropins + .iter() + .map(|dropin| dropin.name.as_str()) + .collect::>(), + ["90-a.sh", "90-b.sh"] + ); assert_eq!(ext_a.calls(), vec!["a:before_launch"]); assert_eq!(ext_b.calls(), vec!["b:before_launch"]); } diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index a292e81994..67846c10ac 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -1357,6 +1357,93 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn guest_init_creates_fd_links_and_preserves_existing_aliases() { + use std::os::unix::fs::MetadataExt as _; + + let dev = tempfile::tempdir().expect("temporary guest dev"); + let run = || run_guest_setup_fd_links(dev.path()); + let output = run(); + assert!(output.status.success(), "{output:?}"); + let aliases = [ + ("fd", "/proc/self/fd"), + ("stdin", "/proc/self/fd/0"), + ("stdout", "/proc/self/fd/1"), + ("stderr", "/proc/self/fd/2"), + ]; + let mut inodes = Vec::new(); + for (name, target) in aliases { + let path = dev.path().join(name); + assert_eq!(fs::read_link(&path).expect("read alias"), Path::new(target)); + inodes.push(fs::symlink_metadata(path).expect("stat alias").ino()); + } + let output = run(); + assert!(output.status.success(), "{output:?}"); + for ((name, _), inode) in aliases.into_iter().zip(inodes) { + assert_eq!( + fs::symlink_metadata(dev.path().join(name)) + .expect("stat retained alias") + .ino(), + inode, + "existing {name} alias stays intact" + ); + } + #[cfg(target_os = "linux")] + { + let stdin = dev.path().join("stdin"); + fs::remove_file(&stdin).expect("replace fixture alias"); + std::os::unix::fs::symlink("fd/0", &stdin).expect("equivalent relative alias"); + let output = run(); + assert!(output.status.success(), "{output:?}"); + assert_eq!( + fs::read_link(stdin).expect("retained relative alias"), + Path::new("fd/0") + ); + } + } + + #[test] + fn guest_init_rejects_conflicting_fd_entries_without_replacing_them() { + let dev = tempfile::tempdir().expect("temporary guest dev"); + let fd = dev.path().join("fd"); + fs::write(&fd, b"keep this entry").expect("conflicting file"); + let output = run_guest_setup_fd_links(dev.path()); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("must refer to /proc/self/fd")); + assert_eq!(fs::read(&fd).expect("retained file"), b"keep this entry"); + + fs::remove_file(&fd).expect("remove fixture file"); + std::os::unix::fs::symlink("missing-target", &fd).expect("dangling link"); + let output = run_guest_setup_fd_links(dev.path()); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("must refer to /proc/self/fd")); + assert_eq!( + fs::read_link(fd).expect("retained link"), + Path::new("missing-target") + ); + } + + fn run_guest_setup_fd_links(dev: &Path) -> std::process::Output { + let script = include_str!("../scripts/openshell-vm-sandbox-init.sh"); + let start = script + .find("setup_fd_links() {\n") + .expect("fd setup function"); + let tail = &script[start..]; + let end = tail.find("\n}\n").expect("fd setup function end") + "\n}\n".len(); + Command::new("bash") + .args([ + "-c", + &format!( + "set -euo pipefail\n{}\nsetup_fd_links \"$1\"\n", + &tail[..end] + ), + "test", + ]) + .arg(dev) + .output() + .expect("run shipped fd setup") + } + /// Runs the shipped guest init's `configure_hosts` against `root`. /// /// The function is cut out of the real script so the test exercises the diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index ad300d98cf..d74d9aee97 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -4,6 +4,7 @@ #![allow(unsafe_code)] use std::ffi::CString; +use std::io::Read as _; use std::path::{Path, PathBuf}; use std::process::{Command as StdCommand, Stdio}; use std::ptr; @@ -25,6 +26,96 @@ pub enum VmBackend { Qemu, } +impl VmBackend { + pub(crate) fn validate_host(&self) -> Result<(), String> { + match self { + Self::Libkrun => Ok(()), + Self::Qemu if cfg!(all(target_os = "linux", target_arch = "x86_64")) => Ok(()), + Self::Qemu => Err("QEMU requires a Linux x86_64 host with KVM".to_string()), + } + } + + pub(crate) fn resolve_kernel_image( + &self, + path: Option<&Path>, + ) -> Result, String> { + match (self, path) { + (Self::Libkrun, None) => Ok(None), + (Self::Libkrun, Some(path)) => Err(format!( + "libkrun does not support external kernel images: {}", + path.display() + )), + (Self::Qemu, path) => resolve_qemu_kernel_image(path).map(Some), + } + } +} + +fn resolve_qemu_kernel_image(path: Option<&Path>) -> Result { + let path = match path { + Some(path) => path.to_path_buf(), + None => qemu_runtime_dir()?.join("vmlinux"), + }; + validate_qemu_kernel_image(&path)?; + Ok(path) +} + +/// Check the architecture and boot-image header for QEMU direct boot. +/// An uncompressed Linux ELF kernel also needs a PVH entry note, which QEMU +/// checks when it loads the image. +fn validate_qemu_kernel_image(path: &Path) -> Result<(), String> { + if !path.is_file() { + return Err(format!( + "selected kernel image is not a regular file: {}", + path.display() + )); + } + let file = std::fs::File::open(path).map_err(|error| { + format!( + "cannot open selected kernel image {}: {error}", + path.display() + ) + })?; + let mut header = Vec::new(); + file.take(0x238).read_to_end(&mut header).map_err(|error| { + format!( + "cannot read selected kernel image {}: {error}", + path.display() + ) + })?; + if header.starts_with(b"\x7fELF") { + // ELFCLASS64, ELFDATA2LSB and EM_X86_64. + if header.len() >= 64 + && header[4] == 2 + && header[5] == 1 + && u16::from_le_bytes([header[18], header[19]]) == 62 + { + return Ok(()); + } + return Err(format!( + "QEMU kernel must be a 64-bit little-endian x86_64 ELF image: {}", + path.display() + )); + } + // Linux/x86 boot protocol: boot_flag, HdrS, version and XLF_KERNEL_64. + // https://www.kernel.org/doc/html/latest/arch/x86/boot.html + if header.len() == 0x238 + && header[0x1fe..0x200] == [0x55, 0xaa] + && &header[0x202..0x206] == b"HdrS" + { + if u16::from_le_bytes([header[0x206], header[0x207]]) >= 0x020c && header[0x236] & 1 != 0 { + return Ok(()); + } + return Err(format!( + "QEMU kernel bzImage must target x86_64: {}", + path.display() + )); + } + Err(format!( + "unsupported QEMU kernel format in {}; expected x86_64 ELF or bzImage", + path.display() + )) +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct VsockPortMap { pub guest_port: u32, @@ -71,13 +162,13 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { if !config.mounts.is_empty() { return Err("virtiofs mounts are not yet supported with the QEMU backend".to_string()); } - let gpu_bdf = config - .gpu_bdf - .as_deref() - .ok_or("gpu_bdf is required for QEMU backend")?; - let vsock_cid = config - .vsock_cid - .ok_or("vsock_cid is required for QEMU backend")?; + if config.vsock_port_map.is_some() { + return Err( + "QEMU uses host AF_VSOCK; Unix-socket vsock port maps require libkrun".to_string(), + ); + } + VmBackend::Qemu.validate_host()?; + let kernel = resolve_qemu_kernel_image(config.kernel_image.as_deref())?; if !config.root_disk.is_file() { return Err(format!( @@ -97,6 +188,8 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { return Err(format!("image disk not found: {}", image_disk.display())); } + let mut qemu_cmd = qemu_command(config, &kernel)?; + if let Err(err) = procguard::die_with_parent_cleanup(procguard_kill_children) { return Err(format!("procguard arm failed: {err}")); } @@ -107,17 +200,45 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { let guest_env = qemu_guest_env_vars(config); write_guest_env_file(&config.overlay_disk, &guest_env)?; - let vmlinux = if let Some(kernel_image) = &config.kernel_image { - kernel_image.clone() - } else { - qemu_runtime_dir()?.join("vmlinux") - }; - if !vmlinux.is_file() { - return Err(format!("VM kernel not found: {}", vmlinux.display())); + #[cfg(target_os = "linux")] + { + use nix::sys::signal::Signal; + use std::os::unix::process::CommandExt as _; + unsafe { + qemu_cmd.pre_exec(|| { + nix::sys::prctl::set_pdeathsig(Signal::SIGKILL) + .map_err(|err| std::io::Error::other(format!("pdeathsig: {err}"))) + }); + } } - let kernel_cmdline = build_kernel_cmdline(config); + let mut qemu_child = qemu_cmd + .spawn() + .map_err(|e| format!("failed to start QEMU: {e}"))?; + + let qemu_pid = qemu_child.id().cast_signed(); + install_signal_forwarding(qemu_pid); + + let status = qemu_child + .wait() + .map_err(|e| format!("failed to wait for QEMU: {e}"))?; + CHILD_PID.store(0, Ordering::Relaxed); + + if status.success() { + Ok(()) + } else { + Err(format!("QEMU exited with status {status}")) + } +} + +fn qemu_command(config: &VmLaunchConfig, kernel: &Path) -> Result { + let vsock_cid = config + .vsock_cid + .ok_or("vsock_cid is required for QEMU backend")?; + if vsock_cid < 3 { + return Err("QEMU guest vsock CID must be at least 3".to_string()); + } let mut qemu_cmd = StdCommand::new("qemu-system-x86_64"); qemu_cmd .arg("-machine") @@ -132,9 +253,9 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { .arg("-no-reboot") .args(qemu_network_args()) .arg("-kernel") - .arg(&vmlinux) + .arg(kernel) .arg("-append") - .arg(&kernel_cmdline) + .arg(build_kernel_cmdline(config)) .args(qemu_disk_args(config)) .arg("-device") .arg("pcie-root-port,id=vsock_root,slot=1") @@ -142,47 +263,22 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { .arg(format!( "vhost-vsock-pci,guest-cid={vsock_cid},bus=vsock_root" )) - .arg("-device") - .arg("pcie-root-port,id=gpu_root,slot=2") - .arg("-device") - .arg(format!("vfio-pci,host={gpu_bdf},bus=gpu_root")) .arg("-serial") .arg(format!("file:{}", config.console_output.display())); + if let Some(gpu_bdf) = &config.gpu_bdf { + qemu_cmd + .arg("-device") + .arg("pcie-root-port,id=gpu_root,slot=2") + .arg("-device") + .arg(format!("vfio-pci,host={gpu_bdf},bus=gpu_root")); + } + qemu_cmd.stdin(Stdio::null()); qemu_cmd.stdout(Stdio::inherit()); qemu_cmd.stderr(Stdio::inherit()); - #[cfg(target_os = "linux")] - { - use nix::sys::signal::Signal; - use std::os::unix::process::CommandExt as _; - unsafe { - qemu_cmd.pre_exec(|| { - nix::sys::prctl::set_pdeathsig(Signal::SIGKILL) - .map_err(|err| std::io::Error::other(format!("pdeathsig: {err}"))) - }); - } - } - - let mut qemu_child = qemu_cmd - .spawn() - .map_err(|e| format!("failed to start QEMU: {e}"))?; - - let qemu_pid = qemu_child.id().cast_signed(); - install_signal_forwarding(qemu_pid); - - let status = qemu_child - .wait() - .map_err(|e| format!("failed to wait for QEMU: {e}"))?; - - CHILD_PID.store(0, Ordering::Relaxed); - - if status.success() { - Ok(()) - } else { - Err(format!("QEMU exited with status {status}")) - } + Ok(qemu_cmd) } fn qemu_network_args() -> [&'static str; 2] { @@ -293,12 +389,7 @@ fn procguard_kill_children() { } fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { - if let Some(kernel_image) = &config.kernel_image { - return Err(format!( - "selected kernel image is not supported by this VM backend: {}", - kernel_image.display() - )); - } + VmBackend::Libkrun.resolve_kernel_image(config.kernel_image.as_deref())?; if !config.root_disk.is_file() { return Err(format!( "root disk image not found: {}", @@ -440,10 +531,10 @@ pub fn configured_runtime_dir() -> Result { } fn qemu_runtime_dir() -> Result { - configured_runtime_dir().map_err(|_| { - "QEMU backend requires OPENSHELL_VM_RUNTIME_DIR to be set (pointing to a directory \ - containing vmlinux). Set the env var or run `mise run vm:setup`." - .to_string() + configured_runtime_dir().map_err(|error| { + format!( + "QEMU requires an external kernel or a vmlinux file in the directory set by {VM_RUNTIME_DIR_ENV}: {error}" + ) }) } @@ -825,6 +916,139 @@ mod tests { } } + #[test] + fn qemu_command_keeps_vsock_and_only_adds_vfio_for_an_assigned_gpu() { + let mut config = qemu_config(); + for gpu in [None, Some("0000:01:00.0".to_string())] { + config.gpu_bdf = gpu.clone(); + let command = qemu_command(&config, Path::new("/vmlinux")).expect("QEMU command"); + let args: Vec<_> = command + .get_args() + .map(|arg| arg.to_string_lossy()) + .collect(); + assert!( + args.iter() + .any(|arg| arg == "vhost-vsock-pci,guest-cid=4,bus=vsock_root") + ); + assert!(args.windows(2).any(|pair| pair == ["-nic", "none"])); + assert_eq!( + args.iter().any(|arg| arg.starts_with("vfio-pci,")), + gpu.is_some() + ); + assert_eq!( + args.iter().any(|arg| arg.contains("gpu_root,slot=2")), + gpu.is_some() + ); + assert_eq!( + qemu_guest_env_vars(&config) + .iter() + .any(|env| env == "GPU_ENABLED=true"), + gpu.is_some() + ); + } + for cid in [None, Some(2)] { + config.vsock_cid = cid; + assert!(qemu_command(&config, Path::new("/vmlinux")).is_err()); + } + } + + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + #[test] + fn qemu_rejects_unsupported_hosts_before_disk_setup() { + let mut config = qemu_config(); + config.gpu_bdf = None; + assert_eq!( + run_vm(&config).unwrap_err(), + "QEMU requires a Linux x86_64 host with KVM" + ); + } + + #[test] + fn kernel_preflight_checks_format_and_guest_architecture() { + let file = tempfile::NamedTempFile::new().unwrap(); + let mut elf = vec![0u8; 64]; + elf[..6].copy_from_slice(b"\x7fELF\x02\x01"); + elf[18] = 62; // EM_X86_64. + std::fs::write(file.path(), &elf).unwrap(); + assert_eq!( + VmBackend::Qemu + .resolve_kernel_image(Some(file.path())) + .expect("x86_64 ELF header"), + Some(file.path().to_path_buf()) + ); + elf[18] = 183; // EM_AARCH64. + std::fs::write(file.path(), &elf).unwrap(); + assert!( + VmBackend::Qemu + .resolve_kernel_image(Some(file.path())) + .unwrap_err() + .contains("x86_64 ELF") + ); + + let mut bzimage = vec![0u8; 0x238]; + bzimage[0x1fe..0x200].copy_from_slice(&[0x55, 0xaa]); + bzimage[0x202..0x206].copy_from_slice(b"HdrS"); + bzimage[0x206..0x208].copy_from_slice(&0x020cu16.to_le_bytes()); + bzimage[0x236] = 1; + std::fs::write(file.path(), &bzimage).unwrap(); + VmBackend::Qemu + .resolve_kernel_image(Some(file.path())) + .expect("x86_64 bzImage header"); + bzimage[0x236] = 0; + std::fs::write(file.path(), &bzimage).unwrap(); + assert!( + VmBackend::Qemu + .resolve_kernel_image(Some(file.path())) + .unwrap_err() + .contains("bzImage must target x86_64") + ); + for invalid in [b"".as_slice(), b"\x7fELF", b"not a kernel"] { + std::fs::write(file.path(), invalid).unwrap(); + assert!( + VmBackend::Qemu + .resolve_kernel_image(Some(file.path())) + .is_err() + ); + } + } + + #[test] + fn launcher_entry_rejects_incompatible_external_kernels_before_disk_setup() { + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(file.path(), b"not a kernel").unwrap(); + let mut config = qemu_config(); + config.kernel_image = Some(file.path().to_path_buf()); + config.gpu_bdf = None; + if cfg!(all(target_os = "linux", target_arch = "x86_64")) { + assert!( + run_vm(&config) + .unwrap_err() + .contains("unsupported QEMU kernel format") + ); + } + config.backend = VmBackend::Libkrun; + assert!( + run_vm(&config) + .unwrap_err() + .contains("libkrun does not support external kernel images") + ); + } + + #[test] + fn qemu_launcher_rejects_unix_vsock_port_maps() { + let mut config = qemu_config(); + config.vsock_port_map = Some(VsockPortMap { + guest_port: 1234, + host_socket: "/tmp/unused.sock".into(), + host_initiated: false, + }); + assert!( + run_vm(&config) + .unwrap_err() + .contains("Unix-socket vsock port maps require libkrun") + ); + } + #[test] fn qemu_guest_env_vars_omit_network_metadata() { let env = qemu_guest_env_vars(&qemu_config()); diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 70f6891ce1..30ae5b32d5 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -214,6 +214,12 @@ Podman also supports `image` mounts, which mount a container image read-only at The MicroVM driver runs each sandbox in its own lightweight VM. It requires host virtualization: Apple Hypervisor on macOS or KVM on Linux. +CPU-only VMs use libkrun by default. On Linux x86_64 hosts with KVM, +operator-installed lifecycle extensions can select QEMU when they need an +external x86_64 kernel, without requesting a GPU. +QEMU VM launches don't support virtiofs mounts. Kernel selection remains under +operator control; the guest must still pass authenticated runtime admission. + The VM driver is opt-in. Enable it in the gateway TOML file: ```toml