From f4120d278d9f3d35ad7145aecb894b957fc473de Mon Sep 17 00:00:00 2001 From: Dmitrii Merkurev Date: Thu, 17 Sep 2026 23:05:28 +0000 Subject: [PATCH] Support vhost-user-vsock with the QEMU VMM The qemu_cli VMM only wired up the kernel's vhost-vsock device, so run_cvd's vsock servers fail in containers without AF_VSOCK. Emit a vhost-user-vsock-pci-non-transitional device backed by vhost_device_vsock instead, and let `auto` enable it for qemu_cli and riscv64 guests. QEMU connects to the chardev once instead of retrying like crosvm, so for QEMU wait for vhost.socket to listen first. The wait reads /proc/net/unix rather than lsof, which isn't a dependency and doesn't report unix socket state before 4.94.0. Both wait loops now sleep 20 ms instead of spinning. Bug: b/561671404 Test: cvd start --vm_manager=qemu_cli --vhost_user_vsock=true, for an x86_64 guest on an x86_64 host and an arm64 guest on an x86_64 host. Both reach sys.boot_completed=1 with adb working over vsock. --- .../cuttlefish/common/libs/utils/BUILD.bazel | 1 + .../libs/utils/wait_for_unix_socket.cpp | 100 +++++++++++++----- .../host/commands/assemble_cvd/flags.cc | 10 +- .../run_cvd/launch/vhost_device_vsock.cpp | 8 ++ .../host/libs/vm_manager/qemu_manager.cpp | 19 +++- 5 files changed, 107 insertions(+), 31 deletions(-) diff --git a/base/cvd/cuttlefish/common/libs/utils/BUILD.bazel b/base/cvd/cuttlefish/common/libs/utils/BUILD.bazel index a4ed70b78bf..acf95060717 100644 --- a/base/cvd/cuttlefish/common/libs/utils/BUILD.bazel +++ b/base/cvd/cuttlefish/common/libs/utils/BUILD.bazel @@ -431,6 +431,7 @@ cf_cc_library( "//cuttlefish/common/libs/fs", "//cuttlefish/common/libs/utils:wait_for_file", "//cuttlefish/files:file_is_socket", + "//cuttlefish/posix:strerror", "//cuttlefish/process:command", "//cuttlefish/process:managed_stdio", "//cuttlefish/result", diff --git a/base/cvd/cuttlefish/common/libs/utils/wait_for_unix_socket.cpp b/base/cvd/cuttlefish/common/libs/utils/wait_for_unix_socket.cpp index c04161b7f44..713019962c8 100644 --- a/base/cvd/cuttlefish/common/libs/utils/wait_for_unix_socket.cpp +++ b/base/cvd/cuttlefish/common/libs/utils/wait_for_unix_socket.cpp @@ -16,23 +16,90 @@ #include "cuttlefish/common/libs/utils/wait_for_unix_socket.h" -#include - #include -#include #include +#include + +#ifdef __linux__ +#include + +#include +#include +#include +#include + +#include "cuttlefish/posix/strerror.h" +#else +#include #include #include "absl/log/log.h" +#include "cuttlefish/process/command.h" +#include "cuttlefish/process/managed_stdio.h" +#endif + #include "cuttlefish/common/libs/fs/shared_fd.h" #include "cuttlefish/common/libs/utils/wait_for_file.h" #include "cuttlefish/files/file_is_socket.h" -#include "cuttlefish/process/command.h" -#include "cuttlefish/process/managed_stdio.h" #include "cuttlefish/result/result.h" namespace cuttlefish { +namespace { + +#ifdef __linux__ +Result IsUnixSocketListeningViaProc(const std::string& path) { + std::ifstream proc_unix("/proc/net/unix"); + CF_EXPECTF(proc_unix.is_open(), "Failed to open /proc/net/unix: {}", + StrError(errno)); + // Columns: Num RefCount Protocol Flags Type St Inode Path + // `Flags` carries `__SO_ACCEPTCON` once the socket has called listen(2). + // Accepted sockets alias the listener's path, so check every match. + std::string line; + while (std::getline(proc_unix, line)) { + std::istringstream iss(line); + std::string num, refcount, protocol, flags, type, st, inode, socket_path; + if (!(iss >> num >> refcount >> protocol >> flags >> type >> st >> inode >> + socket_path)) { + continue; + } + if (socket_path == path && + (std::strtoul(flags.c_str(), nullptr, 16) & __SO_ACCEPTCON) != 0) { + return true; + } + } + return false; +} +#else +Result IsUnixSocketListeningViaLsof(const std::string& path) { + static const std::regex socket_state_regex("TST=(.*)"); + + Command lsof("/usr/bin/lsof"); + lsof.AddParameter(/*"format"*/ "-F", /*"connection state"*/ "TST"); + lsof.AddParameter(path); + const std::string lsof_out = CF_EXPECT(RunAndCaptureStdout(std::move(lsof))); + + VLOG(0) << "lsof stdout:|" << lsof_out << "|"; + + std::smatch socket_state_match; + if (!std::regex_search(lsof_out, socket_state_match, socket_state_regex)) { + return false; + } + return socket_state_match.size() == 2 && socket_state_match[1] == "LISTEN"; +} +#endif + +// Whether `path` is a listening unix socket. Must not connect: a probe would be +// consumed as the vhost-user backend's one and only frontend. +Result IsUnixSocketListening(const std::string& path) { +#ifdef __linux__ + return CF_EXPECT(IsUnixSocketListeningViaProc(path)); +#else + return CF_EXPECT(IsUnixSocketListeningViaLsof(path)); +#endif +} + +} // namespace Result WaitForUnixSocket(const std::string& path, int timeoutSec) { const auto targetTime = @@ -59,7 +126,7 @@ Result WaitForUnixSocket(const std::string& path, int timeoutSec) { return {}; } - sched_yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); } return CF_ERR("This shouldn't be executed"); @@ -74,8 +141,6 @@ Result WaitForUnixSocketListeningWithoutConnect(const std::string& path, "Waiting for socket path creation failed"); CF_EXPECT(FileIsSocket(path), "Specified path is not a socket"); - std::regex socket_state_regex("TST=(.*)"); - while (true) { const auto currentTime = std::chrono::system_clock::now(); @@ -83,24 +148,11 @@ Result WaitForUnixSocketListeningWithoutConnect(const std::string& path, return CF_ERR("Timed out"); } - Command lsof("/usr/bin/lsof"); - lsof.AddParameter(/*"format"*/ "-F", /*"connection state"*/ "TST"); - lsof.AddParameter(path); - std::string lsof_out = CF_EXPECT(RunAndCaptureStdout(std::move(lsof))); - - VLOG(0) << "lsof stdout:|" << lsof_out << "|"; - - std::smatch socket_state_match; - if (std::regex_search(lsof_out, socket_state_match, socket_state_regex)) { - if (socket_state_match.size() == 2) { - const std::string& socket_state = socket_state_match[1]; - if (socket_state == "LISTEN") { - return {}; - } - } + if (CF_EXPECT(IsUnixSocketListening(path))) { + return {}; } - sched_yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); } return CF_ERR("This shouldn't be executed"); diff --git a/base/cvd/cuttlefish/host/commands/assemble_cvd/flags.cc b/base/cvd/cuttlefish/host/commands/assemble_cvd/flags.cc index a67bc2baa01..63b6ae7c664 100644 --- a/base/cvd/cuttlefish/host/commands/assemble_cvd/flags.cc +++ b/base/cvd/cuttlefish/host/commands/assemble_cvd/flags.cc @@ -905,10 +905,11 @@ Result InitializeCuttlefishConfiguration( } if (vhost_user_vsock_vec[instance_index] == kVhostUserVsockModeAuto) { - std::set default_on_arch = {Arch::Arm64}; + std::set default_on_arch = {Arch::Arm64, Arch::RiscV64}; if (guest_configs[instance_index].vhost_user_vsock) { instance.set_vhost_user_vsock(true); - } else if (VmManagerIsCrosvm(tmp_config_obj) && + } else if ((VmManagerIsCrosvm(tmp_config_obj) || + VmManagerIsQemu(tmp_config_obj)) && default_on_arch.find( guest_configs[instance_index].target_arch) != default_on_arch.end()) { @@ -918,8 +919,9 @@ Result InitializeCuttlefishConfiguration( } } else if (vhost_user_vsock_vec[instance_index] == kVhostUserVsockModeTrue) { - CF_EXPECT_EQ(tmp_config_obj.vm_manager(), VmmMode::kCrosvm, - "For now, only crosvm supports vhost_user_vsock"); + CF_EXPECT( + VmManagerIsCrosvm(tmp_config_obj) || VmManagerIsQemu(tmp_config_obj), + "For now, only crosvm and qemu support vhost_user_vsock"); instance.set_vhost_user_vsock(true); } else if (vhost_user_vsock_vec[instance_index] == kVhostUserVsockModeFalse) { diff --git a/base/cvd/cuttlefish/host/commands/run_cvd/launch/vhost_device_vsock.cpp b/base/cvd/cuttlefish/host/commands/run_cvd/launch/vhost_device_vsock.cpp index 848b8b5e312..39fb4daed8a 100644 --- a/base/cvd/cuttlefish/host/commands/run_cvd/launch/vhost_device_vsock.cpp +++ b/base/cvd/cuttlefish/host/commands/run_cvd/launch/vhost_device_vsock.cpp @@ -120,6 +120,14 @@ Result VhostDeviceVsock::WaitForAvailability() { fmt::format("{}/vsock_{}_{}/vm.vsock", TempDir(), instance_.vsock_guest_cid(), std::to_string(getuid())), 30)); + // vhost.socket is bound after vm.vsock. QEMU connects to it once at + // startup, while crosvm retries via --vhost-user-connect-timeout-ms. + if (VmManagerIsQemu(cfconfig_)) { + CF_EXPECT(WaitForUnixSocketListeningWithoutConnect( + fmt::format("{}/vsock_{}_{}/vhost.socket", TempDir(), + instance_.vsock_guest_cid(), std::to_string(getuid())), + 30)); + } } return {}; } diff --git a/base/cvd/cuttlefish/host/libs/vm_manager/qemu_manager.cpp b/base/cvd/cuttlefish/host/libs/vm_manager/qemu_manager.cpp index 869556097f4..aecfd8b26d0 100644 --- a/base/cvd/cuttlefish/host/libs/vm_manager/qemu_manager.cpp +++ b/base/cvd/cuttlefish/host/libs/vm_manager/qemu_manager.cpp @@ -32,11 +32,13 @@ #include "absl/log/log.h" #include "absl/strings/str_split.h" +#include "fmt/format.h" #include "vulkan/vulkan.h" #include "cuttlefish/common/libs/utils/files.h" #include "cuttlefish/common/libs/utils/host_info.h" #include "cuttlefish/common/libs/utils/in_sandbox.h" +#include "cuttlefish/common/libs/utils/known_paths.h" #include "cuttlefish/common/libs/utils/wait_for_unix_socket.h" #include "cuttlefish/files/file_exists.h" #include "cuttlefish/host/libs/config/config_constants.h" @@ -873,9 +875,20 @@ Result> QemuManager::StartCommands( qemu_cmd.AddParameter("timestamp=on"); #ifdef __linux__ - qemu_cmd.AddParameter("-device"); - qemu_cmd.AddParameter("vhost-vsock-pci-non-transitional,guest-cid=", - instance.vsock_guest_cid()); + if (instance.vhost_user_vsock()) { + const std::string vhost_socket = + fmt::format("{}/vsock_{}_{}/vhost.socket", TempDir(), + instance.vsock_guest_cid(), getuid()); + qemu_cmd.AddParameter("-chardev"); + qemu_cmd.AddParameter("socket,id=char_vsock,path=", vhost_socket); + qemu_cmd.AddParameter("-device"); + qemu_cmd.AddParameter( + "vhost-user-vsock-pci-non-transitional,chardev=char_vsock"); + } else { + qemu_cmd.AddParameter("-device"); + qemu_cmd.AddParameter("vhost-vsock-pci-non-transitional,guest-cid=", + instance.vsock_guest_cid()); + } #endif qemu_cmd.AddParameter("-device");