From ef5f40e58fbf6c0e62b3c27cb7a959ca2209792f Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Wed, 28 Oct 2026 19:55:57 -0400 Subject: [PATCH 1/2] Add camera PiP and shared OVXR frame consumers Restack the reviewed camera layer on the rebased session integration. Retain upstream SBOM packaging and finished-wheel checks; verify camera support after all wheel modifications and before artifact upload. Signed-off-by: Hougant Chen --- .github/workflows/build-ubuntu.yml | 12 + CMakeLists.txt | 1 + docs/source/getting_started/ovxr.rst | 52 +- pyproject.toml | 1 + scripts/check_camera_quad_wheel.py | 55 + src/core/CMakeLists.txt | 4 + src/core/ovxr_pip_layer/CMakeLists.txt | 4 + src/core/ovxr_pip_layer/cpp/CMakeLists.txt | 30 + src/core/ovxr_pip_layer/cpp/pip_layer.cpp | 911 ++++++++++++++++ src/core/python/CMakeLists.txt | 21 + src/core/python/pyproject.toml.in | 1 + src/core/python/requirements-ovxr.txt | 2 + src/core/python/setup.py | 22 +- .../ovxr_session_manager/__init__.py | 9 +- .../ovxr_session_manager/camera_mailbox.py | 233 +++++ .../ovxr_session_manager/camera_panels.py | 127 +++ .../ovxr_session_manager/frame_pump.py | 102 +- .../ovxr_session_manager/gpu_output.py | 196 ++++ .../ovxr_session_manager/interfaces.py | 50 +- .../ovxr_session_manager/openxr_quad.py | 174 ++++ .../ovxr_session_manager/ovxr_session.py | 3 +- tests/cpp/core/CMakeLists.txt | 4 + tests/cpp/core/ovxr_pip_layer/CMakeLists.txt | 23 + .../pip_layer_contract_test.cpp | 421 ++++++++ .../test_camera_mailbox.py | 168 +++ .../ovxr_session_manager/test_openxr_quad.py | 175 ++++ .../ovxr_session_manager/test_ovxr_session.py | 977 ++++++++++++++---- 27 files changed, 3544 insertions(+), 234 deletions(-) create mode 100644 scripts/check_camera_quad_wheel.py create mode 100644 src/core/ovxr_pip_layer/CMakeLists.txt create mode 100644 src/core/ovxr_pip_layer/cpp/CMakeLists.txt create mode 100644 src/core/ovxr_pip_layer/cpp/pip_layer.cpp create mode 100644 src/python/isaaccapture/ovxr_session_manager/camera_mailbox.py create mode 100644 src/python/isaaccapture/ovxr_session_manager/camera_panels.py create mode 100644 src/python/isaaccapture/ovxr_session_manager/gpu_output.py create mode 100644 src/python/isaaccapture/ovxr_session_manager/openxr_quad.py create mode 100644 tests/cpp/core/ovxr_pip_layer/CMakeLists.txt create mode 100644 tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp create mode 100644 tests/python/core/ovxr_session_manager/test_camera_mailbox.py create mode 100644 tests/python/core/ovxr_session_manager/test_openxr_quad.py diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index cb31c998f0..b784d26351 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -162,6 +162,7 @@ jobs: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DBUILD_PLUGIN_OAK_CAMERA=ON \ -DBUILD_VIZ=ON \ + -DBUILD_OVXR_CAMERA_QUADS=ON \ -DENABLE_CLOUDXR_BUNDLE_CHECK=ON \ -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake \ "${coverage_args[@]}" @@ -341,6 +342,17 @@ jobs: isaaccapture-sbom check --wheel "${wheel}" done + - name: Verify camera support in release wheel artifacts + if: matrix.build_type == 'Release' + run: | + set -euo pipefail + shopt -s nullglob + wheels=(install/wheels/isaaccapture-*.whl) + [[ "${#wheels[@]}" -gt 0 ]] || { echo "No Isaac Capture wheels found"; exit 1; } + for wheel in "${wheels[@]}"; do + python3 scripts/check_camera_quad_wheel.py "${wheel}" --camera-quads + done + - name: Create tarball to preserve permissions if: matrix.build_type == 'Release' run: tar -cvf isaacteleop-install.tar -C install . diff --git a/CMakeLists.txt b/CMakeLists.txt index ded7797a6b..521ec03c30 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; s option(BUILD_EXAMPLES "Build examples" ON) option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF) option(BUILD_TESTING "Build unit tests" ON) +option(BUILD_OVXR_CAMERA_QUADS "Build optional application-owned OVXR camera quad support" OFF) # Televiz (src/viz) needs Vulkan, the CUDA Toolkit, and glslangValidator. # Only Vulkan and CUDA gate the default -- without them the machine cannot build # viz at all, so a core-only source build still configures. glslangValidator is diff --git a/docs/source/getting_started/ovxr.rst b/docs/source/getting_started/ovxr.rst index 2b6f1de545..ccec8c4e7f 100644 --- a/docs/source/getting_started/ovxr.rst +++ b/docs/source/getting_started/ovxr.rst @@ -41,14 +41,62 @@ from the Isaac Capture wheel. These packages retain their own licenses: CloudXR native libraries are bundled in Isaac Capture wheels and remain subject to their separate NVIDIA license and CloudXR EULA acceptance. +Optional camera panels +---------------------- + +Camera picture-in-picture panels require the native camera helper. The Ubuntu +release workflow includes it in its wheels and checks the packaged artifact. +Installing those wheels requires no CMake configuration. + +For a source build, follow the source-build guide for your application's Python +version, then enable camera support and build the wheel: + +.. code-block:: bash + + cmake -S . -B build -DBUILD_OVXR_CAMERA_QUADS=ON + cmake --build build --target python_package python_stubs python_wheel --parallel + +Install the generated wheel with its ``ovxr`` extra using the instructions +above. The extra requires CUDA 13 Python bindings for GPU camera output; +``BUILD_OVXR_CAMERA_QUADS`` controls whether the wheel contains the native helper. +The helper is disabled by default. + +Use a CUDA 13-compatible NVIDIA driver (R580 or newer), meeting any stricter +requirements of the selected bindings and renderer. See the +`CUDA compatibility requirements `_. +Installing Python packages or a CUDA Toolkit does not update the host driver. + +Install the Vulkan development SDK before configuring camera support. Linux +builds also require the CUDA Toolkit. CMake discovers these installed dependencies. +Use ``CUDAToolkit_ROOT`` to select a CUDA Toolkit installation and ``VULKAN_SDK`` +to help locate a Vulkan SDK. The GPU driver and Vulkan runtime loader are required +on the host. + Use from another application ---------------------------- ``isaaccapture.ovxr_session_manager.ovxr_session`` provides ``OvxrSession`` and ``OvxrSessionConfig``. Implement ``OvxrApplicationInterface`` to supply the application's renderer, Stage and ordinal ledger in an ``OvxrSessionBinding``. -``camera_paths`` is required and must contain both stereo cameras. Set -``up_axis="Y"`` for a Y-up Stage; the default is ``"Z"``. +The binding also supplies optional ``frame_consumers`` after the application has +resolved its render-product paths. Capture does not depend on an Isaac Lab task +or scene configuration. ``camera_paths`` is required and must contain both stereo +cameras. Set ``up_axis="Y"`` for a Y-up Stage; the default is ``"Z"``. + +``CameraPanelConsumer`` implements the ``OvxrFrameConsumer`` lifecycle. Configure panels with a tuple of +``CameraPanelConfig`` values, each pairing a render-product path with its placement +and image bound. + +Consumers receive ``FrameOutputs`` on the XR owner thread. Use ``latest_color`` +to select an output, then ``copy_host_rgba`` or ``map_gpu`` to consume it without +unpacking native result structures. Selected handles are valid only during that +callback. CPU snapshots are shared within that frame; GPU resources retain their +completion and retirement rules. A consumer may contribute composition layers that remain valid +through submission. ``on_openxr_ready`` may be called repeatedly and must be +idempotent. ``finish_frame`` releases borrowed outputs before submission or +cancellation; ``close`` must tolerate partial startup and retain resources if +safe retirement fails. The camera consumer owns one native panel collection, +which is loaded directly and is not an OpenXR API layer. For a uv-managed application, declare Isaac Capture with its ``ovxr`` extra and select the source-built wheel in the application's project configuration: diff --git a/pyproject.toml b/pyproject.toml index c88d5779ec..b7f75e9bd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ ui = [ ] ovxr = [ "ovxr==0.1.0.5842", + "cuda-bindings>=13,<14", ] retargeters = [ "dex-retargeting>=0.4.6,<0.6.0", diff --git a/scripts/check_camera_quad_wheel.py b/scripts/check_camera_quad_wheel.py new file mode 100644 index 0000000000..00ac948ef4 --- /dev/null +++ b/scripts/check_camera_quad_wheel.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Check optional camera-helper delivery in an actual project wheel. + +Run against wheels built with BUILD_OVXR_CAMERA_QUADS ON and OFF, including +ON -> OFF -> ON in the same build directory to detect stale packaged data. +""" + +import argparse +import hashlib +import json +from pathlib import Path +from zipfile import ZipFile + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("wheel", type=Path) + parser.add_argument( + "--camera-quads", action=argparse.BooleanOptionalAction, required=True + ) + args = parser.parse_args() + prefix = "isaaccapture/ovxr_session_manager/native/" + expected = { + prefix + "libisaaccapture_camera_panels.so", + } + with ZipFile(args.wheel) as wheel: + members = { + name + for name in wheel.namelist() + if name.startswith(prefix) and not name.endswith("/") + } + wanted = expected if args.camera_quads else set() + if members != wanted: + raise RuntimeError( + f"Camera helper package differs: expected {sorted(wanted)}, got {sorted(members)}" + ) + if any("/openxr_layer/" in name for name in wheel.namelist()): + raise RuntimeError("Wheel contains obsolete OpenXR API-layer packaging") + print( + json.dumps( + { + "wheel": str(args.wheel), + "sha256": hashlib.sha256(args.wheel.read_bytes()).hexdigest(), + "camera_quads": args.camera_quads, + "result": "PASS", + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index b627490ba8..9cc4772373 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -38,6 +38,10 @@ add_subdirectory(deviceio_session) # Build MCAP library (depends on deviceio for DeviceIOSession and tracker interfaces) add_subdirectory(mcap) +if(BUILD_OVXR_CAMERA_QUADS) + add_subdirectory(ovxr_pip_layer) +endif() + # CloudXR runtime helper (stages the vendored CloudXR runtime SDK bundle). # Its pure-Python source lives in src/python/isaaccapture/cloudxr/, as does the # retargeting engine's and the session manager's. diff --git a/src/core/ovxr_pip_layer/CMakeLists.txt b/src/core/ovxr_pip_layer/CMakeLists.txt new file mode 100644 index 0000000000..2b9f1e328c --- /dev/null +++ b/src/core/ovxr_pip_layer/CMakeLists.txt @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_subdirectory(cpp) diff --git a/src/core/ovxr_pip_layer/cpp/CMakeLists.txt b/src/core/ovxr_pip_layer/cpp/CMakeLists.txt new file mode 100644 index 0000000000..26ebb48eaf --- /dev/null +++ b/src/core/ovxr_pip_layer/cpp/CMakeLists.txt @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Vulkan REQUIRED) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + find_package(CUDAToolkit REQUIRED) +endif() + +add_library(ovxr_pip_layer SHARED pip_layer.cpp) +target_compile_features(ovxr_pip_layer PRIVATE cxx_std_20) +target_include_directories(ovxr_pip_layer PRIVATE + $) +target_link_libraries(ovxr_pip_layer PRIVATE Vulkan::Vulkan log_bridge::log_bridge_core) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Keep static logger/fmt symbols private to the helper's existing C ABI. + set(_pip_exports "${CMAKE_CURRENT_BINARY_DIR}/ovxr_pip_exports.map") + file(GENERATE OUTPUT "${_pip_exports}" + CONTENT "{ global: isaacteleop_pip_*; local: *; };\n") + target_link_options(ovxr_pip_layer PRIVATE "-Wl,--version-script,${_pip_exports}") + set_property(TARGET ovxr_pip_layer APPEND PROPERTY LINK_DEPENDS "${_pip_exports}") +endif() +if(CUDAToolkit_FOUND AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_compile_definitions(ovxr_pip_layer PRIVATE ISAAC_TELEOP_PIP_CUDA) + target_link_libraries(ovxr_pip_layer PRIVATE CUDA::cudart_static) +endif() +set_target_properties(ovxr_pip_layer PROPERTIES + OUTPUT_NAME isaaccapture_camera_panels + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python_package/$/isaaccapture/ovxr_session_manager/native") + +add_library(ovxr_pip_layer::ovxr_pip_layer ALIAS ovxr_pip_layer) diff --git a/src/core/ovxr_pip_layer/cpp/pip_layer.cpp b/src/core/ovxr_pip_layer/cpp/pip_layer.cpp new file mode 100644 index 0000000000..c5c57998c2 --- /dev/null +++ b/src/core/ovxr_pip_layer/cpp/pip_layer.cpp @@ -0,0 +1,911 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Application-owned camera quads on a borrowed OVXR session/device. The frame +// owner uploads and returns layer descriptors before OVXR submits. This helper +// never creates a session or intercepts OpenXR frame calls. +#define XR_USE_GRAPHICS_API_VULKAN +#include + +// Keep Vulkan declarations before OpenXR platform structs, including after formatting. +#include +#include + +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) +# include +# include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ +constexpr size_t kMaxCameras = 3; + +struct Dispatch +{ + PFN_xrGetInstanceProcAddr gipa = nullptr; + PFN_xrCreateReferenceSpace createSpace = nullptr; + PFN_xrDestroySpace destroySpace = nullptr; + PFN_xrEnumerateSwapchainFormats formats = nullptr; + PFN_xrCreateSwapchain createSwapchain = nullptr; + PFN_xrDestroySwapchain destroySwapchain = nullptr; + PFN_xrEnumerateSwapchainImages images = nullptr; + PFN_xrAcquireSwapchainImage acquire = nullptr; + PFN_xrWaitSwapchainImage wait = nullptr; + PFN_xrReleaseSwapchainImage release = nullptr; +}; + +struct CameraSession +{ + XrInstance instance = XR_NULL_HANDLE; + XrSession session = XR_NULL_HANDLE; + VkPhysicalDevice physical = VK_NULL_HANDLE; + VkDevice device = VK_NULL_HANDLE; + VkQueue queue = VK_NULL_HANDLE; + uint32_t queueFamily = 0; + XrSpace view = XR_NULL_HANDLE; + XrSpace local = XR_NULL_HANDLE; + XrSwapchain swapchain = XR_NULL_HANDLE; + std::vector images; + VkBuffer staging = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + void* pixels = nullptr; + VkCommandPool pool = VK_NULL_HANDLE; + VkCommandBuffer command = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + uint32_t width = 0; + uint32_t height = 0; + uint32_t acquiredIndex = 0; + bool acquired = false; + bool waited = false; + bool fencePending = false; + bool retired = false; + bool ready = false; +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + cudaExternalMemory_t cudaMemory = nullptr; + void* cudaPixels = nullptr; + cudaStream_t cudaStream = nullptr; + int cudaDevice = -1; + int vulkanCudaDevice = -1; + bool cudaPending = false; +#endif +}; + +struct Panel +{ + bool enabled = true; + bool headLocked = true; + XrPosef pose{ { 0, 0, 0, 1 }, { 0, -0.15f, -0.8f } }; + XrExtent2Df size{ 0.48f, 0.30f }; +}; + +// Each frame consumer owns one helper; calls stay on its XR owner thread. +struct PanelSession +{ + Dispatch dispatch{}; + std::array cameras{}; + std::array panels{}; + std::array quads{}; + bool attached = false; +}; +constexpr uint32_t kMaxImageDimension = 4096; + +template +bool resolve(PFN_xrGetInstanceProcAddr gipa, XrInstance instance, const char* name, Function& target) +{ + PFN_xrVoidFunction raw = nullptr; + if (XR_FAILED(gipa(instance, name, &raw)) || !raw) + { + return false; + } + target = reinterpret_cast(raw); + return true; +} + +bool vkOk(VkResult result) +{ + return result == VK_SUCCESS; +} + +bool releaseImageResources(CameraSession& camera) +{ + if (camera.device) + { +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (camera.cudaDevice >= 0) + { + // Failed completion quarantines both the borrowed input and our + // external allocation. Never free it merely because teardown ran. + if (cudaSetDevice(camera.cudaDevice) != cudaSuccess || + (camera.cudaPending && cudaStreamSynchronize(camera.cudaStream) != cudaSuccess)) + { + camera.retired = true; + return false; + } + camera.cudaPending = false; + } +#endif + if (camera.fence) + { + if (camera.fencePending) + { + // This helper owns only its copy, not every queue of the + // borrowed device. Retain storage until its own fence proves + // completion; device-wide idle is not our synchronization seam. + if (vkWaitForFences(camera.device, 1, &camera.fence, VK_TRUE, UINT64_MAX) != VK_SUCCESS) + { + isaaccapture::Logger::get("isaaccapture.ovxr_pip_layer.CameraSession") + ->error("IsaacTeleop PiP: GPU completion unproved; retaining upload allocations"); + camera.retired = true; + camera.ready = false; + return false; + } + } + vkDestroyFence(camera.device, camera.fence, nullptr); + camera.fence = VK_NULL_HANDLE; + } + if (camera.pool) + { + vkDestroyCommandPool(camera.device, camera.pool, nullptr); + camera.pool = VK_NULL_HANDLE; + camera.command = VK_NULL_HANDLE; + } +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (camera.cudaPixels) + { + if (cudaFree(camera.cudaPixels) != cudaSuccess) + { + camera.retired = true; + return false; + } + camera.cudaPixels = nullptr; + } + if (camera.cudaMemory) + { + if (cudaDestroyExternalMemory(camera.cudaMemory) != cudaSuccess) + { + camera.retired = true; + return false; + } + camera.cudaMemory = nullptr; + } + if (camera.cudaStream) + { + if (cudaStreamDestroy(camera.cudaStream) != cudaSuccess) + { + camera.retired = true; + return false; + } + camera.cudaStream = nullptr; + } + camera.cudaDevice = -1; +#endif + if (camera.pixels) + { + vkUnmapMemory(camera.device, camera.memory); + } + if (camera.staging) + { + vkDestroyBuffer(camera.device, camera.staging, nullptr); + } + if (camera.memory) + { + vkFreeMemory(camera.device, camera.memory, nullptr); + } + } + camera.fence = VK_NULL_HANDLE; + camera.pool = VK_NULL_HANDLE; + camera.command = VK_NULL_HANDLE; + camera.pixels = nullptr; + camera.staging = VK_NULL_HANDLE; + camera.memory = VK_NULL_HANDLE; + camera.fencePending = false; + camera.ready = false; + return true; +} + +bool releaseSessionResources(CameraSession& camera, const Dispatch& dispatch) +{ + if (!releaseImageResources(camera)) + { + return false; + } + if (camera.swapchain) + { + // OpenXR destruction requires completed application graphics work, not + // reacquiring/releasing a runtime image. Our owned copies retired above. + if (XR_FAILED(dispatch.destroySwapchain(camera.swapchain))) + { + return false; + } + camera.swapchain = XR_NULL_HANDLE; + } + camera.acquired = camera.waited = false; + for (auto* space : { &camera.view, &camera.local }) + { + if (*space && XR_FAILED(dispatch.destroySpace(*space))) + { + return false; + } + *space = XR_NULL_HANDLE; + } + camera.images.clear(); + return true; +} + +bool createSwapchain(CameraSession& camera, const Dispatch& dispatch, uint32_t width, uint32_t height) +{ + if (camera.acquired || camera.retired) + { + return false; + } + if (!releaseImageResources(camera)) + { + return false; + } + if (camera.swapchain) + { + if (XR_FAILED(dispatch.destroySwapchain(camera.swapchain))) + { + camera.retired = true; + camera.ready = false; + return false; // Retain the old handle for explicit owner cleanup. + } + camera.swapchain = XR_NULL_HANDLE; + camera.images.clear(); + } + // A new handle is not usable until enumeration succeeds. Invalidate the + // old extent so a failed resize cannot admit a later old-size upload. + camera.width = camera.height = 0; + uint32_t count = 0; + if (XR_FAILED(dispatch.formats(camera.session, 0, &count, nullptr)) || !count) + { + return false; + } + std::vector formats(count); + if (XR_FAILED(dispatch.formats(camera.session, count, &count, formats.data()))) + { + return false; + } + auto chosen = std::find(formats.begin(), formats.end(), VK_FORMAT_R8G8B8A8_UNORM); + if (chosen == formats.end()) + { + return false; // No color conversion or unverified swapchain format fallback. + } + XrSwapchainCreateInfo info{ XR_TYPE_SWAPCHAIN_CREATE_INFO }; + info.usageFlags = XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT | XR_SWAPCHAIN_USAGE_TRANSFER_DST_BIT; + info.format = *chosen; + info.sampleCount = 1; + info.width = width; + info.height = height; + info.faceCount = 1; + info.arraySize = 1; + info.mipCount = 1; + if (XR_FAILED(dispatch.createSwapchain(camera.session, &info, &camera.swapchain))) + { + return false; + } + count = 0; + if (XR_FAILED(dispatch.images(camera.swapchain, 0, &count, nullptr)) || !count) + { + return false; + } + camera.images.resize(count); + for (auto& image : camera.images) + { + image = {}; + image.type = XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR; + } + if (XR_FAILED(dispatch.images( + camera.swapchain, count, &count, reinterpret_cast(camera.images.data())))) + { + return false; + } + camera.width = width; + camera.height = height; + return true; +} + +bool createUploadResources(CameraSession& camera, bool gpu = false) +{ + const auto fail = [&]() + { + releaseImageResources(camera); // No GPU submission exists yet. + return false; + }; + const VkDeviceSize length = static_cast(camera.width) * camera.height * 4; + VkBufferCreateInfo bufferInfo{ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + VkExternalMemoryBufferCreateInfo external{ VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO }; + external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; + if (gpu) + { + bufferInfo.pNext = &external; + } +#endif + bufferInfo.size = length; + bufferInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + if (!vkOk(vkCreateBuffer(camera.device, &bufferInfo, nullptr, &camera.staging))) + { + return fail(); + } + VkMemoryRequirements requirements{}; + vkGetBufferMemoryRequirements(camera.device, camera.staging, &requirements); + VkPhysicalDeviceMemoryProperties properties{}; + vkGetPhysicalDeviceMemoryProperties(camera.physical, &properties); + uint32_t type = UINT32_MAX; + for (uint32_t i = 0; i < properties.memoryTypeCount; ++i) + { + const VkMemoryPropertyFlags flags = + gpu ? VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT : + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + if ((requirements.memoryTypeBits & (1u << i)) && (properties.memoryTypes[i].propertyFlags & flags) == flags) + { + type = i; + break; + } + } + if (type == UINT32_MAX) + { + return fail(); + } + VkMemoryAllocateInfo allocation{ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO }; +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + VkMemoryDedicatedAllocateInfo dedicated{ VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO }; + dedicated.buffer = camera.staging; + VkExportMemoryAllocateInfo exportInfo{ VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO }; + exportInfo.pNext = &dedicated; + exportInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; + if (gpu) + { + allocation.pNext = &exportInfo; + } +#endif + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = type; + if (!vkOk(vkAllocateMemory(camera.device, &allocation, nullptr, &camera.memory)) || + !vkOk(vkBindBufferMemory(camera.device, camera.staging, camera.memory, 0)) || + (!gpu && !vkOk(vkMapMemory(camera.device, camera.memory, 0, length, 0, &camera.pixels)))) + { + return fail(); + } +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (gpu) + { + const auto getFd = reinterpret_cast(vkGetDeviceProcAddr(camera.device, "vkGetMemoryFdKHR")); + VkMemoryGetFdInfoKHR fdInfo{ VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR }; + fdInfo.memory = camera.memory; + fdInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; + int fd = -1; + if (!getFd || getFd(camera.device, &fdInfo, &fd) != VK_SUCCESS) + { + return fail(); + } + cudaExternalMemoryHandleDesc memory{}; + memory.type = cudaExternalMemoryHandleTypeOpaqueFd; + memory.handle.fd = fd; + memory.size = requirements.size; + memory.flags = cudaExternalMemoryDedicated; + if (cudaImportExternalMemory(&camera.cudaMemory, &memory) != cudaSuccess) + { + close(fd); // Successful import alone transfers fd ownership to CUDA. + return fail(); + } + cudaExternalMemoryBufferDesc buffer{}; + buffer.size = length; + if (cudaExternalMemoryGetMappedBuffer(&camera.cudaPixels, camera.cudaMemory, &buffer) != cudaSuccess || + cudaStreamCreateWithFlags(&camera.cudaStream, cudaStreamNonBlocking) != cudaSuccess) + { + return fail(); + } + } +#endif + VkCommandPoolCreateInfo poolInfo{ VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO }; + poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + poolInfo.queueFamilyIndex = camera.queueFamily; + if (!vkOk(vkCreateCommandPool(camera.device, &poolInfo, nullptr, &camera.pool))) + { + return fail(); + } + VkCommandBufferAllocateInfo commandInfo{ VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO }; + commandInfo.commandPool = camera.pool; + commandInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + commandInfo.commandBufferCount = 1; + VkFenceCreateInfo fenceInfo{ VK_STRUCTURE_TYPE_FENCE_CREATE_INFO }; + if (!vkOk(vkAllocateCommandBuffers(camera.device, &commandInfo, &camera.command)) || + !vkOk(vkCreateFence(camera.device, &fenceInfo, nullptr, &camera.fence))) + { + return fail(); + } + return true; +} + +bool upload(CameraSession& camera, const Dispatch& dispatch, const void* rgba) +{ + camera.ready = false; + if (camera.retired) + { + return false; + } + if (camera.fencePending) + { + if (!vkOk(vkWaitForFences(camera.device, 1, &camera.fence, VK_TRUE, UINT64_MAX))) + { + camera.retired = true; + return false; + } + camera.fencePending = false; + } + const VkDeviceSize length = static_cast(camera.width) * camera.height * 4; + if (rgba) + { + std::memcpy(camera.pixels, rgba, static_cast(length)); + } + if (!camera.acquired) + { + XrSwapchainImageAcquireInfo acquireInfo{ XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO }; + if (dispatch.acquire(camera.swapchain, &acquireInfo, &camera.acquiredIndex) != XR_SUCCESS) + { + return false; + } + camera.acquired = true; + } + const auto release = [&]() -> bool + { + if (camera.waited) + { + XrSwapchainImageReleaseInfo info{ XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO }; + if (dispatch.release(camera.swapchain, &info) != XR_SUCCESS) + { + camera.retired = true; + return false; + } + camera.waited = false; + camera.acquired = false; + } + return true; + }; + if (!camera.waited) + { + XrSwapchainImageWaitInfo waitInfo{ XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO }; + waitInfo.timeout = XR_INFINITE_DURATION; + const XrResult waited = dispatch.wait(camera.swapchain, &waitInfo); + if (waited == XR_TIMEOUT_EXPIRED) + { + return false; // Retry this same acquisition on the next publication. + } + if (waited != XR_SUCCESS) + { + camera.retired = true; + return false; + } + camera.waited = true; + } + if (camera.acquiredIndex >= camera.images.size()) + { + release(); + camera.retired = true; + return false; + } + if (!vkOk(vkResetFences(camera.device, 1, &camera.fence)) || !vkOk(vkResetCommandPool(camera.device, camera.pool, 0))) + { + release(); + return false; + } + VkCommandBufferBeginInfo begin{ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; + if (!vkOk(vkBeginCommandBuffer(camera.command, &begin))) + { + release(); + return false; + } +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + VkBufferMemoryBarrier fromCuda{ VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER }; + if (camera.cudaPixels) + { + fromCuda.srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; + fromCuda.dstQueueFamilyIndex = camera.queueFamily; + fromCuda.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + fromCuda.buffer = camera.staging; + fromCuda.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(camera.command, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, + nullptr, 1, &fromCuda, 0, nullptr); + } +#endif + const VkImage image = camera.images[camera.acquiredIndex].image; + VkImageMemoryBarrier toTransfer{ VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER }; + toTransfer.oldLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toTransfer.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.image = image; + toTransfer.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT; + toTransfer.subresourceRange = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 }; + toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + vkCmdPipelineBarrier(camera.command, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 0, nullptr, 1, &toTransfer); + VkBufferImageCopy copy{}; + copy.imageSubresource = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1 }; + copy.imageExtent = { camera.width, camera.height, 1 }; + vkCmdCopyBufferToImage(camera.command, camera.staging, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (camera.cudaPixels) + { + VkBufferMemoryBarrier toCuda = fromCuda; + toCuda.srcQueueFamilyIndex = camera.queueFamily; + toCuda.dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; + toCuda.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toCuda.dstAccessMask = 0; + vkCmdPipelineBarrier(camera.command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, + nullptr, 1, &toCuda, 0, nullptr); + } +#endif + VkImageMemoryBarrier toColor = toTransfer; + toColor.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toColor.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + toColor.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toColor.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT; + vkCmdPipelineBarrier(camera.command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, 0, nullptr, 0, nullptr, 1, &toColor); + if (!vkOk(vkEndCommandBuffer(camera.command))) + { + release(); + return false; + } + VkSubmitInfo submit{ VK_STRUCTURE_TYPE_SUBMIT_INFO }; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &camera.command; + if (!vkOk(vkQueueSubmit(camera.queue, 1, &submit, camera.fence))) + { + camera.retired = true; + release(); + return false; + } + camera.fencePending = true; + // Complete the bounded owned copy before the runtime reuses this image. + if (!vkOk(vkWaitForFences(camera.device, 1, &camera.fence, VK_TRUE, UINT64_MAX))) + { + camera.retired = true; + return false; + } + camera.fencePending = false; + if (!release()) + { + return false; + } + camera.ready = true; + return true; +} + +} // namespace + +extern "C" PanelSession* isaacteleop_pip_create() +{ + return new (std::nothrow) PanelSession; +} + +extern "C" uint32_t isaacteleop_pip_max_cameras() +{ + return kMaxCameras; +} +extern "C" uint32_t isaacteleop_pip_max_image_dimension() +{ + return kMaxImageDimension; +} + +// Explicit application binding. All entry points are called on the OVXR owner +// thread, serialized with that context's queue and frame operations. +extern "C" int isaacteleop_pip_attach(PanelSession* owner, + XrInstance instance, + XrSession session, + PFN_xrGetInstanceProcAddr gipa, + VkPhysicalDevice physical, + VkDevice device, + VkQueue queue, + uint32_t queueFamily) +{ + if (!instance || !session || !gipa || !physical || !device || !queue || !owner || owner->attached) + { + return -1; + } + Dispatch dispatch{}; + dispatch.gipa = gipa; + if (!resolve(gipa, instance, "xrCreateReferenceSpace", dispatch.createSpace) || + !resolve(gipa, instance, "xrDestroySpace", dispatch.destroySpace) || + !resolve(gipa, instance, "xrEnumerateSwapchainFormats", dispatch.formats) || + !resolve(gipa, instance, "xrCreateSwapchain", dispatch.createSwapchain) || + !resolve(gipa, instance, "xrDestroySwapchain", dispatch.destroySwapchain) || + !resolve(gipa, instance, "xrEnumerateSwapchainImages", dispatch.images) || + !resolve(gipa, instance, "xrAcquireSwapchainImage", dispatch.acquire) || + !resolve(gipa, instance, "xrWaitSwapchainImage", dispatch.wait) || + !resolve(gipa, instance, "xrReleaseSwapchainImage", dispatch.release)) + { + return -2; + } + // Register before allocation so even failed partial setup has an explicit + // detach owner. Error -3 tells the binding to retain this cleanup obligation. + owner->dispatch = dispatch; + owner->attached = true; + auto& cameras = owner->cameras; + for (auto& camera : cameras) + { + camera.instance = instance; + camera.session = session; + camera.physical = physical; + camera.device = device; + camera.queue = queue; + camera.queueFamily = queueFamily; + XrReferenceSpaceCreateInfo space{ XR_TYPE_REFERENCE_SPACE_CREATE_INFO }; + space.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_VIEW; + space.poseInReferenceSpace.orientation.w = 1.0f; + if (XR_FAILED(dispatch.createSpace(session, &space, &camera.view))) + { + return -3; + } + } + return 0; +} + +extern "C" int isaacteleop_pip_destroy(PanelSession* owner) +{ + if (!owner) + return 0; + if (owner->attached) + { + for (auto& camera : owner->cameras) + { + if (!releaseSessionResources(camera, owner->dispatch)) + return -1; // Retain the owner until all borrowed reads retire. + } + } + delete owner; + return 0; +} + +extern "C" const XrCompositionLayerBaseHeader* isaacteleop_pip_layer(PanelSession* owner, uint32_t cameraIndex) +{ + if (!owner || !owner->attached || cameraIndex >= kMaxCameras) + { + return nullptr; + } + const auto& camera = owner->cameras[cameraIndex]; + const auto& panel = owner->panels[cameraIndex]; + if (!camera.ready || camera.retired || !panel.enabled || !camera.swapchain || + !(panel.headLocked ? camera.view : camera.local)) + { + return nullptr; + } + auto& quad = owner->quads[cameraIndex]; + quad = { XR_TYPE_COMPOSITION_LAYER_QUAD }; + quad.layerFlags = 0; // Opaque camera panels, independent of projection alpha. + quad.space = panel.headLocked ? camera.view : camera.local; + quad.eyeVisibility = XR_EYE_VISIBILITY_BOTH; + quad.subImage.swapchain = camera.swapchain; + quad.subImage.imageRect.extent = { static_cast(camera.width), static_cast(camera.height) }; + quad.pose = panel.pose; + quad.size = panel.size; + return reinterpret_cast(&quad); +} + +extern "C" int isaacteleop_pip_configure( + PanelSession* owner, uint32_t cameraIndex, int enabled, int headLocked, const float* pose7, const float* size2) +{ + if (!owner || cameraIndex >= kMaxCameras || !pose7 || !size2 || !std::isfinite(size2[0]) || + !std::isfinite(size2[1]) || size2[0] <= 0 || size2[1] <= 0) + { + return -1; + } + float orientationLength = 0.0f; + for (int i = 0; i < 7; ++i) + { + if (!std::isfinite(pose7[i])) + { + return -1; + } + if (i >= 3) + { + orientationLength += pose7[i] * pose7[i]; + } + } + if (orientationLength < 0.99f || orientationLength > 1.01f) + { + return -1; + } + auto& panel = owner->panels[cameraIndex]; + panel.enabled = enabled != 0; + panel.headLocked = headLocked != 0; + panel.pose = { { pose7[3], pose7[4], pose7[5], pose7[6] }, { pose7[0], pose7[1], pose7[2] } }; + panel.size = { size2[0], size2[1] }; + return 0; +} + +extern "C" int isaacteleop_pip_publish_rgba( + PanelSession* owner, uint32_t cameraIndex, const void* rgba, uint32_t width, uint32_t height, size_t length) +{ + if (!owner || cameraIndex >= kMaxCameras || !rgba || !width || !height || width > kMaxImageDimension || + height > kMaxImageDimension || static_cast(width) * height * 4 != length) + { + return -1; + } + if (!owner->attached) + { + return -2; + } + CameraSession& camera = owner->cameras[cameraIndex]; + const auto& panel = owner->panels[cameraIndex]; + if (!camera.device || !camera.queue || camera.retired || !(panel.headLocked ? camera.view : camera.local)) + { + return -3; + } + const Dispatch& dispatch = owner->dispatch; + if ((camera.width != width || camera.height != height || !camera.swapchain) && + !createSwapchain(camera, dispatch, width, height)) + { + return -4; + } +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (camera.cudaDevice >= 0) + { + return -9; // Do not replace live CUDA aliases with CPU staging resources. + } +#endif + if ((!camera.staging || !camera.memory || !camera.pixels || !camera.pool || !camera.command || !camera.fence) && + !createUploadResources(camera)) + { + return -5; + } + return upload(camera, dispatch, rgba) ? 0 : -6; +} + + +// Application API, not an OVXR export. The caller owns a completed OVRTX map +// until this synchronous upload returns. -7 means completion is unproved and +// that map MUST remain retained; no further upload is admitted on the session. +extern "C" int isaacteleop_pip_supports_cuda() +{ +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + return 1; +#else + return 0; +#endif +} + +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) +namespace +{ +cudaError_t copyCameraArray(CameraSession& camera, const void* array, uint32_t width, uint32_t height, int sourceDevice) +{ + if (sourceDevice == camera.cudaDevice) + { + return cudaMemcpy2DFromArrayAsync(camera.cudaPixels, static_cast(width) * 4, + static_cast(array), 0, 0, static_cast(width) * 4, + height, cudaMemcpyDeviceToDevice, camera.cudaStream); + } + // RenderProducts can be assigned to a different GPU from the XR binding. + // Preserve their assignment; copy to the binding's device, not host pixels. + cudaMemcpy3DPeerParms copy{}; + copy.srcArray = const_cast(static_cast(array)); + copy.srcDevice = sourceDevice; + copy.dstPtr = { camera.cudaPixels, static_cast(width) * 4, static_cast(width) * 4, height }; + copy.dstDevice = camera.cudaDevice; + // With a CUDA array, extent.width is in array elements (RGBA pixels). + copy.extent = { width, height, 1 }; + return cudaMemcpy3DPeerAsync(©, camera.cudaStream); +} + +int copyCameraArrayAndWait(CameraSession& camera, const void* array, uint32_t width, uint32_t height, int sourceDevice) +{ + camera.cudaPending = true; + const auto copied = copyCameraArray(camera, array, width, height, sourceDevice); + // CUDA calls may surface an earlier asynchronous error. Even an enqueue + // failure must prove that this stream has stopped reading the borrowed map. + if (cudaStreamSynchronize(camera.cudaStream) != cudaSuccess) + { + camera.retired = true; + return -7; + } + camera.cudaPending = false; + return copied == cudaSuccess ? 0 : -6; +} +} // namespace +#endif + +extern "C" int isaacteleop_pip_publish_cuda_array( + PanelSession* owner, uint32_t cameraIndex, const void* array, uint32_t width, uint32_t height, int device) +{ +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (!owner || cameraIndex >= kMaxCameras || !array || !width || !height || width > kMaxImageDimension || + height > kMaxImageDimension || device < 0) + { + return -1; + } + if (!owner->attached) + { + return -2; + } + CameraSession& camera = owner->cameras[cameraIndex]; + if (!camera.device || !camera.queue || camera.retired) + { + return -3; + } + // Resolve the borrowed Vulkan device once. A camera output's CUDA ordinal + // is a source identity, not proof it is the destination Vulkan device. + if (camera.vulkanCudaDevice < 0) + { + VkPhysicalDeviceIDProperties id{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES }; + VkPhysicalDeviceProperties2 properties{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2 }; + properties.pNext = &id; + vkGetPhysicalDeviceProperties2(camera.physical, &properties); + int count = 0; + if (cudaGetDeviceCount(&count) != cudaSuccess) + { + return -8; + } + for (int candidate = 0; candidate < count; ++candidate) + { + cudaDeviceProp properties{}; + if (cudaGetDeviceProperties(&properties, candidate) == cudaSuccess && + std::memcmp(id.deviceUUID, properties.uuid.bytes, VK_UUID_SIZE) == 0) + { + camera.vulkanCudaDevice = candidate; + break; + } + } + if (camera.vulkanCudaDevice < 0) + { + return -8; + } + } + if (cudaSetDevice(camera.vulkanCudaDevice) != cudaSuccess) + { + return -8; + } + const Dispatch& dispatch = owner->dispatch; + if ((camera.width != width || camera.height != height || !camera.swapchain) && + !createSwapchain(camera, dispatch, width, height)) + { + return -4; + } + if (!camera.cudaPixels || !camera.pool || !camera.command || !camera.fence) + { + if (!releaseImageResources(camera)) + { + return -7; + } + camera.cudaDevice = camera.vulkanCudaDevice; + if (!createUploadResources(camera, true)) + { + return -5; + } + } + if (camera.fencePending) + { + if (vkWaitForFences(camera.device, 1, &camera.fence, VK_TRUE, UINT64_MAX) != VK_SUCCESS) + { + camera.retired = true; + return -6; + } + camera.fencePending = false; + } + camera.ready = false; + const int copied = copyCameraArrayAndWait(camera, array, width, height, device); + if (copied != 0) + { + return copied; + } + // No external semaphore wait is introduced: preserve the current host-wait + // scheduling policy. This is a GPU-only copy, not end-to-end zero-copy. + return upload(camera, dispatch, nullptr) ? 0 : -6; +#else + return -8; +#endif +} diff --git a/src/core/python/CMakeLists.txt b/src/core/python/CMakeLists.txt index b2edfa611f..c55fd4a7d0 100644 --- a/src/core/python/CMakeLists.txt +++ b/src/core/python/CMakeLists.txt @@ -41,6 +41,13 @@ foreach(_license_file IN LISTS _license_files_out) endforeach() # Generate pyproject.toml from template with ISAAC_TELEOP_PYPROJECT_VERSION +if(BUILD_OVXR_CAMERA_QUADS) + set(OVXR_CAMERA_PACKAGE_DATA_BLOCK + "\"isaaccapture.ovxr_session_manager\" = [\"native/*.so\"]\n") +else() + set(OVXR_CAMERA_PACKAGE_DATA_BLOCK "") +endif() + configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/pyproject.toml.in" "${CMAKE_CURRENT_BINARY_DIR}/pyproject.toml" @@ -87,6 +94,10 @@ add_custom_target(python_package ALL COMMENT "Preparing Python package structure" ) +if(TARGET ovxr_pip_layer) + add_dependencies(python_package ovxr_pip_layer) +endif() + # viz_py only exists when BUILD_VIZ=ON. Use add_dependencies (not a # DEPENDS genex) so the build-order constraint is unconditional under # all generators — Windows ninja runs the wheel build in parallel @@ -260,7 +271,17 @@ if(SKBUILD) USE_SOURCE_PERMISSIONS PATTERN "*.py" EXCLUDE PATTERN "__pycache__" EXCLUDE + # Reused staging can retain a formerly enabled camera helper. The option, + # not old files in the build tree, owns its inclusion in a scikit wheel. + PATTERN "native" EXCLUDE ) + if(BUILD_OVXR_CAMERA_QUADS) + install(TARGETS ovxr_pip_layer + LIBRARY DESTINATION "isaaccapture/ovxr_session_manager/native" + COMPONENT isaacteleop_wheel + ) + + endif() install( DIRECTORY "${ROBOTIC_GROUNDING_STAGED_DIR}" DESTINATION "." diff --git a/src/core/python/pyproject.toml.in b/src/core/python/pyproject.toml.in index 05eec7faf9..33c1855baf 100644 --- a/src/core/python/pyproject.toml.in +++ b/src/core/python/pyproject.toml.in @@ -102,6 +102,7 @@ robotic_grounding = [ "assets/xmls/sharpawave/NOTICE", "assets/xmls/sharpawave/README.md", ] +@OVXR_CAMERA_PACKAGE_DATA_BLOCK@ [tool.uv] # Only use managed Python installations (not system Python) diff --git a/src/core/python/requirements-ovxr.txt b/src/core/python/requirements-ovxr.txt index 735767ae6b..076edbaff8 100644 --- a/src/core/python/requirements-ovxr.txt +++ b/src/core/python/requirements-ovxr.txt @@ -3,3 +3,5 @@ # Application overrides: docs/source/getting_started/ovxr.rst. ovxr==0.1.0.5842 +# GPU camera output uses CUDA 13 bindings, matching the Isaac Lab runtime. +cuda-bindings>=13,<14 diff --git a/src/core/python/setup.py b/src/core/python/setup.py index a32e276ab9..c99a3cb283 100644 --- a/src/core/python/setup.py +++ b/src/core/python/setup.py @@ -8,7 +8,10 @@ we need to override setuptools to recognize this as a platform wheel, not pure Python. """ +from pathlib import Path + from setuptools import setup +from setuptools.command.build_py import build_py from setuptools.dist import Distribution @@ -19,4 +22,21 @@ def has_ext_modules(self): return True -setup(distclass=BinaryDistribution) +class BuildPy(build_py): + """Retire optional camera data left by a previous build configuration.""" + + def run(self): + super().run() + root = Path(self.build_lib) / "isaaccapture/ovxr_session_manager" + # Retire outputs from the former API-layer layout on incremental builds. + for name in ( + "libXrApiLayer_isaacteleop_pip.so", + "XrApiLayer_isaacteleop_pip.json", + ): + (root / "openxr_layer" / name).unlink(missing_ok=True) + if not self.package_data.get("isaaccapture.ovxr_session_manager"): + # Setuptools reuses build_lib; omitted package data is not deleted. + (root / "native/libisaaccapture_camera_panels.so").unlink(missing_ok=True) + + +setup(distclass=BinaryDistribution, cmdclass={"build_py": BuildPy}) diff --git a/src/python/isaaccapture/ovxr_session_manager/__init__.py b/src/python/isaaccapture/ovxr_session_manager/__init__.py index 8528115853..d85ab4d2ad 100644 --- a/src/python/isaaccapture/ovxr_session_manager/__init__.py +++ b/src/python/isaaccapture/ovxr_session_manager/__init__.py @@ -5,8 +5,11 @@ from .ovxr_session import ( OvxrSessionBinding, + OvxrScenePublication, + OvxrResourceOwner, OvxrSessionConfig, OvxrApplicationInterface, + OvxrFrameConsumer, OvxrSession, StageOrdinalLedger, ) @@ -15,10 +18,14 @@ "OvxrSessionBinding", "OvxrSessionConfig", "OvxrApplicationInterface", + "OvxrFrameConsumer", "OvxrSession", "StageOrdinalLedger", ] -from .interfaces import OvxrScenePublication, OvxrResourceOwner +from .camera_panels import CameraPanelConfig, CameraPanelConsumer +from .camera_mailbox import FrameOutputs + +__all__ += ["CameraPanelConfig", "CameraPanelConsumer", "FrameOutputs"] __all__ += ["OvxrScenePublication", "OvxrResourceOwner"] diff --git a/src/python/isaaccapture/ovxr_session_manager/camera_mailbox.py b/src/python/isaaccapture/ovxr_session_manager/camera_mailbox.py new file mode 100644 index 0000000000..3c003cf710 --- /dev/null +++ b/src/python/isaaccapture/ovxr_session_manager/camera_mailbox.py @@ -0,0 +1,233 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded completed camera outputs for later application presentation. + +CPU snapshots own their bytes. GPU snapshots retain an OVRTX map until +next production or teardown; they may be consumed only on the render thread, +with synchronous copy completion. Cadence gaps retain the last completed image. +No borrowed result/output handle survives the callback. +""" + +from __future__ import annotations + +import ctypes +import threading +from dataclasses import dataclass +from typing import Any, Callable, TypeVar + +from .gpu_output import MappedGpuImage, MappedOutput, OutputMapCleanupError, _api + +_Image = TypeVar("_Image") + + +@dataclass(frozen=True) +class CameraImage: + width: int + height: int + rgba: bytes | None + sensor_time: float + revision: int + gpu: MappedGpuImage | None = None + + +def _text(value: Any) -> str: + if not value.ptr: + return "" + return ctypes.string_at(value.ptr, value.length).decode("utf-8") + + +def _latest_color_handle(outputs: Any, product_path: str) -> tuple[int, float] | None: + """Select the latest *complete* LdrColor frame, not the first array entry.""" + latest: tuple[int, float] | None = None + produced = False + for product_index in range(outputs.output_count): + product = outputs.outputs[product_index] + if _text(product.render_product_path) != product_path: + continue + for frame_index in range(product.output_frame_count): + produced = True + frame = product.output_frames[frame_index] + for var_index in range(frame.render_var_count): + var = frame.output_render_vars[var_index] + if _text(var.source_name) != "LdrColor" or not var.output_handle: + continue + candidate = (int(var.output_handle), float(frame.frame_end_time)) + if latest is None or candidate[1] >= latest[1]: + latest = candidate + if produced and latest is None: + raise RuntimeError(f"{product_path} produced a frame without LdrColor") + return latest + + +def _copy_host_rgba( + renderer: int, handle: int, max_bytes: int +) -> tuple[int, int, bytes]: + """Map one completed OVRTX output to CPU and copy before unmap.""" + b, lib = _api() + description = b.ovrtx_map_output_description_t( + device_type=b.Device.CPU, sync_stream=None + ) + mapped = b.ovrtx_render_var_output_t() + result = lib.ovrtx_map_render_var_output( + renderer, + handle, + ctypes.byref(description), + b.OVRTX_TIMEOUT_INFINITE, + ctypes.byref(mapped), + ) + if result.status != b.OVRTX_API_SUCCESS: + raise RuntimeError("OVRTX camera output map failed") + lease = MappedOutput(renderer=renderer, mapped=mapped, owner=threading.get_ident()) + failure = None + try: + if mapped.status != b.EventStatus.COMPLETED or mapped.num_tensors != 1: + raise RuntimeError( + "OVRTX camera output is incomplete or has unexpected tensors" + ) + tensor = mapped.tensors[0].dl.contents + shape = tuple(int(tensor.shape[i]) for i in range(tensor.ndim)) + dtype = tensor.dtype + if dtype.code.value != 1 or dtype.bits != 8 or not tensor.data: + raise RuntimeError("OVRTX camera output is not CPU uint8") + if len(shape) == 3 and shape[2] == 4 and dtype.lanes == 1: + height, width, _ = shape + expected_strides = (width * 4, 4, 1) + elif len(shape) == 2 and dtype.lanes == 4: + height, width = shape + expected_strides = (width, 1) + else: + raise RuntimeError(f"unexpected OVRTX camera tensor shape {shape}") + length = width * height * 4 + if width <= 0 or height <= 0 or length > max_bytes: + raise RuntimeError("OVRTX camera output exceeds bounded mailbox size") + if ( + tensor.strides + and tuple(tensor.strides[i] for i in range(tensor.ndim)) != expected_strides + ): + raise RuntimeError("OVRTX camera output is not contiguous RGBA8") + return width, height, ctypes.string_at(tensor.data + tensor.byte_offset, length) + except BaseException as error: + failure = error + raise + finally: + try: + lease.close() + except Exception as cleanup: + # Keep the native handle even when validation or the host copy failed. + raise OutputMapCleanupError(lease, failure, cleanup) from failure + + +class FrameOutputs: + """Select and copy completed color outputs during the owner-thread callback. + + Output handles are opaque and valid only within this frame. CPU snapshots + are shared; GPU mappings belong to the consumer and must be retired by it. + """ + + def __init__(self, renderer: int, outputs: Any) -> None: + self._renderer = renderer + self._outputs = outputs + self._host_images = {} + + def latest_color(self, product_path: str) -> tuple[int, float] | None: + """Return the latest color handle and sensor time, or None for a cadence gap.""" + return _latest_color_handle(self._outputs.contents, product_path) + + def map_gpu( + self, handle: int, max_bytes: int, mapper: Callable[[int, int, int], _Image] + ) -> _Image: + """Map an output using the consumer's borrowed or owned GPU copy policy.""" + return mapper(self._renderer, handle, max_bytes) + + def copy_host_rgba(self, handle: int, max_bytes: int) -> tuple[int, int, bytes]: + """Copy once per output and enforce each consumer's byte bound on reuse.""" + if handle not in self._host_images: + self._host_images[handle] = _copy_host_rgba( + self._renderer, handle, max_bytes + ) + width, height, rgba = self._host_images[handle] + if len(rgba) > max_bytes: + raise RuntimeError("OVRTX camera output exceeds bounded mailbox size") + return width, height, rgba + + +class CompletedCameraMailbox: + """Latest camera image and explicit error/omission state for one product.""" + + def __init__( + self, + product_path: str, + max_image_bytes: int, + *, + gpu_mapper=None, + ) -> None: + if not product_path.startswith("/") or max_image_bytes <= 0: + raise ValueError( + "camera mailbox requires an absolute product path and positive byte bound" + ) + self.product_path = product_path + self.max_image_bytes = max_image_bytes + self._gpu_mapper = gpu_mapper + self._mapped = None + self._gpu_error = None + self._lock = threading.Lock() + self._image: CameraImage | None = None + self._revision = 0 + self._error: str | None = None + + def snapshot(self) -> CameraImage | None: + with self._lock: + return self._image + + def error(self) -> str | None: + with self._lock: + return self._error + + def on_outputs(self, frame: FrameOutputs) -> None: + """Consume on the render thread; contain errors and retain only owned data/maps.""" + try: + if self._gpu_error is not None: + raise RuntimeError(self._gpu_error) + selected = frame.latest_color(self.product_path) + if selected is None: + return + handle, sensor_time = selected + gpu = None + if self._gpu_mapper is not None: + self.close() + gpu = frame.map_gpu(handle, self.max_image_bytes, self._gpu_mapper) + self._mapped = gpu + width, height, rgba = gpu.width, gpu.height, None + else: + width, height, rgba = frame.copy_host_rgba(handle, self.max_image_bytes) + if len(rgba) != width * height * 4 or len(rgba) > self.max_image_bytes: + raise RuntimeError("camera copier returned invalid RGBA8 size") + with self._lock: + self._revision += 1 + self._image = CameraImage( + width, + height, + bytes(rgba) if rgba is not None else None, + sensor_time, + self._revision, + gpu, + ) + self._error = None + except Exception as exc: + if isinstance(exc, OutputMapCleanupError): + self._mapped = exc.image + self._gpu_error = str(exc) + with self._lock: + self._image = None + self._error = str(exc) + + def close(self) -> None: + """Frame owner only, after synchronous quad consumption, before renderer destruction.""" + if self._mapped is not None: + try: + self._mapped.close() + except Exception as error: + self._gpu_error = str(error) + raise + self._mapped = None diff --git a/src/python/isaaccapture/ovxr_session_manager/camera_panels.py b/src/python/isaaccapture/ovxr_session_manager/camera_panels.py new file mode 100644 index 0000000000..ce9188931e --- /dev/null +++ b/src/python/isaaccapture/ovxr_session_manager/camera_panels.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Camera picture-in-picture as an application-neutral OVXR frame consumer.""" + +from __future__ import annotations + +import logging +import math +from dataclasses import dataclass +from typing import Literal + +from .camera_mailbox import CompletedCameraMailbox, FrameOutputs +from .gpu_output import map_cuda_array_rgba +from .openxr_quad import OpenXRCameraPanels +from .ovxr_session import OvxrFrameConsumer + +_LOGGER = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class CameraPanelConfig: + """One host-produced camera and its head-locked presentation policy.""" + + render_product_path: str + max_image_bytes: int = 64 * 1024 * 1024 + panel_width_m: float = 0.48 + position: tuple[float, float, float] = (0.0, -0.15, -0.8) + orientation_xyzw: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 1.0) + space: Literal["head_locked"] = "head_locked" + label: str | None = None + + def __post_init__(self) -> None: + if not self.render_product_path.startswith("/"): + raise ValueError("camera render product path must be absolute") + if self.max_image_bytes <= 0: + raise ValueError("camera panel max_image_bytes must be positive") + if not math.isfinite(self.panel_width_m) or self.panel_width_m <= 0.0: + raise ValueError("camera panel panel_width_m must be finite and positive") + if len(self.position) != 3 or not all( + math.isfinite(value) for value in self.position + ): + raise ValueError("camera panel position must contain three finite values") + if len(self.orientation_xyzw) != 4 or not all( + math.isfinite(value) for value in self.orientation_xyzw + ): + raise ValueError("camera panel orientation must contain four finite values") + norm = math.sqrt(sum(value * value for value in self.orientation_xyzw)) + if norm <= 0.0: + raise ValueError("camera panel orientation must have non-zero norm") + object.__setattr__( + self, + "orientation_xyzw", + tuple(value / norm for value in self.orientation_xyzw), + ) + if self.space != "head_locked": + raise ValueError("camera panel space must be 'head_locked'") + + +class CameraPanelConsumer(OvxrFrameConsumer): + """Own camera uploads and composition layers until the XR owner retires them.""" + + def __init__(self, panels: tuple[CameraPanelConfig, ...]) -> None: + self.panels = tuple(panels) + paths = self.render_product_paths + if len(paths) != len(set(paths)): + raise ValueError("camera panels require unique paths") + self._native = None + self._mailboxes = [] + self._layers = () + + @property + def render_product_paths(self) -> tuple[str, ...]: + return tuple(panel.render_product_path for panel in self.panels) + + def on_session_started(self, binding, config, request_shutdown) -> None: + self._native = OpenXRCameraPanels(self.panels) + for panel in self.panels: + self._mailboxes.append( + CompletedCameraMailbox( + panel.render_product_path, + panel.max_image_bytes, + gpu_mapper=map_cuda_array_rgba + if self._native.supports_gpu + else None, + ) + ) + + def on_openxr_ready(self, openxr, vulkan) -> bool: + if vulkan is None: + return False + self._native.attach(openxr, vulkan) + return True + + def on_outputs(self, frame: FrameOutputs) -> None: + layers = [] + self._layers = () + for index, mailbox in enumerate(self._mailboxes): + try: + mailbox.on_outputs(frame) + self._native.publish( + index, mailbox.snapshot(), invalid=mailbox.error() is not None + ) + if layer := self._native.layer(index): + layers.append(layer) + except Exception: + _LOGGER.exception("OpenXR camera quad publication failed") + try: + self._native.hide(index) + except Exception: + _LOGGER.exception("OpenXR camera quad hiding failed") + self._layers = tuple(layers) + + def composition_layers(self) -> tuple[int, ...]: + return self._layers + + def finish_frame(self) -> None: + for mailbox in self._mailboxes: + mailbox.close() + + def close(self) -> None: + self.finish_frame() + if self._native is not None: + self._native.close() + self._native = None + self._mailboxes.clear() + self._layers = () diff --git a/src/python/isaaccapture/ovxr_session_manager/frame_pump.py b/src/python/isaaccapture/ovxr_session_manager/frame_pump.py index f13eba25bc..f5b0dc8c63 100644 --- a/src/python/isaaccapture/ovxr_session_manager/frame_pump.py +++ b/src/python/isaaccapture/ovxr_session_manager/frame_pump.py @@ -9,6 +9,8 @@ from collections.abc import Callable, Iterable from .interfaces import OvxrApplicationInterface, OvxrSessionBinding, OvxrSessionConfig from .runtime_owner import CloudXRRuntimeOwner +from .camera_mailbox import FrameOutputs +from .gpu_output import completed_outputs, native_renderer_handle _LOGGER = logging.getLogger(__name__) @@ -46,9 +48,11 @@ def __init__( self._context_lifecycle_lock = threading.Lock() self._last_ordinal: int | None = None self._first_ordinal_validated = False + self._frame_consumers = () def attach(self, capability: object) -> None: self._binding = self._host.attach_on_render_thread(capability) + self._frame_consumers = tuple(self._binding.frame_consumers) @staticmethod def validate_provider() -> None: @@ -77,6 +81,13 @@ def create_context(self) -> None: if self._shutdown_requested(): return ovxr = _import_ovxr() + paths = tuple( + dict.fromkeys( + path + for consumer in self._frame_consumers + for path in consumer.render_product_paths + ) + ) extensions = list( dict.fromkeys( ( @@ -91,9 +102,14 @@ def create_context(self) -> None: ovstage=self._binding.stage, application_name=self._config.application_name, display_mode=ovxr.DisplayMode.STEREO, + # The server mode must reach OVXR; the browser AR preference cannot + # enable transparent rendering in an opaque server projection. session_mode={"ar": ovxr.SessionMode.AR, "vr": ovxr.SessionMode.VR}[ self._config.session_mode ], + # Empty means OVXR creates its default eye products. Supplying + # those names explicitly instead means existing products and + # changes the production/authoring contract. render_products=list(self._config.render_products), camera_paths=list(self._config.camera_paths), create_render_products=self._config.create_render_products, @@ -110,6 +126,12 @@ def create_context(self) -> None: ) with self._host.owner_lane(): self._context = ovxr.Context(desc) + if self._frame_consumers: + self._context.set_frame_render_products(list(paths)) + for consumer in self._frame_consumers: + consumer.on_session_started( + self._binding, self._config, self.request_shutdown + ) def request_exit(self) -> None: """Wake an in-flight OpenXR frame wait without stopping CloudXR.""" @@ -147,6 +169,10 @@ def external_handles(self): return None from isaaccapture.oxr import OpenXRSessionHandles + vulkan = self._context.get_vulkan_handles() if self._frame_consumers else None + for consumer in self._frame_consumers: + if not consumer.on_openxr_ready(handles, vulkan): + return None return OpenXRSessionHandles( handles.xr_instance, handles.xr_session, @@ -194,6 +220,8 @@ def render(self, ordinal: int) -> bool: raise self._binding.ordinal_ledger.resolve_external( frame_ordinal, + # Publication is independent of the eventual bind result: + # a later failure must not reuse an already sealed ordinal. expected_published=sealed, ) if result not in ( @@ -206,7 +234,19 @@ def render(self, ordinal: int) -> bool: ) if result != ovxr.Result.SUCCESS: return True - result, _frame = self._context.submit_frame(active_plan) + layers = [] + if self._frame_consumers and active_plan.step_result: + try: + renderer = native_renderer_handle(self._binding.renderer) + outputs = completed_outputs(renderer, active_plan.step_result) + layers = self._on_completed_outputs(renderer, outputs) + except Exception: + _LOGGER.exception("Optional bound-frame output observation failed") + finally: + self._finish_frame_outputs() + result, _frame = self._context.submit_frame( + active_plan, additional_layers=layers + ) if result not in ( ovxr.Result.SUCCESS, ovxr.Result.TIMEOUT, @@ -215,6 +255,17 @@ def render(self, ordinal: int) -> bool: raise RuntimeError( f"OVXR submit_frame failed: {ovxr.result_name(result)}: {self._context.last_error}" ) + if ( + result == ovxr.Result.SUCCESS + and _frame.submitted + and self._frame_consumers + ): + # Submit consumed the plan; consumer callbacks must not + # cancel it even if a callback fails. + active_plan = None + self._pending_plan = None + for consumer in self._frame_consumers: + consumer.on_frame_submitted() active_plan = None self._pending_plan = None except BaseException as error: @@ -222,7 +273,10 @@ def render(self, ordinal: int) -> bool: raise finally: if active_plan is not None: + # An unproved copy/unmap keeps the plan and context alive. Never + # consume borrowed output storage merely to recover a frame. try: + self._finish_frame_outputs() self._context.cancel_frame(active_plan) self._pending_plan = None except BaseException as cleanup_error: @@ -243,15 +297,28 @@ def close(self) -> None: self._host.close_publication() except BaseException as exc: first_error = exc - if self._pending_plan is not None: - try: - self._context.cancel_frame(self._pending_plan) - except BaseException as exc: - # Native destroy remains responsible for the pending frame. - if first_error is None: - first_error = exc - else: - self._pending_plan = None + try: + for consumer in reversed(self._frame_consumers): + # A failed completion/unmap must keep the renderer alive: never + # destroy storage that an optional consumer might still access. + consumer.close() + if self._pending_plan is not None: + self._finish_frame_outputs() + try: + self._context.cancel_frame(self._pending_plan) + except BaseException as exc: + # All external mappings are retired. Native destroy remains + # responsible for draining the context's pending frame. + if first_error is None: + first_error = exc + else: + self._pending_plan = None + except BaseException as retirement_error: + if first_error is not None: + first_error.add_note(f"Consumer retirement failed: {retirement_error}") + # Keep the primary error and the failed retirement cause visible. + raise first_error from retirement_error + raise context_retired = self._context is None if self._context is not None: assert self._binding is not None @@ -321,6 +388,21 @@ def close(self) -> None: if first_error is not None: raise first_error + def _on_completed_outputs(self, renderer, outputs) -> list[int]: + frame = FrameOutputs(renderer, outputs) + layers = [] + for consumer in self._frame_consumers: + try: + consumer.on_outputs(frame) + layers.extend(consumer.composition_layers()) + except Exception: + _LOGGER.exception("Optional frame consumer failed") + return layers + + def _finish_frame_outputs(self) -> None: + for consumer in self._frame_consumers: + consumer.finish_frame() + def retry_runtime_stop(self) -> None: """Retry service cleanup only after native borrowers have retired.""" if self._context is None: diff --git a/src/python/isaaccapture/ovxr_session_manager/gpu_output.py b/src/python/isaaccapture/ovxr_session_manager/gpu_output.py new file mode 100644 index 0000000000..3e9478fc29 --- /dev/null +++ b/src/python/isaaccapture/ovxr_session_manager/gpu_output.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Public OVRTX mappings and bounded application-owned CUDA snapshots.""" + +from __future__ import annotations + +import ctypes +import threading +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any + + +@lru_cache(maxsize=1) +def _api(): + from ovrtx._src import bindings as b + + root = b.ovrtx_loaded_bin_dir() + if root is None: + raise RuntimeError("OVRTX loaded provider is unavailable") + lib = ctypes.CDLL(str(Path(root) / b.OVRTX_LOADER_LIB_NAME)) + lib.ovrtx_map_render_var_output.argtypes = [ + ctypes.c_void_p, + ctypes.c_uint64, + ctypes.POINTER(b.ovrtx_map_output_description_t), + b.ovrtx_timeout_t, + ctypes.POINTER(b.ovrtx_render_var_output_t), + ] + lib.ovrtx_map_render_var_output.restype = b.ovrtx_result_t + lib.ovrtx_unmap_render_var_output.argtypes = [ + ctypes.c_void_p, + ctypes.c_uint64, + b.ovrtx_cuda_sync_t, + ] + lib.ovrtx_unmap_render_var_output.restype = b.ovrtx_result_t + lib.ovrtx_fetch_results.argtypes = [ + ctypes.c_void_p, + b.ovrtx_step_result_handle_t, + b.ovrtx_timeout_t, + ctypes.POINTER(b.ovrtx_render_product_set_outputs_t), + ] + lib.ovrtx_fetch_results.restype = b.ovrtx_result_t + return b, lib + + +def native_renderer_handle(renderer) -> int: + """Normalize the borrowed renderer object before calling the OVRTX C API.""" + # Accept Renderer objects and raw native pointers at this boundary. + handle = getattr(renderer, "handle", getattr(renderer, "_handle", renderer)) + value = ctypes.cast(handle, ctypes.c_void_p).value + if not value: + raise ValueError("borrowed OVRTX renderer has no native handle") + return value + + +def completed_outputs(renderer: int, step_result: int): + """Borrow completed output metadata until OVXR consumes the bound plan.""" + b, lib = _api() + outputs = b.ovrtx_render_product_set_outputs_t() + result = lib.ovrtx_fetch_results( + renderer, step_result, b.OVRTX_TIMEOUT_INFINITE, ctypes.byref(outputs) + ) + if result.status != b.OVRTX_API_SUCCESS: + raise RuntimeError("OVRTX completed output fetch failed") + return ctypes.pointer(outputs) + + +def _check_cuda(result): + if int(result[0]): + raise RuntimeError(f"GPU output synchronization failed: CUDA {result[0]}") + return result[1] if len(result) > 1 else None + + +def _wait_ready(cuda, sync) -> None: + # Both fields are independently valid in the public OVRTX contract. + if sync.wait_event: + _check_cuda(cuda.cudaEventSynchronize(sync.wait_event)) + if sync.stream: + stream = cuda.cudaStreamLegacy if sync.stream == 1 else sync.stream + _check_cuda(cuda.cudaStreamSynchronize(stream)) + + +class OutputMapCleanupError(RuntimeError): + """The caller must retain this map and stop admission until owner cleanup.""" + + def __init__(self, image, cause, cleanup): + super().__init__(f"Output map failed: {cause}; cleanup failed: {cleanup}") + self.image = image + + +@dataclass(kw_only=True) +class MappedOutput: + """One native output map retained until its owner proves unmap succeeded.""" + + renderer: int + mapped: Any + owner: int + closed: bool = False + busy: bool = False + + def close(self) -> None: + """Owner only, after every consumer has completed its last source read.""" + if threading.get_ident() != self.owner: + raise RuntimeError("Output must be unmapped on its renderer owner") + if self.busy: + raise RuntimeError("Output consumer completion is unproved") + if not self.closed: + b, lib = _api() + result = lib.ovrtx_unmap_render_var_output( + self.renderer, self.mapped.map_handle, b.ovrtx_cuda_sync_t() + ) + if result.status != b.OVRTX_API_SUCCESS: + raise RuntimeError("OVRTX output unmap failed") + self.closed = True + + +@dataclass +class MappedGpuImage(MappedOutput): + width: int + height: int + pointer: int + pitch: int + device: int + is_array: bool = False + + +def map_cuda_array_rgba(renderer: int, handle: int, max_bytes: int) -> MappedGpuImage: + """Retain the native image array, avoiding OVRTX array-to-linear conversion.""" + from cuda.bindings import runtime as cuda + + b, lib = _api() + mapped = b.ovrtx_render_var_output_t() + desc = b.ovrtx_map_output_description_t( + device_type=b.Device.CUDA_ARRAY, sync_stream=None + ) + result = lib.ovrtx_map_render_var_output( + renderer, + handle, + ctypes.byref(desc), + b.OVRTX_TIMEOUT_INFINITE, + ctypes.byref(mapped), + ) + if result.status != b.OVRTX_API_SUCCESS: + raise RuntimeError("OVRTX GPU output map failed") + image = MappedGpuImage( + 0, 0, 0, 0, 0, renderer=renderer, mapped=mapped, owner=threading.get_ident() + ) + try: + if mapped.status != b.EventStatus.COMPLETED or mapped.num_tensors != 1: + raise RuntimeError( + "OVRTX GPU output is incomplete or has unexpected tensors" + ) + dl = mapped.tensors[0].dl.contents + shape = tuple(int(dl.shape[i]) for i in range(dl.ndim)) + if dl.device.device_type.value != 2 or not dl.data: + raise ValueError("mirror GPU output must be CUDA memory") + image.device = int(dl.device.device_id) + _check_cuda(cuda.cudaSetDevice(image.device)) + if ( + dl.dtype.code.value != 3 + or dl.byte_offset + or len(shape) != 3 + or shape[2] != 4 + ): + raise ValueError("mirror GPU output must be an opaque RGBA CUDA array") + info = cuda.cudaArrayGetInfo(int(dl.data)) + _check_cuda(info) + channel, extent = info[1:3] + height, width, _ = shape + if (channel.x, channel.y, channel.z, channel.w, int(channel.f)) != ( + 8, + 8, + 8, + 8, + 1, + ) or (extent.width, extent.height) != (width, height): + raise ValueError( + "camera CUDA array is not unsigned RGBA8 at the declared size" + ) + pitch = width * 4 + image.is_array = True + if width <= 0 or height <= 0 or pitch < width * 4 or pitch * height > max_bytes: + raise ValueError("mirror GPU output exceeds bounded image size") + image.width, image.height, image.pitch = width, height, pitch + image.pointer = int(dl.data) + int(dl.byte_offset) + # Wait for producer completion before reading the mapped image. + _wait_ready(cuda, mapped.cuda_sync) + return image + except BaseException as cause: + try: + image.close() + except Exception as cleanup: + raise OutputMapCleanupError(image, cause, cleanup) from cause + raise diff --git a/src/python/isaaccapture/ovxr_session_manager/interfaces.py b/src/python/isaaccapture/ovxr_session_manager/interfaces.py index 48783edbbe..c23de8e73e 100644 --- a/src/python/isaaccapture/ovxr_session_manager/interfaces.py +++ b/src/python/isaaccapture/ovxr_session_manager/interfaces.py @@ -5,11 +5,12 @@ from __future__ import annotations import math -from abc import abstractmethod +from abc import ABC, abstractmethod from collections.abc import Callable, Hashable from contextlib import AbstractContextManager from dataclasses import dataclass from typing import Any, Literal, Protocol +from .camera_mailbox import FrameOutputs class OvxrScenePublication(Protocol): @@ -93,6 +94,7 @@ class OvxrSessionBinding: renderer: object stage: object ordinal_ledger: StageOrdinalLedger + frame_consumers: tuple[OvxrFrameConsumer, ...] = () prepare_frame_scene: Callable[[], None] | None = None def __post_init__(self) -> None: @@ -104,7 +106,11 @@ def __post_init__(self) -> None: @dataclass class OvxrSessionConfig: - """Configure an application-owned renderer/Stage XR session.""" + """Application-neutral OVXR and CloudXR configuration. + + The supported OVXR package supplies bound-frame outputs. Optional PiP and + output consumers copy those outputs on the sole frame owner before retirement. + """ camera_paths: tuple[str, ...] application_name: str = "isaacteleop-ovxr-session" @@ -141,3 +147,43 @@ def __post_init__(self) -> None: ) if len(self.camera_paths) != 2 or any(not path for path in self.camera_paths): raise ValueError("OVXR stereo requires exactly two non-empty camera paths") + + +class OvxrFrameConsumer(ABC): + """Optional presentation on the OVXR owner thread. + + Outputs are borrowed until finish_frame, before submit/cancel. Composition + layers remain owned by the consumer through submit. on_openxr_ready may be + called repeatedly and must be idempotent. close must tolerate + partial startup and retain resources when retirement cannot be proved. + """ + + @property + def render_product_paths(self) -> tuple[str, ...]: + return () + + def on_session_started(self, binding, config, request_shutdown) -> None: + return None + + def on_openxr_ready(self, openxr, vulkan) -> bool: + """Bind borrowed XR resources; return False while required handles are absent.""" + return True + + @abstractmethod + def on_outputs(self, frame: FrameOutputs) -> None: + """Consume this completed frame without retaining its output descriptors.""" + raise NotImplementedError + + def composition_layers(self) -> tuple[int, ...]: + """Borrow native layer pointers valid through the following submission.""" + return () + + def on_frame_submitted(self) -> None: + return None + + def finish_frame(self) -> None: + return None + + @abstractmethod + def close(self) -> None: + raise NotImplementedError diff --git a/src/python/isaaccapture/ovxr_session_manager/openxr_quad.py b/src/python/isaaccapture/ovxr_session_manager/openxr_quad.py new file mode 100644 index 0000000000..fd922c4168 --- /dev/null +++ b/src/python/isaaccapture/ovxr_session_manager/openxr_quad.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Application-owned camera panels on borrowed OpenXR/Vulkan resources.""" + +from __future__ import annotations + +import ctypes +from pathlib import Path + +from .camera_mailbox import CameraImage + + +class OpenXRCameraPanels: + """One native owner for panel spaces, swapchains and upload completion.""" + + def __init__(self, panels: tuple, *, library_path: str | None = None) -> None: + library = ( + Path(library_path) + if library_path + else Path(__file__).parent / "native/libisaaccapture_camera_panels.so" + ) + self._native = ctypes.CDLL(str(library.resolve(strict=True))) + pointer = ctypes.c_void_p + uint = ctypes.c_uint32 + signatures = { + "create": ([], pointer), + "max_cameras": ([], uint), + "max_image_dimension": ([], uint), + "supports_cuda": ([], ctypes.c_int), + "attach": ([pointer] * 7 + [uint], ctypes.c_int), + "destroy": ([pointer], ctypes.c_int), + "layer": ([pointer, uint], pointer), + "configure": ( + [ + pointer, + uint, + ctypes.c_int, + ctypes.c_int, + ctypes.POINTER(ctypes.c_float), + ctypes.POINTER(ctypes.c_float), + ], + ctypes.c_int, + ), + "publish_rgba": ( + [pointer, uint, pointer, uint, uint, ctypes.c_size_t], + ctypes.c_int, + ), + "publish_cuda_array": ( + [pointer, uint, pointer, uint, uint, ctypes.c_int], + ctypes.c_int, + ), + } + for name, (args, result) in signatures.items(): + function = getattr(self._native, f"isaacteleop_pip_{name}") + function.argtypes, function.restype = args, result + self._panels = tuple(panels) + limit = self._native.isaacteleop_pip_max_cameras() + if len(panels) > limit: + raise ValueError(f"native camera helper supports at most {limit} cameras") + self._max_dimension = self._native.isaacteleop_pip_max_image_dimension() + self.supports_gpu = bool(self._native.isaacteleop_pip_supports_cuda()) + self._visible = [False] * len(panels) + self._published_revision = [0] * len(panels) + self.last_errors = [None] * len(panels) + self._attached = False + self._handle = self._native.isaacteleop_pip_create() + if not self._handle: + raise MemoryError("could not create camera panel owner") + + def attach(self, openxr, vulkan) -> None: + if self._attached: + return + if self.supports_gpu and not (vulkan.external_memory_handle_types & 1): + raise RuntimeError( + "camera GPU upload requires enabled opaque-FD external memory" + ) + result = self._native.isaacteleop_pip_attach( + self._handle, + openxr.xr_instance, + openxr.xr_session, + openxr.xr_get_instance_proc_addr, + vulkan.vk_physical_device, + vulkan.vk_device, + vulkan.vk_queue, + vulkan.queue_family_index, + ) + if result != 0: + # The owner also retains partial attachment until close succeeds. + raise RuntimeError(f"camera session binding failed: {result}") + self._attached = True + + def layer(self, index: int): + return self._native.isaacteleop_pip_layer(self._handle, index) + + def _set_panel(self, index: int, enabled: bool, width: int, height: int) -> None: + panel = self._panels[index] + pose = (ctypes.c_float * 7)(*panel.position, *panel.orientation_xyzw) + size = (ctypes.c_float * 2)( + panel.panel_width_m, panel.panel_width_m * height / width + ) + result = self._native.isaacteleop_pip_configure( + self._handle, index, int(enabled), 1, pose, size + ) + if result != 0: + raise RuntimeError(f"OpenXR camera quad configuration failed: {result}") + self._visible[index] = enabled + + def publish( + self, index: int, image: CameraImage | None, *, invalid: bool = False + ) -> bool: + if invalid: + self.hide(index) + self.last_errors[index] = "completed camera output was invalid" + return False + if image is None or image.revision <= self._published_revision[index]: + return False + try: + if ( + not 0 < image.width <= self._max_dimension + or not 0 < image.height <= self._max_dimension + ): + raise ValueError("camera image exceeds native helper dimensions") + self._set_panel(index, True, image.width, image.height) + if image.gpu is not None: + gpu = image.gpu + if gpu.closed or gpu.busy or not gpu.is_array or not self.supports_gpu: + raise RuntimeError("camera GPU array is unavailable") + gpu.busy = True + result = self._native.isaacteleop_pip_publish_cuda_array( + self._handle, + index, + gpu.pointer, + image.width, + image.height, + gpu.device, + ) + # -7 cannot prove copy completion: retain the borrowed map. + if result != -7: + gpu.busy = False + else: + pixels = ctypes.create_string_buffer(image.rgba) + result = self._native.isaacteleop_pip_publish_rgba( + self._handle, + index, + pixels, + image.width, + image.height, + len(image.rgba), + ) + except Exception: + self.hide(index) + raise + if result != 0: + self.hide(index) + self.last_errors[index] = f"OpenXR camera quad upload failed: {result}" + return False + self._published_revision[index] = image.revision + self.last_errors[index] = None + return True + + def hide(self, index: int) -> None: + if self._visible[index]: + self._set_panel(index, False, 1, 1) + + def close(self) -> None: + if self._handle: + result = self._native.isaacteleop_pip_destroy(self._handle) + if result != 0: + raise RuntimeError( + "camera GPU completion/cleanup is unproved; retain OVXR context" + ) + self._handle = None + self._attached = False diff --git a/src/python/isaaccapture/ovxr_session_manager/ovxr_session.py b/src/python/isaaccapture/ovxr_session_manager/ovxr_session.py index efd567ae44..eb574a4295 100644 --- a/src/python/isaaccapture/ovxr_session_manager/ovxr_session.py +++ b/src/python/isaaccapture/ovxr_session_manager/ovxr_session.py @@ -17,6 +17,7 @@ StageOrdinalLedger as StageOrdinalLedger, OvxrSessionBinding as OvxrSessionBinding, OvxrSessionConfig as OvxrSessionConfig, + OvxrFrameConsumer as OvxrFrameConsumer, OvxrScenePublication as OvxrScenePublication, OvxrResourceOwner as OvxrResourceOwner, ) @@ -340,7 +341,7 @@ def request_shutdown(self) -> None: self._request_worker_shutdown() def wait_for_shutdown(self, timeout_s: float) -> bool: - """Wait interruptibly on the same event used by signals and mirror close.""" + """Wait interruptibly for a shutdown request.""" return self._shutdown_requested.wait(timeout_s) @property diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 56c3b9100b..30980a5985 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -9,3 +9,7 @@ add_subdirectory(live_trackers) add_subdirectory(tracker_metadata) add_subdirectory(log_bridge) add_subdirectory(plugin_manager) + +if(BUILD_OVXR_CAMERA_QUADS) + add_subdirectory(ovxr_pip_layer) +endif() diff --git a/tests/cpp/core/ovxr_pip_layer/CMakeLists.txt b/tests/cpp/core/ovxr_pip_layer/CMakeLists.txt new file mode 100644 index 0000000000..64a2f5adfb --- /dev/null +++ b/tests/cpp/core/ovxr_pip_layer/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Keep white-box lifecycle checks without exposing a plugin's private source +# tree as an interface include path. COPYONLY tracks both authoritative inputs. +set(_pip_whitebox_dir "${CMAKE_CURRENT_BINARY_DIR}/whitebox") +file(MAKE_DIRECTORY "${_pip_whitebox_dir}") +configure_file( + "${PROJECT_SOURCE_DIR}/src/core/ovxr_pip_layer/cpp/pip_layer.cpp" + "${_pip_whitebox_dir}/pip_layer.cpp" COPYONLY) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/pip_layer_contract_test.cpp" + "${_pip_whitebox_dir}/pip_layer_contract_test.cpp" COPYONLY) +add_executable(ovxr_pip_layer_contract_test "${_pip_whitebox_dir}/pip_layer_contract_test.cpp") +target_compile_features(ovxr_pip_layer_contract_test PRIVATE cxx_std_20) +target_include_directories(ovxr_pip_layer_contract_test PRIVATE + $) +target_link_libraries(ovxr_pip_layer_contract_test PRIVATE Vulkan::Vulkan log_bridge::log_bridge_core) +if(CUDAToolkit_FOUND AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_compile_definitions(ovxr_pip_layer_contract_test PRIVATE ISAAC_TELEOP_PIP_CUDA) + target_link_libraries(ovxr_pip_layer_contract_test PRIVATE CUDA::cudart) +endif() +add_test(NAME ovxr_pip_layer_contract_test COMMAND ovxr_pip_layer_contract_test) diff --git a/tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp b/tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp new file mode 100644 index 0000000000..3e397c74f4 --- /dev/null +++ b/tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp @@ -0,0 +1,421 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Single translation unit so the application layer's private state is exercised +// against mocked OpenXR dispatch without a GPU or a second XR session. +// The package is built in Release; its contract checks must still execute. +#undef NDEBUG +#include "pip_layer.cpp" + +#include +#include +#include + +namespace +{ +int acquireCalls = 0; +int waitCalls = 0; +int releaseCalls = 0; +bool firstWaitTimeout = false; +bool releaseFails = false; +bool destroySwapchainFails = false; +bool destroySpaceFails = false; +int createSpaceCalls = 0; +int vkMapCalls = 0; +int vkBufferDestroys = 0; +int vkFenceWaits = 0; +bool vkFenceWaitFails = false; +bool vkIdleFails = false; +std::vector destroyOrder; +std::vector forwardedSwapchains; + +XrResult XRAPI_CALL acquire(XrSwapchain, const XrSwapchainImageAcquireInfo*, uint32_t* index) +{ + ++acquireCalls; + *index = 7; // Forces an exit after the swapchain state transition. + return XR_SUCCESS; +} +XrResult XRAPI_CALL wait(XrSwapchain, const XrSwapchainImageWaitInfo*) +{ + ++waitCalls; + return firstWaitTimeout && waitCalls == 1 ? XR_TIMEOUT_EXPIRED : XR_SUCCESS; +} +XrResult XRAPI_CALL release(XrSwapchain, const XrSwapchainImageReleaseInfo*) +{ + ++releaseCalls; + return releaseFails ? XR_ERROR_RUNTIME_FAILURE : XR_SUCCESS; +} +XrResult XRAPI_CALL destroySwapchain(XrSwapchain) +{ + destroyOrder.push_back(1); + return destroySwapchainFails ? XR_ERROR_RUNTIME_FAILURE : XR_SUCCESS; +} +XrResult XRAPI_CALL unavailableFormats(XrSession, uint32_t, uint32_t*, int64_t*) +{ + return XR_ERROR_RUNTIME_FAILURE; +} +int resizeCreates = 0; +bool imageEnumerationFails = true; +XrResult XRAPI_CALL availableFormats(XrSession, uint32_t capacity, uint32_t* count, int64_t* formats) +{ + *count = 1; + if (capacity) + formats[0] = VK_FORMAT_R8G8B8A8_UNORM; + return XR_SUCCESS; +} +XrResult XRAPI_CALL createResizeSwapchain(XrSession, const XrSwapchainCreateInfo*, XrSwapchain* swapchain) +{ + *swapchain = reinterpret_cast(uintptr_t{ 100 } + ++resizeCreates); + return XR_SUCCESS; +} +XrResult XRAPI_CALL resizeImages(XrSwapchain, uint32_t capacity, uint32_t* count, XrSwapchainImageBaseHeader* images) +{ + *count = 1; + if (capacity) + { + if (imageEnumerationFails) + return XR_ERROR_RUNTIME_FAILURE; + reinterpret_cast(images)[0].image = VkImage(99); + } + return XR_SUCCESS; +} +XrResult XRAPI_CALL destroySpace(XrSpace) +{ + destroyOrder.push_back(2); + return destroySpaceFails ? XR_ERROR_RUNTIME_FAILURE : XR_SUCCESS; +} +XrResult XRAPI_CALL createSpace(XrSession, const XrReferenceSpaceCreateInfo*, XrSpace* space) +{ + if (++createSpaceCalls == 2) + return XR_ERROR_RUNTIME_FAILURE; + *space = reinterpret_cast(uintptr_t{ 30 }); + return XR_SUCCESS; +} +XrResult XRAPI_CALL testGipa(XrInstance, const char* name, PFN_xrVoidFunction* function) +{ + const std::pair functions[] = { + { "xrCreateReferenceSpace", reinterpret_cast(createSpace) }, + { "xrDestroySpace", reinterpret_cast(destroySpace) }, + { "xrEnumerateSwapchainFormats", reinterpret_cast(availableFormats) }, + { "xrCreateSwapchain", reinterpret_cast(createResizeSwapchain) }, + { "xrDestroySwapchain", reinterpret_cast(destroySwapchain) }, + { "xrEnumerateSwapchainImages", reinterpret_cast(resizeImages) }, + { "xrAcquireSwapchainImage", reinterpret_cast(acquire) }, + { "xrWaitSwapchainImage", reinterpret_cast(wait) }, + { "xrReleaseSwapchainImage", reinterpret_cast(release) }, + }; + for (const auto& [entry, address] : functions) + { + if (entry == name) + { + *function = address; + return XR_SUCCESS; + } + } + return XR_ERROR_FUNCTION_UNSUPPORTED; +} +} // namespace + +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkCreateBuffer(VkDevice, + const VkBufferCreateInfo*, + const VkAllocationCallbacks*, + VkBuffer* buffer) +{ + *buffer = VkBuffer(11); + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkGetBufferMemoryRequirements(VkDevice, VkBuffer, VkMemoryRequirements* req) +{ + req->size = 64; + req->memoryTypeBits = 1; +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceMemoryProperties(VkPhysicalDevice, + VkPhysicalDeviceMemoryProperties* props) +{ + props->memoryTypeCount = 1; + props->memoryTypes[0].propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkAllocateMemory(VkDevice, + const VkMemoryAllocateInfo*, + const VkAllocationCallbacks*, + VkDeviceMemory* memory) +{ + *memory = VkDeviceMemory(12); + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkBindBufferMemory(VkDevice, VkBuffer, VkDeviceMemory, VkDeviceSize) +{ + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL +vkMapMemory(VkDevice, VkDeviceMemory, VkDeviceSize, VkDeviceSize, VkMemoryMapFlags, void** data) +{ + static uint32_t bytes = 0; + ++vkMapCalls; + if (vkMapCalls == 1) + { + return VK_ERROR_MEMORY_MAP_FAILED; + } + *data = &bytes; + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkCreateCommandPool(VkDevice, + const VkCommandPoolCreateInfo*, + const VkAllocationCallbacks*, + VkCommandPool* pool) +{ + *pool = VkCommandPool(13); + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkAllocateCommandBuffers(VkDevice, + const VkCommandBufferAllocateInfo*, + VkCommandBuffer* command) +{ + *command = VkCommandBuffer(14); + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkCreateFence(VkDevice, + const VkFenceCreateInfo*, + const VkAllocationCallbacks*, + VkFence* fence) +{ + *fence = VkFence(15); + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkWaitForFences(VkDevice, uint32_t, const VkFence*, VkBool32, uint64_t) +{ + ++vkFenceWaits; + return vkFenceWaitFails ? VK_TIMEOUT : VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkDeviceWaitIdle(VkDevice) +{ + return vkIdleFails ? VK_TIMEOUT : VK_SUCCESS; +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkDestroyFence(VkDevice, VkFence, const VkAllocationCallbacks*) +{ +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkDestroyCommandPool(VkDevice, VkCommandPool, const VkAllocationCallbacks*) +{ +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkUnmapMemory(VkDevice, VkDeviceMemory) +{ +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkDestroyBuffer(VkDevice, VkBuffer, const VkAllocationCallbacks*) +{ + ++vkBufferDestroys; +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkFreeMemory(VkDevice, VkDeviceMemory, const VkAllocationCallbacks*) +{ +} + +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) +namespace +{ +int sameDeviceCopies = 0; +int peerCopies = 0; +cudaError_t copyResult = cudaSuccess; +cudaError_t syncResult = cudaSuccess; +int cudaSyncCalls = 0; +cudaMemcpy3DPeerParms peerCopy{}; +} +extern "C" cudaError_t CUDARTAPI cudaMemcpy2DFromArrayAsync( + void*, size_t pitch, cudaArray_const_t, size_t, size_t, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t) +{ + ++sameDeviceCopies; + assert(pitch == 640 * 4 && width == pitch && height == 480 && kind == cudaMemcpyDeviceToDevice); + return copyResult; +} +extern "C" cudaError_t CUDARTAPI cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* parameters, cudaStream_t) +{ + ++peerCopies; + peerCopy = *parameters; + return copyResult; +} +extern "C" cudaError_t CUDARTAPI cudaStreamSynchronize(cudaStream_t) +{ + ++cudaSyncCalls; + return syncResult; +} +#endif + +int main() +{ +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + CameraSession copyCamera{}; + copyCamera.cudaDevice = 0; + copyCamera.cudaPixels = reinterpret_cast(uintptr_t{ 42 }); + const void* sourceArray = reinterpret_cast(uintptr_t{ 43 }); + assert(copyCameraArray(copyCamera, sourceArray, 640, 480, 0) == cudaSuccess); + assert(sameDeviceCopies == 1 && peerCopies == 0); + assert(copyCameraArray(copyCamera, sourceArray, 640, 480, 1) == cudaSuccess); + assert(sameDeviceCopies == 1 && peerCopies == 1); + assert(peerCopy.srcDevice == 1 && peerCopy.dstDevice == 0); + assert(peerCopy.srcArray == sourceArray && peerCopy.dstPtr.ptr == copyCamera.cudaPixels); + assert(peerCopy.extent.width == 640 && peerCopy.extent.height == 480 && peerCopy.extent.depth == 1); + assert(peerCopy.dstPtr.pitch == 640 * 4 && peerCopy.dstPtr.xsize == 640 * 4); + copyResult = cudaErrorInvalidDevice; + assert(copyCameraArray(copyCamera, sourceArray, 640, 480, 1) == cudaErrorInvalidDevice); + assert(copyCameraArrayAndWait(copyCamera, sourceArray, 640, 480, 1) == -6); + assert(cudaSyncCalls == 1 && !copyCamera.cudaPending && !copyCamera.retired); + syncResult = cudaErrorUnknown; + assert(copyCameraArrayAndWait(copyCamera, sourceArray, 640, 480, 1) == -7); + assert(cudaSyncCalls == 2 && copyCamera.cudaPending && copyCamera.retired); + copyResult = syncResult = cudaSuccess; +#endif + const auto instance = reinterpret_cast(uintptr_t{ 1 }); + const auto session = reinterpret_cast(uintptr_t{ 2 }); + Dispatch dispatch{}; + dispatch.acquire = acquire; + dispatch.wait = wait; + dispatch.release = release; + dispatch.destroySwapchain = destroySwapchain; + dispatch.destroySpace = destroySpace; + dispatch.formats = unavailableFormats; + auto* owner = isaacteleop_pip_create(); + assert(owner); + owner->dispatch = dispatch; + owner->attached = true; + CameraSession camera{}; + camera.instance = instance; + camera.session = session; + camera.view = reinterpret_cast(uintptr_t{ 3 }); + camera.local = reinterpret_cast(uintptr_t{ 4 }); + camera.swapchain = reinterpret_cast(uintptr_t{ 5 }); + camera.width = camera.height = 1; + camera.ready = true; + // Resize cannot forget an old swapchain whose destruction failed. No new + // GPU allocation is needed to exercise this native ownership boundary. + CameraSession resizing = camera; + destroySwapchainFails = true; + const auto previousSwapchain = resizing.swapchain; + assert(!createSwapchain(resizing, dispatch, 2, 2)); + assert(resizing.swapchain == previousSwapchain && resizing.retired && !resizing.ready); + destroySwapchainFails = false; + // A partially created resized swapchain must not advertise the old extent. + // Return to the old size after failed enumeration: it still must recreate. + resizing = camera; + Dispatch resizeDispatch = dispatch; + resizeDispatch.formats = availableFormats; + resizeDispatch.createSwapchain = createResizeSwapchain; + resizeDispatch.images = resizeImages; + assert(!createSwapchain(resizing, resizeDispatch, 2, 2)); + assert(resizing.width == 0 && resizing.height == 0 && !resizing.ready); + imageEnumerationFails = false; + assert(createSwapchain(resizing, resizeDispatch, 1, 1)); + assert(resizeCreates == 2 && resizing.width == 1 && resizing.images[0].image == VkImage(99)); + assert(releaseSessionResources(resizing, resizeDispatch)); + destroyOrder.clear(); + owner->cameras[0] = camera; + + assert(isaacteleop_pip_attach(owner, instance, session, nullptr, {}, {}, {}, 0) == -1); + const auto* layer = isaacteleop_pip_layer(owner, 0); + assert(layer && layer->type == XR_TYPE_COMPOSITION_LAYER_QUAD && layer->layerFlags == 0); + assert(isaacteleop_pip_layer(owner, 1) == nullptr); + assert(isaacteleop_pip_layer(owner, 3) == nullptr); + const auto* quad = reinterpret_cast(layer); + assert(quad->subImage.swapchain == camera.swapchain && quad->eyeVisibility == XR_EYE_VISIBILITY_BOTH); + + // All three descriptors stay alive through the caller's submit, in caller order. + for (size_t i = 1; i < kMaxCameras; ++i) + { + owner->cameras[i] = camera; + owner->cameras[i].swapchain = reinterpret_cast(uintptr_t{ 5 } + i); + owner->cameras[i].view = reinterpret_cast(uintptr_t{ 30 } + i); + owner->cameras[i].local = XR_NULL_HANDLE; + } + const auto* second = isaacteleop_pip_layer(owner, 1); + const auto* third = isaacteleop_pip_layer(owner, 2); + assert(second && third && second != layer && second != third); + assert(quad->subImage.swapchain == camera.swapchain); + owner->panels[1].enabled = false; + assert(isaacteleop_pip_layer(owner, 1) == nullptr); + assert(isaacteleop_pip_layer(owner, 0) == layer && isaacteleop_pip_layer(owner, 2) == third); + + auto& owned = owner->cameras[0]; + uint32_t rgba = 0; + owned.pixels = &rgba; +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + owned.device = VkDevice(24); + owned.cudaDevice = 1; + owned.queue = VkQueue(25); + owned.width = owned.height = 1; + assert(isaacteleop_pip_publish_rgba(owner, 0, &rgba, 1, 1, 4) == -9); + assert(owned.pixels == &rgba); // Reject without overwriting any resource. + owned.cudaDevice = -1; + owned.device = VK_NULL_HANDLE; +#endif + firstWaitTimeout = true; + assert(!upload(owned, dispatch, &rgba)); + assert(acquireCalls == 1 && waitCalls == 1 && releaseCalls == 0); + assert(owned.acquired && !owned.waited && !owned.ready); + assert(!upload(owned, dispatch, &rgba)); + assert(acquireCalls == 1 && waitCalls == 2 && releaseCalls == 1); + assert(!owned.acquired && !owned.ready); + + // Model a fresh session for the independent release-failure cell. + owned.retired = false; + releaseFails = true; + assert(!upload(owned, dispatch, &rgba)); + assert(acquireCalls == 2 && releaseCalls == 2); + assert(owned.retired && !owned.ready); + + CameraSession partial{}; + partial.device = VkDevice(21); + partial.physical = VkPhysicalDevice(22); + partial.width = partial.height = 1; + assert(!createUploadResources(partial)); + assert(!partial.staging && !partial.memory && !partial.pixels); + assert(vkBufferDestroys == 1); + assert(createUploadResources(partial)); + assert(partial.staging && partial.memory && partial.pixels && partial.pool && partial.fence); + partial.fencePending = true; + vkFenceWaitFails = true; + vkIdleFails = true; + assert(!releaseImageResources(partial)); + assert(partial.staging && partial.memory && partial.fence); + assert(vkBufferDestroys == 1); + vkIdleFails = false; + assert(!releaseImageResources(partial)); // Another queue's idle is not our completion contract. + assert(partial.staging && partial.memory && partial.fence); + vkFenceWaitFails = false; + assert(releaseImageResources(partial)); + assert(!partial.staging && !partial.memory && !partial.fence); + assert(vkBufferDestroys == 2 && vkFenceWaits == 3); + + // This is application-owned storage in this test, not Vulkan storage. + owned.pixels = nullptr; + const int waitsBeforeClose = waitCalls; + const int releasesBeforeClose = releaseCalls; + destroySwapchainFails = true; + assert(isaacteleop_pip_destroy(owner) == -1); + assert(owner->attached); // Failed destruction still retains the owner. + destroySwapchainFails = false; + assert(isaacteleop_pip_destroy(owner) == 0); + // No application work remains. Destroying an acquired swapchain does not + // require waiting for a runtime image that may never become available. + assert(waitCalls == waitsBeforeClose && releaseCalls == releasesBeforeClose); + owner = nullptr; + assert((destroyOrder == std::vector{ 1, 1, 2, 2, 1, 2, 1, 2 })); + assert(isaacteleop_pip_layer(owner, 0) == nullptr); + assert(isaacteleop_pip_destroy(owner) == 0); + // Independent owners do not share panel policy. + owner = isaacteleop_pip_create(); + auto* other = isaacteleop_pip_create(); + owner->panels[0].enabled = false; + assert(other->panels[0].enabled); + assert(isaacteleop_pip_max_cameras() == owner->cameras.size()); + assert(isaacteleop_pip_max_image_dimension() > 0); + assert(isaacteleop_pip_destroy(other) == 0); + // Partial reference-space creation cannot lose the only cleanup owner. + destroySpaceFails = true; + assert(isaacteleop_pip_attach(owner, instance, session, testGipa, VkPhysicalDevice(1), VkDevice(2), VkQueue(3), 0) == + -3); + assert(owner->attached); + assert(owner->cameras[0].view); + assert(isaacteleop_pip_destroy(owner) == -1); + assert(owner->cameras[0].view); + destroySpaceFails = false; + assert(isaacteleop_pip_destroy(owner) == 0); + owner = nullptr; + std::puts("pip layer contract: PASS"); +} diff --git a/tests/python/core/ovxr_session_manager/test_camera_mailbox.py b/tests/python/core/ovxr_session_manager/test_camera_mailbox.py new file mode 100644 index 0000000000..7fa006ab92 --- /dev/null +++ b/tests/python/core/ovxr_session_manager/test_camera_mailbox.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pure callback lifetime and output-selection tests for the borrowed camera step.""" + +import ctypes +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from isaaccapture.ovxr_session_manager import camera_mailbox +from isaaccapture.ovxr_session_manager.camera_mailbox import FrameOutputs + +from isaaccapture.ovxr_session_manager.camera_mailbox import CompletedCameraMailbox + + +def test_cpu_unmap_failure_retains_handle_and_blocks_more_maps(monkeypatch): + """Exercise the real CPU copier; failed unmap must not lose its native lease.""" + b = pytest.importorskip("ovrtx._src.bindings") + from isaaccapture.ovxr_session_manager import gpu_output + + gpu_output._api.cache_clear() + + def map_output(_renderer, _handle, _description, _timeout, output): + mapped = ctypes.cast( + output, ctypes.POINTER(b.ovrtx_render_var_output_t) + ).contents + mapped.map_handle = 71 + mapped.num_tensors = 0 # Validation fails after a successful native map. + mapped.status = b.EventStatus.COMPLETED + return SimpleNamespace(status=b.OVRTX_API_SUCCESS) + + native = SimpleNamespace( + ovrtx_map_render_var_output=Mock(side_effect=map_output), + ovrtx_unmap_render_var_output=Mock( + side_effect=[ + SimpleNamespace(status=1), + SimpleNamespace(status=b.OVRTX_API_SUCCESS), + ] + ), + ovrtx_fetch_results=Mock(), + ) + monkeypatch.setattr(b, "ovrtx_loaded_bin_dir", lambda: "/declared/provider") + monkeypatch.setattr(ctypes, "CDLL", lambda _path: native) + try: + mailbox = CompletedCameraMailbox("/Camera", 4) + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(1.0, 1)))) + assert mailbox.snapshot() is None and mailbox.error() is not None + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(2.0, 2)))) + assert native.ovrtx_map_render_var_output.call_count == 1 + mailbox.close() + assert native.ovrtx_unmap_render_var_output.call_count == 2 + assert [ + call.args[1] for call in native.ovrtx_unmap_render_var_output.call_args_list + ] == [71, 71] + mailbox.close() + assert native.ovrtx_unmap_render_var_output.call_count == 2 + finally: + gpu_output._api.cache_clear() + + +def _string(text): + encoded = text.encode() + return SimpleNamespace(ptr=ctypes.c_char_p(encoded), length=len(encoded)) + + +def _frame(time, handle, *, color=True): + vars_ = [ + SimpleNamespace( + source_name=_string("LdrColor" if color else "XRDepth"), + output_handle=handle, + ) + ] + return SimpleNamespace( + frame_end_time=time, render_var_count=len(vars_), output_render_vars=vars_ + ) + + +def _outputs(*frames): + product = SimpleNamespace( + render_product_path=_string("/Camera"), + output_frame_count=len(frames), + output_frames=list(frames), + ) + return SimpleNamespace(contents=SimpleNamespace(output_count=1, outputs=[product])) + + +def test_newest_completed_image_and_zero_output_retention(monkeypatch): + seen = [] + + def copy(_renderer, handle, _limit): + seen.append(handle) + return 1, 1, bytes((handle, 2, 3, 255)) + + monkeypatch.setattr(camera_mailbox, "_copy_host_rgba", copy) + mailbox = CompletedCameraMailbox("/Camera", 4) + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(3.0, 3), _frame(4.0, 4)))) + image = mailbox.snapshot() + assert image is not None + assert (image.sensor_time, image.revision, image.rgba) == ( + 4.0, + 1, + b"\x04\x02\x03\xff", + ) + mailbox.on_outputs(FrameOutputs(1, _outputs())) + assert mailbox.snapshot() is image + assert seen == [4] + + +def test_produced_but_invalid_image_is_omitted_and_error_is_contained(monkeypatch): + monkeypatch.setattr(camera_mailbox, "_copy_host_rgba", lambda *_: (1, 1, b"rgba")) + mailbox = CompletedCameraMailbox("/Camera", 4) + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(1.0, 1)))) + assert mailbox.snapshot() is not None + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(2.0, 2, color=False)))) + assert mailbox.snapshot() is None + assert "without LdrColor" in mailbox.error() + + +def test_mapping_failure_is_contained_without_reusing_stale_pixels(monkeypatch): + def fail(*_args): + raise RuntimeError("map failed") + + monkeypatch.setattr(camera_mailbox, "_copy_host_rgba", fail) + mailbox = CompletedCameraMailbox("/Camera", 4) + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(1.0, 1)))) + assert mailbox.snapshot() is None + assert mailbox.error() == "map failed" + + +def test_gpu_camera_retains_cadence_gaps_and_retires_on_owner(): + import threading + + class FakeMap: + width = height = 1 + + def __init__(self, handle): + self.pointer = handle + self.closed = False + self.owner = threading.get_ident() + + def close(self): + assert threading.get_ident() == self.owner + assert not self.closed + self.closed = True + + def outputs(**handles): + return _outputs(*[_frame(float(h), h) for h in handles.values()]) + + made = [] + + def mapper(_renderer, handle, _limit): + image = FakeMap(handle) + made.append(image) + return image + + mailbox = CompletedCameraMailbox("/Render/Left", 64, gpu_mapper=mapper) + mailbox.product_path = "/Camera" + mailbox.on_outputs(FrameOutputs(1, outputs(Left=1))) + first = mailbox.snapshot() + assert first.gpu is made[0] and first.rgba is None + mailbox.on_outputs(FrameOutputs(1, outputs())) + assert mailbox.snapshot() is first and not made[0].closed + mailbox.on_outputs(FrameOutputs(1, outputs(Left=2))) + assert made[0].closed and mailbox.snapshot().gpu is made[1] + mailbox.close() + assert made[1].closed diff --git a/tests/python/core/ovxr_session_manager/test_openxr_quad.py b/tests/python/core/ovxr_session_manager/test_openxr_quad.py new file mode 100644 index 0000000000..fcd3f5bc77 --- /dev/null +++ b/tests/python/core/ovxr_session_manager/test_openxr_quad.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +from types import SimpleNamespace + +import pytest + +from isaaccapture.ovxr_session_manager.camera_mailbox import CameraImage +from isaaccapture.ovxr_session_manager.camera_panels import CameraPanelConfig +import isaaccapture.ovxr_session_manager.openxr_quad as quad_module + + +class _Function: + def __init__(self): + self.calls = [] + self.result = 0 + + def __call__(self, *args): + self.calls.append(args) + return self.result + + +def _quad(tmp_path, monkeypatch, *, panels=None): + library = tmp_path / "layer.so" + library.touch() + native = SimpleNamespace( + isaacteleop_pip_create=_Function(), + isaacteleop_pip_max_cameras=_Function(), + isaacteleop_pip_max_image_dimension=_Function(), + isaacteleop_pip_attach=_Function(), + isaacteleop_pip_destroy=_Function(), + isaacteleop_pip_layer=_Function(), + isaacteleop_pip_configure=_Function(), + isaacteleop_pip_publish_rgba=_Function(), + isaacteleop_pip_supports_cuda=_Function(), + isaacteleop_pip_publish_cuda_array=_Function(), + ) + monkeypatch.setattr(quad_module.ctypes, "CDLL", lambda _path: native) + native.isaacteleop_pip_create.result = 123 + native.isaacteleop_pip_max_cameras.result = 3 + native.isaacteleop_pip_max_image_dimension.result = 4096 + return quad_module.OpenXRCameraPanels( + panels or (CameraPanelConfig("/Camera/Head"),), library_path=str(library) + ), native + + +def test_default_config_is_accepted_by_native_quad(tmp_path, monkeypatch): + quad, native = _quad(tmp_path, monkeypatch) + assert quad.publish(0, CameraImage(1, 1, bytes(4), 1.0, 1)) + assert native.isaacteleop_pip_configure.calls[-1][3] == 1 + + +def test_head_locked_image_aspect_and_invalid_frame_hides_quad( + tmp_path, + monkeypatch, +): + quad, native = _quad( + tmp_path, + monkeypatch, + panels=(CameraPanelConfig("/Camera/Head", panel_width_m=0.5),), + ) + with pytest.raises(ValueError, match="head_locked"): + CameraPanelConfig("/Camera/Head", space="viewer_start") + image = CameraImage(640, 480, bytes(640 * 480 * 4), 1.0, 1) + assert quad.publish(0, image) + _handle, index, enabled, head_locked, _pose, size = ( + native.isaacteleop_pip_configure.calls[-1] + ) + assert (index, enabled, head_locked) == (0, 1, 1) + assert size[0] == pytest.approx(0.5) + assert size[1] == pytest.approx(0.375) + assert not quad.publish(0, None, invalid=True) + assert native.isaacteleop_pip_configure.calls[-1][2] == 0 + assert quad.last_errors[0] is not None + + +@pytest.mark.parametrize("result", [-1, -2, -3]) +def test_failed_binding_retains_only_its_partial_cleanup_owner( + tmp_path, monkeypatch, result +): + quad, native = _quad(tmp_path, monkeypatch) + native.isaacteleop_pip_attach.result = result + with pytest.raises(RuntimeError, match="binding failed"): + quad.attach( + SimpleNamespace(xr_instance=1, xr_session=2, xr_get_instance_proc_addr=3), + SimpleNamespace( + vk_physical_device=4, vk_device=5, vk_queue=6, queue_family_index=0 + ), + ) + quad.close() + assert len(native.isaacteleop_pip_destroy.calls) == 1 + + +def test_upload_failure_hides_last_valid_panel(tmp_path, monkeypatch): + quad, native = _quad(tmp_path, monkeypatch) + native.isaacteleop_pip_publish_rgba.result = -6 + assert not quad.publish(0, CameraImage(1, 1, b"\0" * 4, 1.0, 1)) + assert native.isaacteleop_pip_configure.calls[-1][2] == 0 + + +def test_pixel_buffer_failure_hides_last_valid_panel(tmp_path, monkeypatch): + quad, native = _quad(tmp_path, monkeypatch) + assert quad.publish(0, CameraImage(1, 1, b"\0" * 4, 1.0, 1)) + monkeypatch.setattr( + quad_module.ctypes, + "create_string_buffer", + lambda _pixels: (_ for _ in ()).throw(MemoryError()), + ) + with pytest.raises(MemoryError): + quad.publish(0, CameraImage(1, 1, b"\0" * 4, 2.0, 2)) + assert native.isaacteleop_pip_configure.calls[-1][2] == 0 + + +def test_explicit_binding_preserves_loader_and_retains_failed_cleanup( + tmp_path, monkeypatch +): + quad, native = _quad(tmp_path, monkeypatch) + monkeypatch.setenv("XR_ENABLE_API_LAYERS", "XR_APILAYER_OTHER") + monkeypatch.delenv("XR_API_LAYER_PATH", raising=False) + xr = SimpleNamespace(xr_instance=1, xr_session=2, xr_get_instance_proc_addr=3) + vk = SimpleNamespace( + vk_physical_device=4, vk_device=5, vk_queue=6, queue_family_index=7 + ) + quad.attach(xr, vk) + assert native.isaacteleop_pip_attach.calls == [(123, 1, 2, 3, 4, 5, 6, 7)] + assert os.environ["XR_ENABLE_API_LAYERS"] == "XR_APILAYER_OTHER" + assert "XR_API_LAYER_PATH" not in os.environ + native.isaacteleop_pip_destroy.result = -1 + with pytest.raises(RuntimeError, match="retain OVXR"): + quad.close() + assert quad._attached + native.isaacteleop_pip_destroy.result = 0 + quad.close() + assert not quad._attached + + +def test_three_panels_keep_independent_revisions(tmp_path, monkeypatch): + owner, native = _quad( + tmp_path, + monkeypatch, + panels=tuple(CameraPanelConfig(f"/Camera/{i}") for i in range(3)), + ) + for index in range(3): + assert owner.publish(index, CameraImage(1, 1, b"abcd", 1.0, 1)) + assert [call[1] for call in native.isaacteleop_pip_publish_rgba.calls] == [0, 1, 2] + owner.publish(1, None, invalid=True) + assert native.isaacteleop_pip_configure.calls[-1][1:3] == (1, 0) + assert not owner.publish(0, CameraImage(1, 1, b"abcd", 1.0, 1)) + assert owner.publish(2, CameraImage(1, 1, b"efgh", 2.0, 2)) + + +def test_old_native_layer_is_rejected_before_upload(tmp_path, monkeypatch): + _quad(tmp_path, monkeypatch) + monkeypatch.setattr(quad_module.ctypes, "CDLL", lambda _path: SimpleNamespace()) + with pytest.raises(AttributeError, match="isaacteleop_pip_create"): + quad_module.OpenXRCameraPanels( + (CameraPanelConfig("/Camera/Head"),), + library_path=str(tmp_path / "layer.so"), + ) + + +@pytest.mark.parametrize("result", [0, -6, -7]) +def test_gpu_quad_completion_controls_source_retirement(tmp_path, monkeypatch, result): + quad, native = _quad(tmp_path, monkeypatch) + quad.supports_gpu = True + gpu = SimpleNamespace( + closed=False, busy=False, is_array=True, pointer=123, device=1 + ) + image = CameraImage(720, 450, None, 1.0, 1, gpu) + native.isaacteleop_pip_publish_cuda_array.result = result + assert quad.publish(0, image) == (result == 0) + assert gpu.busy == (result == -7) + assert len(native.isaacteleop_pip_publish_cuda_array.calls) == 1 + assert native.isaacteleop_pip_publish_rgba.calls == [] diff --git a/tests/python/core/ovxr_session_manager/test_ovxr_session.py b/tests/python/core/ovxr_session_manager/test_ovxr_session.py index 9c79e130e3..f8daca3c0a 100644 --- a/tests/python/core/ovxr_session_manager/test_ovxr_session.py +++ b/tests/python/core/ovxr_session_manager/test_ovxr_session.py @@ -4,12 +4,21 @@ """Focused ownership and lifecycle tests for external OVXR presentation.""" from __future__ import annotations + import threading import time from contextlib import contextmanager +from dataclasses import replace from types import ModuleType, SimpleNamespace from typing import ClassVar + import pytest + +from isaaccapture.ovxr_session_manager import camera_panels +from isaaccapture.ovxr_session_manager.camera_panels import ( + CameraPanelConfig, + CameraPanelConsumer, +) import isaaccapture.ovxr_session_manager.ovxr_session as ovxr_session from isaaccapture.ovxr_session_manager import frame_pump, runtime_owner @@ -183,6 +192,7 @@ def __init__(self): self.events = [] self.publication_closed = False self.owner_lane_entries = [] + self.frame_consumers = () def offer_render_owner_transfer(self): self.offer_thread = threading.get_native_id() @@ -193,7 +203,10 @@ def attach_on_render_thread(self, _capability): self.events.append("attach") _FakeContext.ordinal_ledger = self.ordinal_ledger return ovxr_session.OvxrSessionBinding( - self.renderer, self.stage, self.ordinal_ledger + self.renderer, + self.stage, + self.ordinal_ledger, + frame_consumers=self.frame_consumers, ) @contextmanager @@ -239,6 +252,11 @@ def abort(self): self.events.append("abort") +def _session_with_consumers(host, config, *, frame_consumers): + host.frame_consumers = frame_consumers + return ovxr_session.OvxrSession(host, config) + + @pytest.fixture(autouse=True) def _runtime_stubs(monkeypatch): _FakeContext.operations = [] @@ -255,6 +273,17 @@ def _runtime_stubs(monkeypatch): _FakeContext.last_desc = None _FakeContext.selected_products = None monkeypatch.setattr(frame_pump, "_import_ovxr", _fake_ovxr) + monkeypatch.setattr( + camera_panels, + "OpenXRCameraPanels", + lambda panels: SimpleNamespace( + supports_gpu=False, + attach=lambda _xr, _vk: None, + layer=lambda _index: None, + publish=lambda _index, _image, **kw: True, + close=lambda: None, + ), + ) oxr = ModuleType("isaaccapture.oxr") oxr.OpenXRSessionHandles = lambda *values: values monkeypatch.setitem(__import__("sys").modules, "isaaccapture.oxr", oxr) @@ -634,6 +663,60 @@ def test_bind_timeout_reconciles_a_sealed_ordinal_then_reserves_a_fresh_one(): assert (first, True, True) in host.ordinal_ledger.external_resolutions +def test_frame_scene_callback_seals_before_the_next_bind_ordinal(): + class Host(_FakeHost): + def attach_on_render_thread(self, capability): + binding = super().attach_on_render_thread(capability) + + def prepare(): + ordinal = self.ordinal_ledger.reserve() + self.ordinal_ledger.publish(ordinal) + self.events.append(("frame-scene", ordinal)) + + return replace(binding, prepare_frame_scene=prepare) + + host = Host() + owner = ovxr_session.OvxrSession(host, _config()) + owner.start() + owner.request_render("physics") + _wait_for_operation("submit") + owner.close() + prepared = [ + e[1] for e in host.events if isinstance(e, tuple) and e[0] == "frame-scene" + ] + assert prepared + assert _FakeContext.bind_ordinals == [ordinal + 1 for ordinal in prepared] + + +def test_shutdown_wait_wakes_on_the_existing_lifecycle_event(): + owner = ovxr_session.OvxrSession(_FakeHost(), _config()) + woke = threading.Event() + thread = threading.Thread(target=lambda: (owner.wait_for_shutdown(30), woke.set())) + thread.start() + try: + owner.request_shutdown() + assert woke.wait(1) + assert owner.wait_for_shutdown(30) + finally: + thread.join(timeout=1) + assert not thread.is_alive() + + +def test_unsupported_provider_rejects_before_host_transfer(monkeypatch): + runtime = _fake_ovxr() + runtime.Context = type( + "OldContext", (_FakeContext,), {"set_frame_render_products": None} + ) + monkeypatch.setattr(frame_pump, "_import_ovxr", lambda: runtime) + host = _FakeHost() + owner = ovxr_session.OvxrSession(host, _config()) + with pytest.raises( + RuntimeError, match="required Context methods: set_frame_render_products" + ): + owner.start() + assert host.offer_thread is None + + def test_missing_core_phase_method_still_rejects_before_host_transfer(monkeypatch): runtime = _fake_ovxr() runtime.Context = type("BrokenContext", (_FakeContext,), {"cancel_frame": None}) @@ -816,6 +899,45 @@ def wait(self, timeout=None): assert _FakeContext.operations.count(("destroy", host.attach_thread)) == 1 +def test_close_times_out_without_blocking_on_startup_teardown_lock(monkeypatch): + host = _FakeHost() + owner = ovxr_session.OvxrSession(host, _config()) + entered = threading.Event() + resume = threading.Event() + destroy = _FakeContext.destroy + failure = RuntimeError("startup failed") + + def blocked_destroy(context, *, stage_ordinal): + entered.set() + assert resume.wait(timeout=2.0) + destroy(context, stage_ordinal=stage_ordinal) + + def fail_start(): + raise failure + + monkeypatch.setattr(_FakeContext, "destroy", blocked_destroy) + monkeypatch.setattr(owner._pump, "bootstrap", fail_start) + monkeypatch.setattr(ovxr_session, "_WORKER_SHUTDOWN_TIMEOUT_SECONDS", 0.01) + try: + with pytest.raises(RuntimeError) as caught: + owner.start() + assert caught.value is failure + assert entered.wait(timeout=1.0) + closing, errors = _call_in_thread(owner.close) + closing.join(timeout=0.5) + assert not closing.is_alive() + assert len(errors) == 1 + assert isinstance(errors[0], TimeoutError) + assert owner._pump._context is not None + assert not owner.retirement_complete + finally: + resume.set() + owner._worker.join(timeout=1.0) + with pytest.raises(RuntimeError) as caught: + owner.close() + assert caught.value is failure + + @pytest.mark.parametrize("abort_error_type", [None, RuntimeError, KeyboardInterrupt]) def test_worker_thread_start_failure_rolls_back_host_and_preserves_first_error( monkeypatch, abort_error_type, caplog @@ -874,6 +996,82 @@ def test_binding_rejects_an_incomplete_borrowed_tuple(): ovxr_session.OvxrSessionBinding(object(), object(), None) +def test_camera_panel_registers_one_borrowed_product_without_a_second_frame_source(): + host = _FakeHost() + path = "/Camera/Head" + config = _config() + panel = CameraPanelConfig( + path, + max_image_bytes=720 * 450 * 4, + panel_width_m=0.5, + position=(0.1, -0.2, -0.8), + space="head_locked", + ) + host.frame_consumers = (CameraPanelConsumer((panel,)),) + owner = ovxr_session.OvxrSession(host, config) + + owner.start() + desc = _FakeContext.last_desc + owner.close() + + assert _FakeContext.selected_products == (path,) + assert not hasattr(desc, "enable_raster_layer") + assert not hasattr(desc, "auxiliary_render_product") + assert [name for name, _thread in _FakeContext.operations].count("submit") == 0 + + +def test_auxiliary_panel_space_remains_application_policy(): + host = _FakeHost() + path = "/Camera/Head" + config = _config() + panel = CameraPanelConfig( + path, + max_image_bytes=720 * 450 * 4, + space="head_locked", + ) + host.frame_consumers = (CameraPanelConsumer((panel,)),) + owner = ovxr_session.OvxrSession(host, config) + + owner.start() + owner.close() + + assert panel.space == "head_locked" + assert _FakeContext.selected_products == (path,) + assert not hasattr(_FakeContext.last_desc, "auxiliary_panel_space") + + +@pytest.mark.parametrize("space", ["stage", "viewer_start", "world"]) +def test_camera_panel_rejects_unsupported_panel_space(space): + with pytest.raises(ValueError, match="space"): + CameraPanelConfig("/Camera/Head", max_image_bytes=720 * 450 * 4, space=space) + + +def test_camera_panel_has_bounded_public_defaults(): + config = CameraPanelConfig( + "/Camera/Head", + ) + + assert config.max_image_bytes == 64 * 1024 * 1024 + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"max_image_bytes": 0}, "max_image_bytes"), + ({"panel_width_m": 0.0}, "panel_width_m"), + ({"orientation_xyzw": (0.0, 0.0, 0.0, 0.0)}, "orientation"), + ], +) +def test_camera_panel_rejects_invalid_policy(kwargs, message): + with pytest.raises(ValueError, match=message): + CameraPanelConfig("/Camera/Head", **kwargs) + + +def test_camera_panel_requires_an_absolute_host_product(): + with pytest.raises(ValueError, match="absolute"): + CameraPanelConfig("relative") + + def test_cloudxr_and_ovxr_are_created_and_destroyed_on_the_owner_thread(monkeypatch): host = _FakeHost() launcher_events = [] @@ -907,6 +1105,228 @@ def launcher_factory(_path, *, accept_eula=False): ] +def test_injected_consumers_follow_frame_and_retirement_order(monkeypatch): + events = [] + threads = [] + outputs = SimpleNamespace(contents=SimpleNamespace(output_count=0, outputs=[])) + caller_thread = threading.get_native_id() + host = _FakeHost() + + class Context(_FakeContext): + def begin_frame(self): + result, plan = super().begin_frame() + plan.step_result = 1 + return result, plan + + class Consumer(ovxr_session.OvxrFrameConsumer): + def __init__(self, name): + self.name = name + + def record(self, event): + events.append((event, self.name)) + threads.append(threading.get_native_id()) + + def on_session_started(self, _binding, _config, request_shutdown): + assert any(name == "create" for name, _ in _FakeContext.operations) + self.request_shutdown = request_shutdown + self.record("start") + + def on_outputs(self, frame): + assert frame.latest_color("/missing") is None + self.record("outputs") + + def finish_frame(self): + self.record("finish") + + def on_frame_submitted(self): + assert any(name == "submit" for name, _ in _FakeContext.operations) + self.record("submitted") + self.request_shutdown() + + def close(self): + assert not any(name == "destroy" for name, _ in _FakeContext.operations) + self.record("close") + + api = _fake_ovxr() + api.Context = Context + monkeypatch.setattr(frame_pump, "_import_ovxr", lambda: api) + monkeypatch.setattr(frame_pump, "native_renderer_handle", lambda _renderer: 7) + monkeypatch.setattr(frame_pump, "completed_outputs", lambda *_args: outputs) + owner = _session_with_consumers( + host, _config(), frame_consumers=(Consumer("first"), Consumer("second")) + ) + with owner: + owner.start() + owner.request_render("step-1") + _wait_for_operation("submit") + assert events == [ + ("start", "first"), + ("start", "second"), + ("outputs", "first"), + ("outputs", "second"), + ("finish", "first"), + ("finish", "second"), + ("submitted", "first"), + ("submitted", "second"), + ("close", "second"), + ("close", "first"), + ] + assert caller_thread != owner.render_worker_native_thread_id + assert set(threads) == {owner.render_worker_native_thread_id} + assert not any(name == "cancel" for name, _ in _FakeContext.operations) + + +def test_consumer_start_failure_retires_borrowed_owners(): + host = _FakeHost() + failure = RuntimeError("consumer start failed") + + class Consumer(ovxr_session.OvxrFrameConsumer): + def on_session_started(self, *_args): + raise failure + + def on_outputs(self, frame): + pass + + def close(self): + host.events.append("consumer-close") + + owner = _session_with_consumers(host, _config(), frame_consumers=(Consumer(),)) + with pytest.raises(RuntimeError) as caught: + owner.start() + assert caught.value is failure + with pytest.raises(RuntimeError): + owner.close() + assert "consumer-close" in host.events + assert "destroy" in [name for name, _ in _FakeContext.operations] + + +def test_quad_detach_failure_reaches_close_and_retains_borrowed_owners(monkeypatch): + host = _FakeHost() + panel = CameraPanelConsumer((CameraPanelConfig("/Render/Head"),)) + owner = _session_with_consumers(host, _config(), frame_consumers=(panel,)) + owner.start() + context = owner._pump._context + quad = panel._native + launcher = _ControlledLauncher() + owner._pump._runtime._launcher = launcher + failure = RuntimeError("native quad detach failed") + + def fail_detach(): + raise failure + + quad.close = fail_detach + with pytest.raises(RuntimeError, match="native quad detach failed") as caught: + owner.close() + + assert caught.value is failure + assert not owner._worker.is_alive() + assert owner._shutdown_requested.is_set() + assert owner._phase is ovxr_session._Phase.CLOSED + assert not owner.retirement_complete + with pytest.raises(RuntimeError) as repeated: + owner.close() + assert repeated.value is failure + assert owner._pump._context is context + assert panel._native is quad + assert owner._pump._runtime._launcher is launcher + assert launcher.events == [] + assert "destroy" not in [name for name, _thread in _FakeContext.operations] + assert not any( + event[0] in {"host-close", "environment-close"} + for event in host.events + if isinstance(event, tuple) + ) + + +def test_shutdown_timeout_preserves_runtime_until_frame_worker_retires(monkeypatch): + host = _FakeHost() + frame_wait_entered = threading.Event() + release_frame = threading.Event() + launcher = _ControlledLauncher() + + class BlockingContext(_FakeContext): + def begin_frame(self): + frame_wait_entered.set() + assert release_frame.wait(timeout=2.0) + return super().begin_frame() + + fake_ovxr = _fake_ovxr() + fake_ovxr.Context = BlockingContext + monkeypatch.setattr(frame_pump, "_import_ovxr", lambda: fake_ovxr) + + def launcher_factory(*args, **kwargs): + return launcher + + monkeypatch.setattr(ovxr_session, "_WORKER_SHUTDOWN_TIMEOUT_SECONDS", 0.01) + config = _config() + config.cloudxr_env_file = "/cloudxr.env" + config.auto_launch_cloudxr = True + owner = ovxr_session.OvxrSession( + host, config, runtime_launcher_factory=launcher_factory + ) + owner.start() + assert owner.request_render("step-1") == 10 + assert frame_wait_entered.wait(timeout=1.0) + context = owner._pump._context + failure = RuntimeError("request exit failed") + _FakeContext.request_exit_error = failure + try: + with pytest.raises( + TimeoutError, match="OVXR session worker did not stop" + ) as error: + owner.close() + assert error.value.__cause__ is failure + assert owner._pump._context is context + assert owner._worker.is_alive() + assert not owner.retirement_complete + assert launcher.events == [] + assert "destroy" not in [name for name, _thread in _FakeContext.operations] + finally: + release_frame.set() + owner._worker.join(timeout=1.0) + assert not owner._worker.is_alive() + with pytest.raises(RuntimeError) as error: + owner.close() + assert error.value is failure + assert [name for name, _thread in launcher.events] == ["stop-start", "stop-end"] + + +def test_cloudxr_stop_can_finish_without_runtime_interruption(monkeypatch, caplog): + host = _FakeHost() + launcher = _ControlledLauncher(block_stop=True) + + def launcher_factory(_path, *, accept_eula=False): + return launcher + + config = _config() + config.cloudxr_env_file = "/cloudxr.env" + config.auto_launch_cloudxr = True + owner = ovxr_session.OvxrSession( + host, config, runtime_launcher_factory=launcher_factory + ) + owner.start() + + close_thread, close_errors = _call_in_thread(owner.close) + try: + assert launcher.stop_started.wait(timeout=1.0) + assert owner._pump._runtime._launcher is launcher + time.sleep(0.03) + assert close_thread.is_alive() + assert "interrupt" not in [name for name, _thread in launcher.events] + assert "interrupting the owned OpenXR runtime" not in caplog.text + finally: + launcher.allow_stop.set() + close_thread.join(timeout=1.0) + + assert not close_thread.is_alive() + assert close_errors == [] + assert owner._pump._runtime._launcher is None + assert launcher.events == [ + ("stop-start", owner.render_worker_native_thread_id), + ("stop-end", owner.render_worker_native_thread_id), + ] + + def test_destroy_failure_retains_borrowed_providers_and_forbids_retry(monkeypatch): host = _FakeHost() launcher = _ControlledLauncher() @@ -1062,21 +1482,57 @@ def launcher_factory(_path, *, accept_eula=False): ] -@pytest.mark.parametrize("failure_type", [RuntimeError, _WorkerAbort]) -def test_render_failure_waits_for_borrowers_and_preserves_first_error( - monkeypatch, failure_type -): +def test_partial_startup_failure_retires_context_before_cloudxr_stop(monkeypatch): host = _FakeHost() - owner = ovxr_session.OvxrSession(host, _config()) - owner.start() - failure = failure_type("bind failed") - _FakeContext.bind_error = failure - failed = threading.Event() - original = owner._record_worker_failure - - def record(error): - original(error) - failed.set() + launcher = _ControlledLauncher(block_stop=True) + + def launcher_factory(_path, *, accept_eula=False): + return launcher + + _FakeContext.create_error = RuntimeError("create failed") + config = _config() + config.cloudxr_env_file = "/cloudxr.env" + config.auto_launch_cloudxr = True + owner = ovxr_session.OvxrSession( + host, config, runtime_launcher_factory=launcher_factory + ) + + start_thread, start_errors = _call_in_thread(owner.start) + assert launcher.stop_started.wait(timeout=1.0) + close_thread, close_errors = _call_in_thread(owner.close) + try: + time.sleep(0.03) + assert "interrupt" not in [name for name, _thread in launcher.events] + finally: + launcher.allow_stop.set() + start_thread.join(timeout=1.0) + close_thread.join(timeout=1.0) + + assert not start_thread.is_alive() + assert not close_thread.is_alive() + assert [str(error) for error in start_errors] == ["create failed"] + assert [str(error) for error in close_errors] == ["create failed"] + assert launcher.events == [ + ("stop-start", owner.render_worker_native_thread_id), + ("stop-end", owner.render_worker_native_thread_id), + ] + + +@pytest.mark.parametrize("failure_type", [RuntimeError, _WorkerAbort]) +def test_render_failure_waits_for_borrowers_and_preserves_first_error( + monkeypatch, failure_type +): + host = _FakeHost() + owner = ovxr_session.OvxrSession(host, _config()) + owner.start() + failure = failure_type("bind failed") + _FakeContext.bind_error = failure + failed = threading.Event() + original = owner._record_worker_failure + + def record(error): + original(error) + failed.set() monkeypatch.setattr(owner, "_record_worker_failure", record) @@ -1157,6 +1613,8 @@ class PacedContext(_FakeContext): def begin_frame(self): self.frames += 1 + # The runtime owns the next-frame wait, including while physics is + # paused. Block in that boundary to check shutdown as well as pace. if self.frames == 4: fourth_frame.set() assert release_frame.wait(1.0) @@ -1259,6 +1717,8 @@ def get_openxr_handles(self): if publish_scene: owner.request_render("paused-scene") _wait_for_operation("submit") + # Scene rendering may proceed before DeviceIO handles exist, exactly + # as for an initial failed peer followed by a later headset connection. peer_ready.set() with owner._worker_condition: ready = owner._worker_condition.wait_for( @@ -1283,72 +1743,278 @@ def test_config_rejects_unknown_session_mode(): _config(session_mode="automatic") -@pytest.mark.parametrize( - "scale", [float("nan"), float("inf"), float("-inf"), 0.0, -1.0] -) -def test_world_scale_rejects_nonfinite_or_nonpositive_values(scale): - with pytest.raises(ValueError): - _config(meters_per_unit=scale) +def test_three_products_keep_selection_order_and_one_native_owner(monkeypatch): + events = [] + paths = ("/Render/Head", "/Render/Left", "/Render/Right") + panels = tuple( + CameraPanelConfig(path, label=label) + for path, label in zip(paths, ("Head", "Left", "Right")) + ) + def create(configs): + assert configs == panels + return SimpleNamespace( + supports_gpu=False, + attach=lambda *_a: events.append("attach"), + close=lambda: events.append("close"), + ) -@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) -def test_cancel_failure_does_not_skip_context_and_service_cleanup( - monkeypatch, error_type -): + monkeypatch.setattr(camera_panels, "OpenXRCameraPanels", create) + consumer = CameraPanelConsumer(panels) + owner = _session_with_consumers(_FakeHost(), _config(), frame_consumers=(consumer,)) + owner.start() + assert _FakeContext.selected_products == paths + assert tuple(box.product_path for box in consumer._mailboxes) == paths + owner.close() + assert events[-1] == "close" + + +def test_camera_products_reject_duplicate_identity(): + with pytest.raises(ValueError, match="unique"): + CameraPanelConsumer((CameraPanelConfig("/A"), CameraPanelConfig("/A"))) + + +@pytest.mark.parametrize("release_fails", [False, True]) +def test_bound_output_reads_finish_before_submit_or_cancel(monkeypatch, release_fails): host = _FakeHost() - launcher = _ControlledLauncher() + import ctypes - def launcher_factory(*args, **kwargs): - return launcher + host.renderer = SimpleNamespace(_handle=ctypes.c_void_p(123)) + owner = ovxr_session.OvxrSession(host, _config()) + owner._pump._binding = host.attach_on_render_thread(object()) + plan = SimpleNamespace(step_result=77) + events = [] + + def bind(_plan, _time, *, stage_ordinal): + host.ordinal_ledger.seal_external(stage_ordinal) + events.append("bind") + return 0, True + + def release(): + events.append("unmap") + if release_fails: + raise RuntimeError("source read completion unproved") + + def submit(_plan, *, additional_layers): + assert events == ["bind", "observe", "copy", "unmap"] + assert additional_layers == [123] + events.append("submit") + return 0, SimpleNamespace(submitted=True) - config = _config() - config.cloudxr_env_file = "/cloudxr.env" - config.auto_launch_cloudxr = True - owner = ovxr_session.OvxrSession( - host, config, runtime_launcher_factory=launcher_factory + owner._pump._context = SimpleNamespace( + begin_frame=lambda: (0, plan), + bind_frame=bind, + submit_frame=submit, + cancel_frame=lambda _plan: events.append("cancel"), ) - failure = error_type("cancel failed") - def cancel(_self, _plan): + class Consumer(ovxr_session.OvxrFrameConsumer): + def on_outputs(self, frame): + events.extend(["observe", "copy"]) + + def composition_layers(self): + return (123,) + + def finish_frame(self): + release() + + def close(self): + release() + + owner._pump._frame_consumers = (Consumer(),) + + def fetch(renderer, result): + assert renderer == 123 and result == 77 + return object() + + monkeypatch.setattr(frame_pump, "completed_outputs", fetch) + if release_fails: + with pytest.raises(RuntimeError, match="completion unproved"): + owner._pump.render(9) + assert "submit" not in events and "cancel" not in events + assert owner._pump._pending_plan is plan + else: + owner._pump.render(9) + assert events[-1] == "submit" and owner._pump._pending_plan is None + + +def test_no_render_plan_does_not_read_outputs_or_append_layers(monkeypatch): + host = _FakeHost() + owner = ovxr_session.OvxrSession(host, _config()) + owner._pump._binding = host.attach_on_render_thread(object()) + plan = SimpleNamespace(step_result=0) + submitted = [] + + def bind(_plan, _time, *, stage_ordinal): + return 0, False + + def unexpected(*args): + pytest.fail("no-render frame attempted an output read or quad operation") + + owner._pump._context = SimpleNamespace( + begin_frame=lambda: (0, plan), + bind_frame=bind, + submit_frame=lambda _plan, **kw: ( + submitted.append(kw) or 0, + SimpleNamespace(submitted=False), + ), + cancel_frame=unexpected, + ) + + class Consumer(ovxr_session.OvxrFrameConsumer): + on_outputs = unexpected + composition_layers = unexpected + + def close(self): + pass + + owner._pump._frame_consumers = (Consumer(),) + monkeypatch.setattr(frame_pump, "completed_outputs", unexpected) + owner._pump.render(9) + assert submitted == [{"additional_layers": []}] + + +@pytest.mark.parametrize("cleanup_boundary", ["outputs", "cancel"]) +def test_frame_cleanup_preserves_the_original_bind_error(cleanup_boundary): + host = _FakeHost() + owner = ovxr_session.OvxrSession(host, _config()) + owner._pump._binding = host.attach_on_render_thread(object()) + + failure = RuntimeError("original bind failure") + plan = SimpleNamespace(step_result=0) + + def bind(*_args, **_kwargs): raise failure - monkeypatch.setattr(_FakeContext, "cancel_frame", cancel) - owner.start() - owner._pump._pending_plan = SimpleNamespace(step_result=0) - with pytest.raises(error_type) as caught: - owner.close() - assert caught.value is failure - assert "destroy" in [name for name, _thread in _FakeContext.operations] - assert owner._pump._context is None - assert owner._pump._pending_plan is None - assert [name for name, _thread in launcher.events] == ["stop-start", "stop-end"] - assert [event[0] for event in host.events if isinstance(event, tuple)][-2:] == [ - "host-close", - "environment-close", + def cleanup(): + raise RuntimeError("cleanup failed") + + owner._pump._context = SimpleNamespace( + begin_frame=lambda: (0, plan), + bind_frame=bind, + cancel_frame=lambda _plan: cleanup() if cleanup_boundary == "cancel" else None, + ) + if cleanup_boundary == "outputs": + owner._pump._finish_frame_outputs = cleanup + with pytest.raises(RuntimeError, match="original bind failure") as error: + owner._pump.render(9) + assert error.value is failure + assert "cleanup failed" in " ".join(error.value.__notes__) + assert owner._pump._pending_plan is plan + + +def test_consumer_retirement_failure_preserves_earlier_publication_error(): + host = _FakeHost() + failure = RuntimeError("publication close failed") + + def close_publication(): + raise failure + + def close_map(): + raise RuntimeError("map retirement failed") + + host.close_publication = close_publication + owner = ovxr_session.OvxrSession(host, _config()) + owner._pump._frame_consumers = (SimpleNamespace(close=close_map),) + with pytest.raises(RuntimeError, match="publication close failed") as error: + owner._pump.close() + assert error.value is failure + assert "map retirement failed" in " ".join(error.value.__notes__) + assert not host.events # Borrowed objects must not retire past a live map. + + +@pytest.mark.parametrize("hide_fails", [False, True]) +@pytest.mark.parametrize("failure_site", ["fetch", "publish"]) +def test_failed_output_hides_cached_quad_until_successful_refresh( + monkeypatch, hide_fails, failure_site +): + import ctypes + + host = _FakeHost() + host.renderer = SimpleNamespace(_handle=ctypes.c_void_p(123)) + owner = ovxr_session.OvxrSession(host, _config()) + owner._pump._binding = host.attach_on_render_thread(object()) + events = [] + + def bind(_plan, _time, *, stage_ordinal): + host.ordinal_ledger.seal_external(stage_ordinal) + return 0, True + + failing = [True] + + def fetch(*_args): + if failing[0] and failure_site == "fetch": + raise RuntimeError("fetch failed") + return object() + + owner._pump._context = SimpleNamespace( + begin_frame=lambda: (0, SimpleNamespace(step_result=77)), + bind_frame=bind, + submit_frame=lambda *_a, **kw: ( + events.append(kw["additional_layers"]) or 0, + SimpleNamespace(submitted=True), + ), + ) + visible = [True] + + def hide(): + if hide_fails: + raise RuntimeError("optional hide failed") + visible[0] = False + + def publish(*_args, **_kwargs): + if failing[0] and failure_site == "publish": + raise RuntimeError("publish failed") + visible[0] = True + + consumer = CameraPanelConsumer((CameraPanelConfig("/Camera/Head"),)) + consumer._native = SimpleNamespace( + hide=lambda _index: hide(), + publish=publish, + layer=lambda _index: 123 if visible[0] else 0, + ) + consumer._mailboxes = [ + SimpleNamespace( + on_outputs=lambda *_a, **_kw: None, + close=lambda: None, + snapshot=object, + error=lambda: None, + ) ] - assert not owner.retirement_complete # Cleanup does not erase the original error. + owner._pump._frame_consumers = (consumer,) + monkeypatch.setattr(frame_pump, "completed_outputs", fetch) + owner._pump.render(9) + assert events == [[]] + failing[0] = False + owner._pump.render(9) + assert events == [[], [123]] -def test_shutdown_timeout_preserves_runtime_until_frame_worker_retires(monkeypatch): +@pytest.mark.parametrize( + "scale", [float("nan"), float("inf"), float("-inf"), 0.0, -1.0] +) +def test_world_scale_rejects_nonfinite_or_nonpositive_values(scale): + with pytest.raises(ValueError): + _config(meters_per_unit=scale) + + +def test_slow_context_retirement_keeps_owned_runtime_alive(monkeypatch): host = _FakeHost() - frame_wait_entered = threading.Event() - release_frame = threading.Event() launcher = _ControlledLauncher() + entered = threading.Event() + resume = threading.Event() + destroy = _FakeContext.destroy - class BlockingContext(_FakeContext): - def begin_frame(self): - frame_wait_entered.set() - assert release_frame.wait(timeout=2.0) - return super().begin_frame() + def slow_destroy(context, *, stage_ordinal): + entered.set() + assert resume.wait(timeout=3.0) + destroy(context, stage_ordinal=stage_ordinal) - fake_ovxr = _fake_ovxr() - fake_ovxr.Context = BlockingContext - monkeypatch.setattr(frame_pump, "_import_ovxr", lambda: fake_ovxr) + monkeypatch.setattr(_FakeContext, "destroy", slow_destroy) def launcher_factory(*args, **kwargs): return launcher - monkeypatch.setattr(ovxr_session, "_WORKER_SHUTDOWN_TIMEOUT_SECONDS", 0.01) config = _config() config.cloudxr_env_file = "/cloudxr.env" config.auto_launch_cloudxr = True @@ -1356,29 +2022,19 @@ def launcher_factory(*args, **kwargs): host, config, runtime_launcher_factory=launcher_factory ) owner.start() - assert owner.request_render("step-1") == 10 - assert frame_wait_entered.wait(timeout=1.0) - context = owner._pump._context - failure = RuntimeError("request exit failed") - _FakeContext.request_exit_error = failure + closing, errors = _call_in_thread(owner.close) try: - with pytest.raises( - TimeoutError, match="OVXR session worker did not stop" - ) as error: - owner.close() - assert error.value.__cause__ is failure - assert owner._pump._context is context - assert owner._worker.is_alive() - assert not owner.retirement_complete + assert entered.wait(timeout=1.0) + # Slow native retirement must keep its runtime alive until completion. + time.sleep(1.1) + assert closing.is_alive() assert launcher.events == [] - assert "destroy" not in [name for name, _thread in _FakeContext.operations] finally: - release_frame.set() - owner._worker.join(timeout=1.0) - assert not owner._worker.is_alive() - with pytest.raises(RuntimeError) as error: - owner.close() - assert error.value is failure + resume.set() + closing.join(timeout=2.0) + assert not closing.is_alive() + assert errors == [] + assert owner.retirement_complete assert [name for name, _thread in launcher.events] == ["stop-start", "stop-end"] @@ -1421,40 +2077,42 @@ def __exit__(self, *_args): assert context is not None -def test_partial_startup_failure_retires_context_before_cloudxr_stop(monkeypatch): +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +def test_cancel_failure_does_not_skip_context_and_service_cleanup( + monkeypatch, error_type +): host = _FakeHost() - launcher = _ControlledLauncher(block_stop=True) + launcher = _ControlledLauncher() - def launcher_factory(_path, *, accept_eula=False): + def launcher_factory(*args, **kwargs): return launcher - _FakeContext.create_error = RuntimeError("create failed") config = _config() config.cloudxr_env_file = "/cloudxr.env" config.auto_launch_cloudxr = True owner = ovxr_session.OvxrSession( host, config, runtime_launcher_factory=launcher_factory ) + failure = error_type("cancel failed") - start_thread, start_errors = _call_in_thread(owner.start) - assert launcher.stop_started.wait(timeout=1.0) - close_thread, close_errors = _call_in_thread(owner.close) - try: - time.sleep(0.03) - assert "interrupt" not in [name for name, _thread in launcher.events] - finally: - launcher.allow_stop.set() - start_thread.join(timeout=1.0) - close_thread.join(timeout=1.0) + def cancel(_self, _plan): + raise failure - assert not start_thread.is_alive() - assert not close_thread.is_alive() - assert [str(error) for error in start_errors] == ["create failed"] - assert [str(error) for error in close_errors] == ["create failed"] - assert launcher.events == [ - ("stop-start", owner.render_worker_native_thread_id), - ("stop-end", owner.render_worker_native_thread_id), + monkeypatch.setattr(_FakeContext, "cancel_frame", cancel) + owner.start() + owner._pump._pending_plan = SimpleNamespace(step_result=0) + with pytest.raises(error_type) as caught: + owner.close() + assert caught.value is failure + assert "destroy" in [name for name, _thread in _FakeContext.operations] + assert owner._pump._context is None + assert owner._pump._pending_plan is None + assert [name for name, _thread in launcher.events] == ["stop-start", "stop-end"] + assert [event[0] for event in host.events if isinstance(event, tuple)][-2:] == [ + "host-close", + "environment-close", ] + assert not owner.retirement_complete # Cleanup does not erase the original error. def test_owner_lane_exit_failure_after_destroy_still_releases_retired_owners( @@ -1492,121 +2150,6 @@ def failing_lane(): ] -def test_cloudxr_stop_can_finish_without_runtime_interruption(monkeypatch, caplog): - host = _FakeHost() - launcher = _ControlledLauncher(block_stop=True) - - def launcher_factory(_path, *, accept_eula=False): - return launcher - - config = _config() - config.cloudxr_env_file = "/cloudxr.env" - config.auto_launch_cloudxr = True - owner = ovxr_session.OvxrSession( - host, config, runtime_launcher_factory=launcher_factory - ) - owner.start() - - close_thread, close_errors = _call_in_thread(owner.close) - try: - assert launcher.stop_started.wait(timeout=1.0) - assert owner._pump._runtime._launcher is launcher - time.sleep(0.03) - assert close_thread.is_alive() - assert "interrupt" not in [name for name, _thread in launcher.events] - assert "interrupting the owned OpenXR runtime" not in caplog.text - finally: - launcher.allow_stop.set() - close_thread.join(timeout=1.0) - - assert not close_thread.is_alive() - assert close_errors == [] - assert owner._pump._runtime._launcher is None - assert launcher.events == [ - ("stop-start", owner.render_worker_native_thread_id), - ("stop-end", owner.render_worker_native_thread_id), - ] - - -def test_close_times_out_without_blocking_on_startup_teardown_lock(monkeypatch): - host = _FakeHost() - owner = ovxr_session.OvxrSession(host, _config()) - entered = threading.Event() - resume = threading.Event() - destroy = _FakeContext.destroy - failure = RuntimeError("startup failed") - - def blocked_destroy(context, *, stage_ordinal): - entered.set() - assert resume.wait(timeout=2.0) - destroy(context, stage_ordinal=stage_ordinal) - - def fail_start(): - raise failure - - monkeypatch.setattr(_FakeContext, "destroy", blocked_destroy) - monkeypatch.setattr(owner._pump, "bootstrap", fail_start) - monkeypatch.setattr(ovxr_session, "_WORKER_SHUTDOWN_TIMEOUT_SECONDS", 0.01) - try: - with pytest.raises(RuntimeError) as caught: - owner.start() - assert caught.value is failure - assert entered.wait(timeout=1.0) - closing, errors = _call_in_thread(owner.close) - closing.join(timeout=0.5) - assert not closing.is_alive() - assert len(errors) == 1 - assert isinstance(errors[0], TimeoutError) - assert owner._pump._context is not None - assert not owner.retirement_complete - finally: - resume.set() - owner._worker.join(timeout=1.0) - with pytest.raises(RuntimeError) as caught: - owner.close() - assert caught.value is failure - - -def test_slow_context_retirement_keeps_owned_runtime_alive(monkeypatch): - host = _FakeHost() - launcher = _ControlledLauncher() - entered = threading.Event() - resume = threading.Event() - destroy = _FakeContext.destroy - - def slow_destroy(context, *, stage_ordinal): - entered.set() - assert resume.wait(timeout=3.0) - destroy(context, stage_ordinal=stage_ordinal) - - monkeypatch.setattr(_FakeContext, "destroy", slow_destroy) - - def launcher_factory(*args, **kwargs): - return launcher - - config = _config() - config.cloudxr_env_file = "/cloudxr.env" - config.auto_launch_cloudxr = True - owner = ovxr_session.OvxrSession( - host, config, runtime_launcher_factory=launcher_factory - ) - owner.start() - closing, errors = _call_in_thread(owner.close) - try: - assert entered.wait(timeout=1.0) - # Slow native retirement must keep its runtime alive until completion. - time.sleep(1.1) - assert closing.is_alive() - assert launcher.events == [] - finally: - resume.set() - closing.join(timeout=2.0) - assert not closing.is_alive() - assert errors == [] - assert owner.retirement_complete - assert [name for name, _thread in launcher.events] == ["stop-start", "stop-end"] - - @pytest.mark.parametrize("axis, expected", [("Z", 4), ("Y", 5)]) def test_host_up_axis_reaches_openxr_context(axis, expected): owner = ovxr_session.OvxrSession(_FakeHost(), _config(up_axis=axis)) From 6d69caa58f22bebddc45538c955cf3ad03c313b9 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Thu, 29 Oct 2026 15:46:10 -0400 Subject: [PATCH 2/2] fix: isolate optional camera-panel failures and drain pending copies Disable unavailable panels without blocking teleoperation input. Recheck the exact pending CUDA stream before retiring borrowed maps, visit independent mailboxes on cleanup failure, and name the upload result codes. Signed-off-by: Hougant Chen --- docs/source/getting_started/ovxr.rst | 6 + src/core/ovxr_pip_layer/cpp/pip_layer.cpp | 80 +++++++---- .../ovxr_session_manager/camera_panels.py | 36 ++++- .../ovxr_session_manager/openxr_quad.py | 59 +++++++- .../pip_layer_contract_test.cpp | 30 +++- .../ovxr_session_manager/test_openxr_quad.py | 133 +++++++++++++++++- .../ovxr_session_manager/test_ovxr_session.py | 103 ++++++++++++++ 7 files changed, 406 insertions(+), 41 deletions(-) diff --git a/docs/source/getting_started/ovxr.rst b/docs/source/getting_started/ovxr.rst index ccec8c4e7f..4ce1a7cb69 100644 --- a/docs/source/getting_started/ovxr.rst +++ b/docs/source/getting_started/ovxr.rst @@ -87,6 +87,12 @@ cameras. Set ``up_axis="Y"`` for a Y-up Stage; the default is ``"Z"``. ``CameraPanelConfig`` values, each pairing a render-product path with its placement and image bound. +Camera-panel attachment failure disables panels and logs the reason without +blocking teleoperation input. An incomplete GPU copy disables the affected panel; +the consumer rechecks its copy stream before releasing the borrowed output. +If completion still cannot be proved, the session reports a failure and retains +the borrowed resources. Other camera mailboxes are still given a cleanup attempt. + Consumers receive ``FrameOutputs`` on the XR owner thread. Use ``latest_color`` to select an output, then ``copy_host_rgba`` or ``map_gpu`` to consume it without unpacking native result structures. Selected handles are valid only during that diff --git a/src/core/ovxr_pip_layer/cpp/pip_layer.cpp b/src/core/ovxr_pip_layer/cpp/pip_layer.cpp index c5c57998c2..9b7d4d5c90 100644 --- a/src/core/ovxr_pip_layer/cpp/pip_layer.cpp +++ b/src/core/ovxr_pip_layer/cpp/pip_layer.cpp @@ -30,6 +30,21 @@ namespace { constexpr size_t kMaxCameras = 3; +// Integer C ABI mirrored by _UploadResult in openxr_quad.py. +enum UploadResult : int +{ + Success = 0, + InvalidArgument = -1, + NotAttached = -2, + CameraUnavailable = -3, + SwapchainFailed = -4, + AllocationFailed = -5, + UploadFailed = -6, + CompletionUnproved = -7, + CudaUnavailable = -8, + UploadModeMismatch = -9, +}; + struct Dispatch { PFN_xrGetInstanceProcAddr gipa = nullptr; @@ -734,42 +749,42 @@ extern "C" int isaacteleop_pip_publish_rgba( if (!owner || cameraIndex >= kMaxCameras || !rgba || !width || !height || width > kMaxImageDimension || height > kMaxImageDimension || static_cast(width) * height * 4 != length) { - return -1; + return UploadResult::InvalidArgument; } if (!owner->attached) { - return -2; + return UploadResult::NotAttached; } CameraSession& camera = owner->cameras[cameraIndex]; const auto& panel = owner->panels[cameraIndex]; if (!camera.device || !camera.queue || camera.retired || !(panel.headLocked ? camera.view : camera.local)) { - return -3; + return UploadResult::CameraUnavailable; } const Dispatch& dispatch = owner->dispatch; if ((camera.width != width || camera.height != height || !camera.swapchain) && !createSwapchain(camera, dispatch, width, height)) { - return -4; + return UploadResult::SwapchainFailed; } #if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) if (camera.cudaDevice >= 0) { - return -9; // Do not replace live CUDA aliases with CPU staging resources. + return UploadResult::UploadModeMismatch; // Do not replace live CUDA aliases with CPU staging resources. } #endif if ((!camera.staging || !camera.memory || !camera.pixels || !camera.pool || !camera.command || !camera.fence) && !createUploadResources(camera)) { - return -5; + return UploadResult::AllocationFailed; } - return upload(camera, dispatch, rgba) ? 0 : -6; + return upload(camera, dispatch, rgba) ? UploadResult::Success : UploadResult::UploadFailed; } // Application API, not an OVXR export. The caller owns a completed OVRTX map -// until this synchronous upload returns. -7 means completion is unproved and -// that map MUST remain retained; no further upload is admitted on the session. +// until this synchronous upload returns. CompletionUnproved retains that map +// until completion is proved; the affected panel retires. extern "C" int isaacteleop_pip_supports_cuda() { #if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) @@ -811,14 +826,31 @@ int copyCameraArrayAndWait(CameraSession& camera, const void* array, uint32_t wi if (cudaStreamSynchronize(camera.cudaStream) != cudaSuccess) { camera.retired = true; - return -7; + return UploadResult::CompletionUnproved; } camera.cudaPending = false; - return copied == cudaSuccess ? 0 : -6; + return copied == cudaSuccess ? UploadResult::Success : UploadResult::UploadFailed; } } // namespace #endif +// Completion-only recovery never retries a copy or destroys native resources. +extern "C" int isaacteleop_pip_wait_cuda(PanelSession* owner, uint32_t cameraIndex) +{ + if (!owner || cameraIndex >= kMaxCameras) + return UploadResult::InvalidArgument; +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + auto& camera = owner->cameras[cameraIndex]; + if (camera.cudaPending) + { + if (cudaSetDevice(camera.cudaDevice) != cudaSuccess || cudaStreamSynchronize(camera.cudaStream) != cudaSuccess) + return UploadResult::CompletionUnproved; + camera.cudaPending = false; + } +#endif + return UploadResult::Success; +} + extern "C" int isaacteleop_pip_publish_cuda_array( PanelSession* owner, uint32_t cameraIndex, const void* array, uint32_t width, uint32_t height, int device) { @@ -826,16 +858,16 @@ extern "C" int isaacteleop_pip_publish_cuda_array( if (!owner || cameraIndex >= kMaxCameras || !array || !width || !height || width > kMaxImageDimension || height > kMaxImageDimension || device < 0) { - return -1; + return UploadResult::InvalidArgument; } if (!owner->attached) { - return -2; + return UploadResult::NotAttached; } CameraSession& camera = owner->cameras[cameraIndex]; if (!camera.device || !camera.queue || camera.retired) { - return -3; + return UploadResult::CameraUnavailable; } // Resolve the borrowed Vulkan device once. A camera output's CUDA ordinal // is a source identity, not proof it is the destination Vulkan device. @@ -848,7 +880,7 @@ extern "C" int isaacteleop_pip_publish_cuda_array( int count = 0; if (cudaGetDeviceCount(&count) != cudaSuccess) { - return -8; + return UploadResult::CudaUnavailable; } for (int candidate = 0; candidate < count; ++candidate) { @@ -862,29 +894,29 @@ extern "C" int isaacteleop_pip_publish_cuda_array( } if (camera.vulkanCudaDevice < 0) { - return -8; + return UploadResult::CudaUnavailable; } } if (cudaSetDevice(camera.vulkanCudaDevice) != cudaSuccess) { - return -8; + return UploadResult::CudaUnavailable; } const Dispatch& dispatch = owner->dispatch; if ((camera.width != width || camera.height != height || !camera.swapchain) && !createSwapchain(camera, dispatch, width, height)) { - return -4; + return UploadResult::SwapchainFailed; } if (!camera.cudaPixels || !camera.pool || !camera.command || !camera.fence) { if (!releaseImageResources(camera)) { - return -7; + return UploadResult::CompletionUnproved; } camera.cudaDevice = camera.vulkanCudaDevice; if (!createUploadResources(camera, true)) { - return -5; + return UploadResult::AllocationFailed; } } if (camera.fencePending) @@ -892,20 +924,20 @@ extern "C" int isaacteleop_pip_publish_cuda_array( if (vkWaitForFences(camera.device, 1, &camera.fence, VK_TRUE, UINT64_MAX) != VK_SUCCESS) { camera.retired = true; - return -6; + return UploadResult::UploadFailed; } camera.fencePending = false; } camera.ready = false; const int copied = copyCameraArrayAndWait(camera, array, width, height, device); - if (copied != 0) + if (copied != UploadResult::Success) { return copied; } // No external semaphore wait is introduced: preserve the current host-wait // scheduling policy. This is a GPU-only copy, not end-to-end zero-copy. - return upload(camera, dispatch, nullptr) ? 0 : -6; + return upload(camera, dispatch, nullptr) ? UploadResult::Success : UploadResult::UploadFailed; #else - return -8; + return UploadResult::CudaUnavailable; #endif } diff --git a/src/python/isaaccapture/ovxr_session_manager/camera_panels.py b/src/python/isaaccapture/ovxr_session_manager/camera_panels.py index ce9188931e..832736c634 100644 --- a/src/python/isaaccapture/ovxr_session_manager/camera_panels.py +++ b/src/python/isaaccapture/ovxr_session_manager/camera_panels.py @@ -68,6 +68,8 @@ def __init__(self, panels: tuple[CameraPanelConfig, ...]) -> None: self._native = None self._mailboxes = [] self._layers = () + self._attach_attempted = False + self._enabled = False @property def render_product_paths(self) -> tuple[str, ...]: @@ -87,12 +89,24 @@ def on_session_started(self, binding, config, request_shutdown) -> None: ) def on_openxr_ready(self, openxr, vulkan) -> bool: - if vulkan is None: - return False - self._native.attach(openxr, vulkan) + if not self._attach_attempted: + self._attach_attempted = True + try: + if vulkan is None: + raise RuntimeError("Vulkan handles are unavailable") + self._native.attach(openxr, vulkan) + except Exception: + # Partial native attachment stays owned until normal close. + _LOGGER.exception( + "Camera panels disabled; teleoperation remains available" + ) + else: + self._enabled = True return True def on_outputs(self, frame: FrameOutputs) -> None: + if not self._enabled: + return layers = [] self._layers = () for index, mailbox in enumerate(self._mailboxes): @@ -115,8 +129,20 @@ def composition_layers(self) -> tuple[int, ...]: return self._layers def finish_frame(self) -> None: - for mailbox in self._mailboxes: - mailbox.close() + first_error = None + operations = [self._native.finish_frame] if self._native is not None else [] + operations.extend(mailbox.close for mailbox in self._mailboxes) + for finish in operations: + try: + finish() + except Exception as error: + # Retain unresolved maps without skipping independent mailboxes. + if first_error is None: + first_error = error + else: + first_error.add_note(str(error)) + if first_error is not None: + raise first_error def close(self) -> None: self.finish_frame() diff --git a/src/python/isaaccapture/ovxr_session_manager/openxr_quad.py b/src/python/isaaccapture/ovxr_session_manager/openxr_quad.py index fd922c4168..f85e602547 100644 --- a/src/python/isaaccapture/ovxr_session_manager/openxr_quad.py +++ b/src/python/isaaccapture/ovxr_session_manager/openxr_quad.py @@ -6,10 +6,29 @@ from __future__ import annotations import ctypes +import logging +from enum import IntEnum from pathlib import Path from .camera_mailbox import CameraImage +_LOGGER = logging.getLogger(__name__) + + +class _UploadResult(IntEnum): + """Integer C ABI shared with UploadResult in pip_layer.cpp.""" + + SUCCESS = 0 + INVALID_ARGUMENT = -1 + NOT_ATTACHED = -2 + CAMERA_UNAVAILABLE = -3 + SWAPCHAIN_FAILED = -4 + ALLOCATION_FAILED = -5 + UPLOAD_FAILED = -6 + COMPLETION_UNPROVED = -7 + CUDA_UNAVAILABLE = -8 + UPLOAD_MODE_MISMATCH = -9 + class OpenXRCameraPanels: """One native owner for panel spaces, swapchains and upload completion.""" @@ -30,6 +49,7 @@ def __init__(self, panels: tuple, *, library_path: str | None = None) -> None: "supports_cuda": ([], ctypes.c_int), "attach": ([pointer] * 7 + [uint], ctypes.c_int), "destroy": ([pointer], ctypes.c_int), + "wait_cuda": ([pointer, uint], ctypes.c_int), "layer": ([pointer, uint], pointer), "configure": ( [ @@ -64,6 +84,8 @@ def __init__(self, panels: tuple, *, library_path: str | None = None) -> None: self._published_revision = [0] * len(panels) self.last_errors = [None] * len(panels) self._attached = False + self._pending_uploads = {} + self._retired_panels = set() self._handle = self._native.isaacteleop_pip_create() if not self._handle: raise MemoryError("could not create camera panel owner") @@ -109,6 +131,8 @@ def _set_panel(self, index: int, enabled: bool, width: int, height: int) -> None def publish( self, index: int, image: CameraImage | None, *, invalid: bool = False ) -> bool: + if index in self._retired_panels: + return False if invalid: self.hide(index) self.last_errors[index] = "completed camera output was invalid" @@ -135,9 +159,15 @@ def publish( image.height, gpu.device, ) - # -7 cannot prove copy completion: retain the borrowed map. - if result != -7: + # Keep the borrowed map until the native copy has completed. + if result != _UploadResult.COMPLETION_UNPROVED: gpu.busy = False + else: + self._pending_uploads[index] = gpu + self._retired_panels.add(index) + _LOGGER.warning( + "Camera panel %s disabled after incomplete GPU copy", index + ) else: pixels = ctypes.create_string_buffer(image.rgba) result = self._native.isaacteleop_pip_publish_rgba( @@ -151,9 +181,11 @@ def publish( except Exception: self.hide(index) raise - if result != 0: + if result != _UploadResult.SUCCESS: self.hide(index) - self.last_errors[index] = f"OpenXR camera quad upload failed: {result}" + self.last_errors[index] = ( + f"OpenXR camera quad upload failed: {_UploadResult(result).name} ({result})" + ) return False self._published_revision[index] = image.revision self.last_errors[index] = None @@ -163,6 +195,25 @@ def hide(self, index: int) -> None: if self._visible[index]: self._set_panel(index, False, 1, 1) + def finish_frame(self) -> None: + first_error = None + for index, gpu in tuple(self._pending_uploads.items()): + result = self._native.isaacteleop_pip_wait_cuda(self._handle, index) + if result != _UploadResult.SUCCESS: + error = RuntimeError( + f"camera {index} GPU copy completion is unproved: " + f"{_UploadResult(result).name} ({result})" + ) + if first_error is None: + first_error = error + else: + first_error.add_note(str(error)) + continue + gpu.busy = False + del self._pending_uploads[index] + if first_error is not None: + raise first_error + def close(self) -> None: if self._handle: result = self._native.isaacteleop_pip_destroy(self._handle) diff --git a/tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp b/tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp index 3e397c74f4..c819ad88ee 100644 --- a/tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp +++ b/tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp @@ -216,6 +216,8 @@ int peerCopies = 0; cudaError_t copyResult = cudaSuccess; cudaError_t syncResult = cudaSuccess; int cudaSyncCalls = 0; +int currentCudaDevice = -1; +cudaStream_t lastCudaStream = nullptr; cudaMemcpy3DPeerParms peerCopy{}; } extern "C" cudaError_t CUDARTAPI cudaMemcpy2DFromArrayAsync( @@ -231,8 +233,14 @@ extern "C" cudaError_t CUDARTAPI cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerPar peerCopy = *parameters; return copyResult; } -extern "C" cudaError_t CUDARTAPI cudaStreamSynchronize(cudaStream_t) +extern "C" cudaError_t CUDARTAPI cudaSetDevice(int device) { + currentCudaDevice = device; + return cudaSuccess; +} +extern "C" cudaError_t CUDARTAPI cudaStreamSynchronize(cudaStream_t stream) +{ + lastCudaStream = stream; ++cudaSyncCalls; return syncResult; } @@ -255,12 +263,26 @@ int main() assert(peerCopy.dstPtr.pitch == 640 * 4 && peerCopy.dstPtr.xsize == 640 * 4); copyResult = cudaErrorInvalidDevice; assert(copyCameraArray(copyCamera, sourceArray, 640, 480, 1) == cudaErrorInvalidDevice); - assert(copyCameraArrayAndWait(copyCamera, sourceArray, 640, 480, 1) == -6); + assert(copyCameraArrayAndWait(copyCamera, sourceArray, 640, 480, 1) == UploadResult::UploadFailed); assert(cudaSyncCalls == 1 && !copyCamera.cudaPending && !copyCamera.retired); syncResult = cudaErrorUnknown; - assert(copyCameraArrayAndWait(copyCamera, sourceArray, 640, 480, 1) == -7); + assert(copyCameraArrayAndWait(copyCamera, sourceArray, 640, 480, 1) == UploadResult::CompletionUnproved); assert(cudaSyncCalls == 2 && copyCamera.cudaPending && copyCamera.retired); + PanelSession pendingOwner{}; + pendingOwner.cameras[1] = copyCamera; + pendingOwner.cameras[1].cudaStream = reinterpret_cast(uintptr_t{ 91 }); + assert(isaacteleop_pip_wait_cuda(&pendingOwner, 1) == UploadResult::CompletionUnproved); + assert(pendingOwner.cameras[1].cudaPending); + assert(currentCudaDevice == copyCamera.cudaDevice); + assert(lastCudaStream == pendingOwner.cameras[1].cudaStream); copyResult = syncResult = cudaSuccess; + assert(isaacteleop_pip_wait_cuda(&pendingOwner, 1) == 0); + assert(!pendingOwner.cameras[1].cudaPending && pendingOwner.cameras[1].retired); + const int completedSyncCalls = cudaSyncCalls; + assert(isaacteleop_pip_wait_cuda(&pendingOwner, 1) == 0); + assert(cudaSyncCalls == completedSyncCalls); + assert(isaacteleop_pip_wait_cuda(nullptr, 0) == UploadResult::InvalidArgument); + assert(isaacteleop_pip_wait_cuda(&pendingOwner, kMaxCameras) == UploadResult::InvalidArgument); #endif const auto instance = reinterpret_cast(uintptr_t{ 1 }); const auto session = reinterpret_cast(uintptr_t{ 2 }); @@ -339,7 +361,7 @@ int main() owned.cudaDevice = 1; owned.queue = VkQueue(25); owned.width = owned.height = 1; - assert(isaacteleop_pip_publish_rgba(owner, 0, &rgba, 1, 1, 4) == -9); + assert(isaacteleop_pip_publish_rgba(owner, 0, &rgba, 1, 1, 4) == UploadResult::UploadModeMismatch); assert(owned.pixels == &rgba); // Reject without overwriting any resource. owned.cudaDevice = -1; owned.device = VK_NULL_HANDLE; diff --git a/tests/python/core/ovxr_session_manager/test_openxr_quad.py b/tests/python/core/ovxr_session_manager/test_openxr_quad.py index fcd3f5bc77..a149bf3238 100644 --- a/tests/python/core/ovxr_session_manager/test_openxr_quad.py +++ b/tests/python/core/ovxr_session_manager/test_openxr_quad.py @@ -9,6 +9,7 @@ from isaaccapture.ovxr_session_manager.camera_mailbox import CameraImage from isaaccapture.ovxr_session_manager.camera_panels import CameraPanelConfig import isaaccapture.ovxr_session_manager.openxr_quad as quad_module +from isaaccapture.ovxr_session_manager.openxr_quad import _UploadResult class _Function: @@ -21,7 +22,7 @@ def __call__(self, *args): return self.result -def _quad(tmp_path, monkeypatch, *, panels=None): +def _quad(tmp_path, monkeypatch, *, panels=None, supports_gpu=False): library = tmp_path / "layer.so" library.touch() native = SimpleNamespace( @@ -30,6 +31,7 @@ def _quad(tmp_path, monkeypatch, *, panels=None): isaacteleop_pip_max_image_dimension=_Function(), isaacteleop_pip_attach=_Function(), isaacteleop_pip_destroy=_Function(), + isaacteleop_pip_wait_cuda=_Function(), isaacteleop_pip_layer=_Function(), isaacteleop_pip_configure=_Function(), isaacteleop_pip_publish_rgba=_Function(), @@ -37,6 +39,7 @@ def _quad(tmp_path, monkeypatch, *, panels=None): isaacteleop_pip_publish_cuda_array=_Function(), ) monkeypatch.setattr(quad_module.ctypes, "CDLL", lambda _path: native) + native.isaacteleop_pip_supports_cuda.result = int(supports_gpu) native.isaacteleop_pip_create.result = 123 native.isaacteleop_pip_max_cameras.result = 3 native.isaacteleop_pip_max_image_dimension.result = 4096 @@ -94,7 +97,7 @@ def test_failed_binding_retains_only_its_partial_cleanup_owner( def test_upload_failure_hides_last_valid_panel(tmp_path, monkeypatch): quad, native = _quad(tmp_path, monkeypatch) - native.isaacteleop_pip_publish_rgba.result = -6 + native.isaacteleop_pip_publish_rgba.result = _UploadResult.UPLOAD_FAILED assert not quad.publish(0, CameraImage(1, 1, b"\0" * 4, 1.0, 1)) assert native.isaacteleop_pip_configure.calls[-1][2] == 0 @@ -160,7 +163,9 @@ def test_old_native_layer_is_rejected_before_upload(tmp_path, monkeypatch): ) -@pytest.mark.parametrize("result", [0, -6, -7]) +@pytest.mark.parametrize( + "result", [0, _UploadResult.UPLOAD_FAILED, _UploadResult.COMPLETION_UNPROVED] +) def test_gpu_quad_completion_controls_source_retirement(tmp_path, monkeypatch, result): quad, native = _quad(tmp_path, monkeypatch) quad.supports_gpu = True @@ -170,6 +175,126 @@ def test_gpu_quad_completion_controls_source_retirement(tmp_path, monkeypatch, r image = CameraImage(720, 450, None, 1.0, 1, gpu) native.isaacteleop_pip_publish_cuda_array.result = result assert quad.publish(0, image) == (result == 0) - assert gpu.busy == (result == -7) + assert gpu.busy == (result == _UploadResult.COMPLETION_UNPROVED) assert len(native.isaacteleop_pip_publish_cuda_array.calls) == 1 assert native.isaacteleop_pip_publish_rgba.calls == [] + + +@pytest.mark.parametrize("wait_result", [0, _UploadResult.COMPLETION_UNPROVED]) +def test_failed_gpu_copy_finishes_before_source_retirement( + tmp_path, monkeypatch, wait_result +): + quad, native = _quad(tmp_path, monkeypatch) + quad.supports_gpu = True + gpu = SimpleNamespace( + closed=False, busy=False, is_array=True, pointer=123, device=1 + ) + image = CameraImage(720, 450, None, 1.0, 1, gpu) + native.isaacteleop_pip_publish_cuda_array.result = _UploadResult.COMPLETION_UNPROVED + assert not quad.publish(0, image) + native.isaacteleop_pip_wait_cuda.result = wait_result + if wait_result: + with pytest.raises(RuntimeError, match="copy completion is unproved"): + quad.finish_frame() + assert gpu.busy + native.isaacteleop_pip_wait_cuda.result = 0 + quad.finish_frame() + assert not gpu.busy + # Retirement disables this panel; a later frame must not retry its copy. + assert not quad.publish(0, CameraImage(720, 450, None, 2.0, 2, gpu)) + assert len(native.isaacteleop_pip_publish_cuda_array.calls) == 1 + quad.close() + + +@pytest.mark.parametrize("completion_recovers", [True, False]) +def test_incomplete_panel_copy_retires_only_safe_mailboxes( + tmp_path, monkeypatch, completion_recovers +): + """Drive real panel/mailbox owners, mocking only native output and copy APIs.""" + import ctypes + import threading + + from isaaccapture.ovxr_session_manager import camera_panels, gpu_output + from isaaccapture.ovxr_session_manager.camera_mailbox import FrameOutputs + + panels = tuple(CameraPanelConfig(f"/Camera/{i}") for i in range(2)) + quad, native = _quad(tmp_path, monkeypatch, panels=panels, supports_gpu=True) + monkeypatch.setattr(camera_panels, "OpenXRCameraPanels", lambda _: quad) + unmapped = [] + bindings = SimpleNamespace(OVRTX_API_SUCCESS=0, ovrtx_cuda_sync_t=lambda: None) + api = SimpleNamespace( + ovrtx_unmap_render_var_output=lambda _renderer, handle, _sync: ( + unmapped.append(handle) or SimpleNamespace(status=0) + ) + ) + monkeypatch.setattr(gpu_output, "_api", lambda: (bindings, api)) + + def map_image(renderer, handle, _limit): + return gpu_output.MappedGpuImage( + width=1, + height=1, + pointer=handle, + pitch=4, + device=0, + is_array=True, + renderer=renderer, + mapped=SimpleNamespace(map_handle=handle), + owner=threading.get_ident(), + ) + + monkeypatch.setattr(camera_panels, "map_cuda_array_rgba", map_image) + consumer = camera_panels.CameraPanelConsumer(panels) + consumer.on_session_started(None, None, None) + consumer.on_openxr_ready( + SimpleNamespace(xr_instance=1, xr_session=2, xr_get_instance_proc_addr=3), + SimpleNamespace( + vk_physical_device=4, + vk_device=5, + vk_queue=6, + queue_family_index=0, + external_memory_handle_types=1, + ), + ) + + def text(value): + return SimpleNamespace(ptr=ctypes.c_char_p(value), length=len(value)) + + outputs = SimpleNamespace( + contents=SimpleNamespace( + output_count=2, + outputs=[ + SimpleNamespace( + render_product_path=text(panel.render_product_path.encode()), + output_frame_count=1, + output_frames=[ + SimpleNamespace( + frame_end_time=1.0, + render_var_count=1, + output_render_vars=[ + SimpleNamespace( + source_name=text(b"LdrColor"), output_handle=i + 1 + ) + ], + ) + ], + ) + for i, panel in enumerate(panels) + ], + ) + ) + native.isaacteleop_pip_publish_cuda_array = lambda _owner, index, *_: ( + _UploadResult.COMPLETION_UNPROVED if index == 0 else 0 + ) + native.isaacteleop_pip_wait_cuda.result = ( + 0 if completion_recovers else _UploadResult.COMPLETION_UNPROVED + ) + consumer.on_outputs(FrameOutputs(1, outputs)) + if not completion_recovers: + with pytest.raises(RuntimeError, match="completion is unproved"): + consumer.close() + assert unmapped == [2] + assert not native.isaacteleop_pip_destroy.calls + native.isaacteleop_pip_wait_cuda.result = 0 + consumer.close() + assert sorted(unmapped) == [1, 2] + assert len(native.isaacteleop_pip_destroy.calls) == 1 diff --git a/tests/python/core/ovxr_session_manager/test_ovxr_session.py b/tests/python/core/ovxr_session_manager/test_ovxr_session.py index f8daca3c0a..b74fc6a5e1 100644 --- a/tests/python/core/ovxr_session_manager/test_ovxr_session.py +++ b/tests/python/core/ovxr_session_manager/test_ovxr_session.py @@ -278,6 +278,7 @@ def _runtime_stubs(monkeypatch): "OpenXRCameraPanels", lambda panels: SimpleNamespace( supports_gpu=False, + finish_frame=lambda: None, attach=lambda _xr, _vk: None, layer=lambda _index: None, publish=lambda _index, _image, **kw: True, @@ -1755,6 +1756,7 @@ def create(configs): assert configs == panels return SimpleNamespace( supports_gpu=False, + finish_frame=lambda: None, attach=lambda *_a: events.append("attach"), close=lambda: events.append("close"), ) @@ -1968,7 +1970,9 @@ def publish(*_args, **_kwargs): visible[0] = True consumer = CameraPanelConsumer((CameraPanelConfig("/Camera/Head"),)) + consumer._enabled = True consumer._native = SimpleNamespace( + finish_frame=lambda: None, hide=lambda _index: hide(), publish=publish, layer=lambda _index: 123 if visible[0] else 0, @@ -2208,3 +2212,102 @@ def fail_frame(_pump): finally: with pytest.raises(RuntimeError): owner.close() + + +@pytest.mark.parametrize("failure", ["vulkan_missing", "external_memory", "attach"]) +def test_unavailable_camera_panels_do_not_block_input_handles( + monkeypatch, tmp_path, caplog, failure +): + import ctypes + from unittest.mock import Mock + from isaaccapture.ovxr_session_manager.openxr_quad import OpenXRCameraPanels + + library = tmp_path / "panels.so" + library.touch() + native = SimpleNamespace( + **{ + "isaacteleop_pip_" + name: Mock(return_value=value) + for name, value in { + "create": 123, + "max_cameras": 3, + "max_image_dimension": 4096, + "supports_cuda": 1, + "attach": -3 if failure == "attach" else 0, + "destroy": 0, + "wait_cuda": 0, + "layer": 0, + "configure": 0, + "publish_rgba": 0, + "publish_cuda_array": 0, + }.items() + } + ) + monkeypatch.setattr(ctypes, "CDLL", lambda _path: native) + monkeypatch.setattr( + camera_panels, + "OpenXRCameraPanels", + lambda panels: OpenXRCameraPanels(panels, library_path=str(library)), + ) + vulkan = ( + None + if failure == "vulkan_missing" + else SimpleNamespace( + external_memory_handle_types=0 if failure == "external_memory" else 1, + vk_physical_device=1, + vk_device=2, + vk_queue=3, + queue_family_index=0, + ) + ) + attempts = [] + + def get_vulkan(self): + attempts.append(None) + if len(attempts) > 3: + raise RuntimeError("optional panels stalled core readiness") + return vulkan + + monkeypatch.setattr(_FakeContext, "get_vulkan_handles", get_vulkan) + consumer = CameraPanelConsumer((CameraPanelConfig("/Camera/Head"),)) + owner = _session_with_consumers(_FakeHost(), _config(), frame_consumers=(consumer,)) + try: + assert owner.start() + assert owner.external_oxr_handles() == (11, 22, 33, 44) + assert consumer.composition_layers() == () + assert "Camera panels disabled" in caplog.text + finally: + owner.close() + assert native.isaacteleop_pip_attach.call_count == int(failure == "attach") + native.isaacteleop_pip_destroy.assert_called_once() + + +@pytest.mark.parametrize("native_fails", [False, True]) +def test_camera_completion_failure_still_retires_independent_mailboxes(native_fails): + events = [] + pending = [True] + + def finish_native(): + events.append("wait") + if native_fails and pending[0]: + raise RuntimeError("native completion failed") + + def blocked(): + events.append("blocked") + if pending[0]: + raise RuntimeError("map still busy") + + consumer = CameraPanelConsumer((CameraPanelConfig("/Camera/Head"),)) + consumer._native = SimpleNamespace( + finish_frame=finish_native, close=lambda: events.append("native close") + ) + consumer._mailboxes = [ + SimpleNamespace(close=blocked), + SimpleNamespace(close=lambda: events.append("independent")), + ] + with pytest.raises(RuntimeError): + consumer.close() + assert events == ["wait", "blocked", "independent"] + assert consumer._native is not None + pending[0] = False + consumer.close() + assert events[-4:] == ["wait", "blocked", "independent", "native close"]