diff --git a/examples/camera_streamer/README.md b/examples/camera_streamer/README.md index 9d20dff01e..2d8fc367ab 100644 --- a/examples/camera_streamer/README.md +++ b/examples/camera_streamer/README.md @@ -11,7 +11,7 @@ configuration tests run without camera hardware. ## Protocol and configuration tests ```bash -cmake -S examples/camera_streamer/argus_sender -B /tmp/camera-protocol-build +cmake -S examples/camera_streamer/argus_sender -B /tmp/camera-protocol-build -DARGUS_SENDER_CPU_TESTS_ONLY=ON cmake --build /tmp/camera-protocol-build --parallel 2 ctest --test-dir /tmp/camera-protocol-build --output-on-failure ``` @@ -19,13 +19,37 @@ ctest --test-dir /tmp/camera-protocol-build --output-on-failure See [the wire specification](argus_sender/WIRE_FORMAT.md). Shared camera profiles are in `configs/`; select sensor modes using the installed camera driver. +## Jetson sender + +Install the camera vendor driver and Jetson Multimedia API on the Orin. Stop any +preview using the sensors and configure the vendor PWM for the selected frame rate. +From `examples/camera_streamer` (or a copy of that directory on the robot): + +```bash +cmake -S argus_sender -B argus_sender/build -DCMAKE_BUILD_TYPE=Release +cmake --build argus_sender/build --parallel 2 +ctest --test-dir argus_sender/build --output-on-failure +argus_sender/build/argus_sender --list-modes +PROFILE=s56-stereo-2560x1984-30 # Choose your resolution +argus_sender/build/argus_sender --config "configs/${PROFILE}.yaml" --check-config +argus_sender/build/argus_sender --config "configs/${PROFILE}.yaml" +``` + +The sender listens for a TCP receiver on port 5000 by default. + ## Licensing This example is Apache-2.0 licensed. Existing upstream notices and external SDK -licenses remain in effect. +licenses remain in effect. The Argus-derived `EGLGlobal.h`, `EGLGlobal.cpp` and +`Error.h` retain their BSD-3-Clause license and notices. ## Rebuilding after a source-directory move CMake caches absolute source paths. After updating an existing deployment, configure and build the native sender and decoder again at the paths shown above. Do not move or reuse an old build directory from a different source location. + +For opt-in hardware smoke tests, install PyYAML for the Python interpreter selected +by CMake. Set `ARGUS_SENDER_HARDWARE_CONFIG` to your stereo profile; the smoke receiver +reads `network.port` from that same file (default 5000), so there is no separate +receiver-port override. diff --git a/examples/camera_streamer/argus_sender/.gitignore b/examples/camera_streamer/argus_sender/.gitignore new file mode 100644 index 0000000000..8cf2ed8f5f --- /dev/null +++ b/examples/camera_streamer/argus_sender/.gitignore @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# CMake build trees and generated files +/build/ +/build-*/ +/cmake-build-*/ +CMakeCache.txt +CMakeFiles/ +cmake_install.cmake +compile_commands.json +CTestTestfile.cmake +Testing/ +_deps/ + +# Compiled artifacts +*.o +*.a +*.so +*.so.* +*.d + +# Camera and encoded stream dumps +*.hvs2 +*.avs1 +*.h265 +*.hevc +*.yuv +*.nv12 +*.raw + +# Runtime and crash output +*.log +core +core.* + +# Editor and operating-system files +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Deployed wrist remap LUTs — generated by setup_video_streaming.sh from the +# calibration tree, rsynced to the AGX with the source tree. +/remaps/ diff --git a/examples/camera_streamer/argus_sender/CMakeLists.txt b/examples/camera_streamer/argus_sender/CMakeLists.txt index e75d45c825..541a2f9c53 100644 --- a/examples/camera_streamer/argus_sender/CMakeLists.txt +++ b/examples/camera_streamer/argus_sender/CMakeLists.txt @@ -2,11 +2,134 @@ # SPDX-License-Identifier: Apache-2.0 cmake_minimum_required(VERSION 3.16) + project(argus_sender LANGUAGES CXX) + set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) include(CTest) + +option(ARGUS_SENDER_CPU_TESTS_ONLY "Build only portable protocol/configuration tests" OFF) +if(ARGUS_SENDER_CPU_TESTS_ONLY) + find_package(Threads REQUIRED) + include(cmake/YamlCpp.cmake) + set(ARGUS_SENDER_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") + include(cmake/Tests.cmake) + return() +endif() + +option(DISABLE_MULTIPROCESS + "Link the single-process Argus implementation instead of nvargus-daemon's socket client" + OFF) + +set(MMAPI_ROOT "/usr/src/jetson_multimedia_api" CACHE PATH + "Jetson Multimedia API root") +set(MMAPI_CLASS_DIR "${MMAPI_ROOT}/samples/common/classes" CACHE PATH + "Jetson Multimedia API common/classes directory") + +set(NVIDIA_LIBRARY_DIRS + /usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}/nvidia + /usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}/tegra + /usr/lib/aarch64-linux-gnu/nvidia + /usr/lib/aarch64-linux-gnu/tegra + /opt/nvidia/l4t-gpu-libs/openrm + /opt/nvidia/l4t-gpu-libs/nvgpu +) + +find_path(ARGUS_INCLUDE_DIR Argus/Argus.h + HINTS "${MMAPI_ROOT}/argus/include" "${CMAKE_CURRENT_SOURCE_DIR}/include") +find_library(ARGUS_SOCKETCLIENT_LIBRARY NAMES nvargus_socketclient + HINTS ${NVIDIA_LIBRARY_DIRS}) +find_library(ARGUS_LIBRARY NAMES nvargus + HINTS ${NVIDIA_LIBRARY_DIRS}) + +find_path(MMAPI_INCLUDE_DIR NvVideoEncoder.h + HINTS "${MMAPI_ROOT}/include") +find_path(CUDA_INCLUDE_DIR cuda.h + HINTS /usr/local/cuda/include /usr/include) +find_path(DRM_INCLUDE_DIR drm.h + HINTS /usr/include/libdrm) + +find_library(EGL_LIBRARY NAMES EGL) +find_library(NVV4L2_LIBRARY NAMES nvv4l2 + HINTS ${NVIDIA_LIBRARY_DIRS}) +find_library(NVBUFSURFACE_LIBRARY NAMES nvbufsurface + HINTS ${NVIDIA_LIBRARY_DIRS}) +find_library(CUDA_DRIVER_LIBRARY NAMES cuda + HINTS ${NVIDIA_LIBRARY_DIRS} /usr/lib/aarch64-linux-gnu) + find_package(Threads REQUIRED) + include(cmake/YamlCpp.cmake) + +set(REQUIRED_PATHS + ARGUS_INCLUDE_DIR + MMAPI_INCLUDE_DIR + CUDA_INCLUDE_DIR + DRM_INCLUDE_DIR + MMAPI_CLASS_DIR +) +foreach(path_var IN LISTS REQUIRED_PATHS) + if(NOT ${path_var}) + message(FATAL_ERROR "${path_var} was not found") + endif() +endforeach() + +set(REQUIRED_LIBS + EGL_LIBRARY + NVV4L2_LIBRARY + NVBUFSURFACE_LIBRARY + CUDA_DRIVER_LIBRARY +) +foreach(lib_var IN LISTS REQUIRED_LIBS) + if(NOT ${lib_var}) + message(FATAL_ERROR "${lib_var} was not found") + endif() +endforeach() + +if(DISABLE_MULTIPROCESS) + if(NOT ARGUS_LIBRARY) + message(FATAL_ERROR + "DISABLE_MULTIPROCESS=ON requires the single-process nvargus library") + endif() + set(ARGUS_LINK_LIBRARY "${ARGUS_LIBRARY}") + set(ARGUS_LINK_MODE "single-process") +else() + if(NOT ARGUS_SOCKETCLIENT_LIBRARY) + message(FATAL_ERROR + "The multiprocess build requires the nvargus_socketclient library") + endif() + set(ARGUS_LINK_LIBRARY "${ARGUS_SOCKETCLIENT_LIBRARY}") + set(ARGUS_LINK_MODE "multiprocess") +endif() + +set(MMAPI_CLASS_SOURCES + ${MMAPI_CLASS_DIR}/NvBuffer.cpp + ${MMAPI_CLASS_DIR}/NvElement.cpp + ${MMAPI_CLASS_DIR}/NvElementProfiler.cpp + ${MMAPI_CLASS_DIR}/NvLogging.cpp + ${MMAPI_CLASS_DIR}/NvV4l2Element.cpp + ${MMAPI_CLASS_DIR}/NvV4l2ElementPlane.cpp + ${MMAPI_CLASS_DIR}/NvVideoEncoder.cpp +) + +foreach(source_file IN LISTS MMAPI_CLASS_SOURCES) + if(NOT EXISTS "${source_file}") + message(FATAL_ERROR "Required Multimedia API source not found: ${source_file}") + endif() +endforeach() + set(ARGUS_SENDER_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") -include(cmake/Tests.cmake) +add_subdirectory(src) + +if(BUILD_TESTING) + include(cmake/Tests.cmake) +endif() + + +message(STATUS "Argus include: ${ARGUS_INCLUDE_DIR}") +message(STATUS "Argus library (${ARGUS_LINK_MODE}): ${ARGUS_LINK_LIBRARY}") +message(STATUS "Multimedia API include: ${MMAPI_INCLUDE_DIR}") +message(STATUS "nvv4l2: ${NVV4L2_LIBRARY}") +message(STATUS "nvbufsurface: ${NVBUFSURFACE_LIBRARY}") diff --git a/examples/camera_streamer/argus_sender/cmake/Tests.cmake b/examples/camera_streamer/argus_sender/cmake/Tests.cmake index bafdce9a00..18eeda3c34 100644 --- a/examples/camera_streamer/argus_sender/cmake/Tests.cmake +++ b/examples/camera_streamer/argus_sender/cmake/Tests.cmake @@ -7,6 +7,7 @@ set(CAMERA_STREAMER_TEST_ROOT "${CAMERA_STREAMER_ROOT}/tests") set(ARGUS_SENDER_TEST_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/src ${CAMERA_STREAMER_TEST_ROOT}/cpp + ${ARGUS_INCLUDE_DIR} ) function(argus_sender_add_cpp_test target test_name) @@ -17,6 +18,7 @@ function(argus_sender_add_cpp_test target test_name) ${ARGUS_SENDER_TEST_INCLUDE_DIRS} ) target_link_libraries(${target} PRIVATE + ${ARGUS_LINK_LIBRARY} Threads::Threads ) add_test(NAME ${test_name} COMMAND ${target}) @@ -33,3 +35,37 @@ add_subdirectory(${CAMERA_STREAMER_TEST_ROOT}/cpp/frame_alignment_policy ${CMAKE add_subdirectory(${CAMERA_STREAMER_TEST_ROOT}/cpp/capture_domain_plan ${CMAKE_CURRENT_BINARY_DIR}/tests/capture_domain_plan) add_subdirectory(${CAMERA_STREAMER_TEST_ROOT}/cpp/capture_priority_policy ${CMAKE_CURRENT_BINARY_DIR}/tests/capture_priority_policy) add_subdirectory(${CAMERA_STREAMER_TEST_ROOT}/cpp/avs1_wire ${CMAKE_CURRENT_BINARY_DIR}/tests/avs1_wire) + +if(ARGUS_SENDER_CPU_TESTS_ONLY) + return() +endif() + +add_subdirectory(${CAMERA_STREAMER_TEST_ROOT}/cpp/nv_remap ${CMAKE_CURRENT_BINARY_DIR}/tests/nv_remap) +add_subdirectory(${CAMERA_STREAMER_TEST_ROOT}/cpp/rate_limiter ${CMAKE_CURRENT_BINARY_DIR}/tests/rate_limiter) +add_subdirectory(${CAMERA_STREAMER_TEST_ROOT}/cpp/alignment_policy ${CMAKE_CURRENT_BINARY_DIR}/tests/alignment_policy) +add_subdirectory(${CAMERA_STREAMER_TEST_ROOT}/cpp/bundle_assembler ${CMAKE_CURRENT_BINARY_DIR}/tests/bundle_assembler) +add_subdirectory(${CAMERA_STREAMER_TEST_ROOT}/cpp/tcp_transport ${CMAKE_CURRENT_BINARY_DIR}/tests/tcp_transport) + +option(ARGUS_SENDER_HARDWARE_TESTS + "Enable tests that capture from physical stereo cameras" + OFF) +set(ARGUS_SENDER_HARDWARE_CONFIG + "${CMAKE_CURRENT_SOURCE_DIR}/configs/hardware_stereo_smoke.yaml" + CACHE FILEPATH "Config used by the hardware smoke test") + +if(ARGUS_SENDER_HARDWARE_TESTS) + find_package(Python3 REQUIRED COMPONENTS Interpreter) + add_test( + NAME avs1_hardware_smoke + COMMAND ${Python3_EXECUTABLE} + ${CAMERA_STREAMER_TEST_ROOT}/hardware/avs1_smoke.py + --sender $ + --config ${ARGUS_SENDER_HARDWARE_CONFIG} + --frames 120 + ) + set_tests_properties(avs1_hardware_smoke PROPERTIES + LABELS "hardware" + RUN_SERIAL TRUE + TIMEOUT 90 + ) +endif() diff --git a/examples/camera_streamer/argus_sender/configs/hardware_stereo_smoke.yaml b/examples/camera_streamer/argus_sender/configs/hardware_stereo_smoke.yaml new file mode 100644 index 0000000000..01e0ada6d6 --- /dev/null +++ b/examples/camera_streamer/argus_sender/configs/hardware_stereo_smoke.yaml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version: 1 + +network: + bind: 127.0.0.1 + port: 55000 + reconnect_ms: 100 + +runtime: + frames: 120 + kpi: true + +alignment: + warmup_frames: 60 + interval_frames: 60 + +streams: + - id: smoke_stereo + type: stereo + cameras: + - {device: 0, mode: 0} + - {device: 1, mode: 0} + output: + fps: 60 + codec: hevc + rotation: 0 + resize_mode: strict + bitrate_mbps: 50 + gop: 60 diff --git a/examples/camera_streamer/argus_sender/src/CMakeLists.txt b/examples/camera_streamer/argus_sender/src/CMakeLists.txt new file mode 100644 index 0000000000..a004b98abb --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/CMakeLists.txt @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +add_executable(argus_sender + main.cpp + Config.cpp + Avs1Wire.cpp + OutputGeometry.cpp + MixedRateScheduler.cpp + CapturePriorityPolicy.cpp + CaptureDomainPlan.cpp + CaptureManager.cpp + FrameBundleAssembler.cpp + StreamProcessor.cpp + CaptureCoordinator.cpp + SenderSession.cpp + Runtime.cpp + CameraCatalog.cpp + FrameCopyWorker.cpp + NvRemap.cpp + FrameAlignmentPolicy.cpp + StereoAlignmentPolicy.cpp + TcpTransport.cpp + HevcEncoder.cpp + StereoTiming.cpp + EGLGlobal.cpp + NvBufSurfaceLite.cpp + ${MMAPI_CLASS_SOURCES} +) + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(argus_sender PRIVATE -Wall -Wextra) + set_source_files_properties(${MMAPI_CLASS_SOURCES} + PROPERTIES COMPILE_OPTIONS -w) +endif() + +target_compile_definitions(argus_sender PRIVATE NVMMAPI_SUPPORTED) + +target_include_directories(argus_sender PRIVATE + ${ARGUS_INCLUDE_DIR} + ${MMAPI_INCLUDE_DIR} + ${MMAPI_CLASS_DIR} + ${CUDA_INCLUDE_DIR} + ${DRM_INCLUDE_DIR} +) + +target_link_libraries(argus_sender PRIVATE + ${ARGUS_LINK_LIBRARY} + ${EGL_LIBRARY} + ${NVV4L2_LIBRARY} + ${NVBUFSURFACE_LIBRARY} + ${CUDA_DRIVER_LIBRARY} + yaml-cpp::yaml-cpp + Threads::Threads +) + +set_target_properties(argus_sender PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") +install(TARGETS argus_sender DESTINATION bin) diff --git a/examples/camera_streamer/argus_sender/src/CameraCatalog.cpp b/examples/camera_streamer/argus_sender/src/CameraCatalog.cpp new file mode 100644 index 0000000000..d8538a179b --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/CameraCatalog.cpp @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "CameraCatalog.h" + +#include "Error.h" + +#include +#include + +namespace argus_sender +{ +namespace +{ + +void printSensorMode(uint32_t index, Argus::SensorMode* mode, const char* indent) +{ + Argus::ISensorMode* iMode = Argus::interface_cast(mode); + if (!iMode) + return; + const Argus::Size2D resolution = iMode->getResolution(); + const Argus::Range durations = iMode->getFrameDurationRange(); + const double minFps = frameRateFromDurationNs(durations.max()); + const double maxFps = frameRateFromDurationNs(durations.min()); + std::printf("%sMode %u: %ux%u, frameDuration=%llu..%llu ns, fps=%.2f..%.2f\n", indent, index, resolution.width(), + resolution.height(), static_cast(durations.min()), + static_cast(durations.max()), minFps, maxFps); +} + +bool getCameraDevices(Argus::CameraProvider* provider, std::vector* devices) +{ + Argus::ICameraProvider* iProvider = Argus::interface_cast(provider); + if (!iProvider) + ORIGINATE_ERROR("Failed to get ICameraProvider interface"); + if (iProvider->getCameraDevices(devices) != Argus::STATUS_OK) + ORIGINATE_ERROR("Failed to get camera devices from provider"); + if (devices->empty()) + ORIGINATE_ERROR("No camera devices are available"); + return true; +} + +bool getSensorModes(Argus::CameraDevice* device, std::vector* modes) +{ + Argus::ICameraProperties* iProperties = Argus::interface_cast(device); + if (!iProperties) + ORIGINATE_ERROR("Failed to get ICameraProperties interface"); + if (iProperties->getAllSensorModes(modes) != Argus::STATUS_OK) + ORIGINATE_ERROR("Failed to get sensor modes from camera device"); + if (modes->empty()) + ORIGINATE_ERROR("No sensor modes are available for this camera device"); + return true; +} + +} // namespace + +double frameRateFromDurationNs(uint64_t durationNs) +{ + return durationNs == 0 ? 0.0 : 1000000000.0 / static_cast(durationNs); +} + +uint32_t roundedFrameRateFromDurationNs(uint64_t durationNs) +{ + if (durationNs == 0) + return 0; + const uint64_t rounded = (1000000000ULL + durationNs / 2) / durationNs; + return static_cast(rounded > 0 ? rounded : 1); +} + +bool listCameraModes(Argus::CameraProvider* provider) +{ + std::vector devices; + PROPAGATE_ERROR(getCameraDevices(provider, &devices)); + std::printf("Available CameraDevices: %zu\n", devices.size()); + for (uint32_t i = 0; i < devices.size(); ++i) + { + std::printf("CameraDevice %u:\n", i); + std::vector modes; + if (!getSensorModes(devices[i], &modes)) + continue; + for (uint32_t mode = 0; mode < modes.size(); ++mode) + printSensorMode(mode, modes[mode], " "); + } + return true; +} + +Argus::CameraDevice* selectCameraDevice(Argus::CameraProvider* provider, uint32_t index) +{ + std::vector devices; + if (!getCameraDevices(provider, &devices)) + return NULL; + if (index >= devices.size()) + { + std::fprintf(stderr, "Camera device %u was requested but only %zu are available.\n", index, devices.size()); + return NULL; + } + return devices[index]; +} + +Argus::SensorMode* selectSensorMode(Argus::CameraDevice* device, uint32_t index) +{ + std::vector modes; + if (!getSensorModes(device, &modes)) + return NULL; + if (index >= modes.size()) + { + std::fprintf(stderr, "Sensor mode %u was requested but only %zu are available.\n", index, modes.size()); + for (uint32_t mode = 0; mode < modes.size(); ++mode) + printSensorMode(mode, modes[mode], " "); + return NULL; + } + return modes[index]; +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/CameraCatalog.h b/examples/camera_streamer/argus_sender/src/CameraCatalog.h new file mode 100644 index 0000000000..633800e502 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/CameraCatalog.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_CAMERA_CATALOG_H +#define ARGUS_SENDER_CAMERA_CATALOG_H + +#include + +#include + +namespace argus_sender +{ + +double frameRateFromDurationNs(uint64_t durationNs); +uint32_t roundedFrameRateFromDurationNs(uint64_t durationNs); + +bool listCameraModes(Argus::CameraProvider* provider); +Argus::CameraDevice* selectCameraDevice(Argus::CameraProvider* provider, uint32_t index); +Argus::SensorMode* selectSensorMode(Argus::CameraDevice* device, uint32_t index); + +} // namespace argus_sender + +#endif // ARGUS_SENDER_CAMERA_CATALOG_H diff --git a/examples/camera_streamer/argus_sender/src/CaptureCoordinator.cpp b/examples/camera_streamer/argus_sender/src/CaptureCoordinator.cpp new file mode 100644 index 0000000000..2d87d7e98a --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/CaptureCoordinator.cpp @@ -0,0 +1,781 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "CaptureCoordinator.h" + +#include "Avs1Wire.h" +#include "CameraCatalog.h" +#include "CaptureManager.h" +#include "CapturePriorityPolicy.h" +#include "Config.h" +#include "EGLGlobal.h" +#include "Error.h" +#include "FrameAlignmentPolicy.h" +#include "FrameBundleAssembler.h" +#include "MixedRateScheduler.h" +#include "Runtime.h" +#include "StreamProcessor.h" +#include "TcpTransport.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace argus_sender +{ +namespace +{ + +const uint32_t kTcpQueueDepth = 1; +const uint32_t kPrioritySyncAttempts = 120; +const uint32_t kPrioritySyncConsecutiveFrames = 3; + +bool releaseSecondaryCaptures(CaptureManager* capture, size_t priorityDomainIndex, uint64_t delayNs) +{ + if (!capture) + return false; + if (capture->domainCount() <= 1) + return true; + if (delayNs > 0) + std::this_thread::sleep_for(std::chrono::nanoseconds(delayNs)); + for (size_t domainIndex = 0; domainIndex < capture->domainCount(); ++domainIndex) + { + if (domainIndex != priorityDomainIndex && !capture->requestDomainCapture(domainIndex)) + { + return false; + } + } + return true; +} + +class PendingCopyScope +{ +public: + explicit PendingCopyScope(std::vector>* processors) : m_processors(processors) + { + } + + ~PendingCopyScope() + { + if (!m_processors) + return; + for (size_t i = 0; i < m_processors->size(); ++i) + (*m_processors)[i]->discardPendingCopies(); + } + +private: + std::vector>* m_processors; +}; + +uint64_t createSessionId() +{ + const uint64_t now = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); + return (now << 16) ^ static_cast(getpid()) ^ 0x4156533100000000ULL; +} + +std::vector buildStreamTemplates(const AppConfig& config, + const CaptureManager& capture, + const std::vector& streamFps) +{ + std::vector templates(config.streams.size()); + for (size_t streamIndex = 0; streamIndex < config.streams.size(); ++streamIndex) + { + const StreamConfig& configStream = config.streams[streamIndex]; + const ResolvedOutputGeometry& geometry = capture.geometry(streamIndex); + Avs1StreamRecord& stream = templates[streamIndex]; + if (configStream.type == STREAM_TYPE_STEREO) + stream.flags |= AVS1_STREAM_STEREO; + stream.codec = configStream.output.codec == CODEC_HEVC ? AVS1_CODEC_HEVC : AVS1_CODEC_RAW; + stream.pixelFormat = AVS1_PIXEL_FORMAT_NV12; + stream.rotation = static_cast(configStream.output.rotation); + stream.width = geometry.width; + stream.height = geometry.height; + stream.fpsNumerator = streamFps[streamIndex]; + stream.fpsDenominator = 1; + stream.samples.resize(configStream.cameras.size()); + for (size_t camera = 0; camera < configStream.cameras.size(); ++camera) + { + Avs1SampleRecord& sample = stream.samples[camera]; + sample.cameraIndex = static_cast(camera); + sample.role = configStream.type == STREAM_TYPE_STEREO ? (camera == 0 ? AVS1_ROLE_LEFT : AVS1_ROLE_RIGHT) : + AVS1_ROLE_MONO; + sample.deviceIndex = configStream.cameras[camera].device; + sample.sensorModeIndex = configStream.cameras[camera].mode; + } + } + return templates; +} + +struct AlignmentFrameLocation +{ + size_t domainIndex = 0; +}; + +bool alignCapturedFrames(CaptureManager* capture, + std::vector* captured, + bool startup, + bool* unavailableWarned) +{ + if (!capture || !captured || captured->size() != capture->streamCount()) + { + return false; + } + + const std::chrono::steady_clock::time_point alignStart = std::chrono::steady_clock::now(); + uint32_t drops = 0; + uint64_t finalSpanNs = 0; + FrameAlignmentDecision finalDecision = FRAME_ALIGNMENT_KEEP; + while (true) + { + std::vector samples; + std::vector locations; + for (size_t streamIndex = 0; streamIndex < captured->size(); ++streamIndex) + { + const CapturedFrameSet& frames = (*captured)[streamIndex]; + if (frames.timings.size() != frames.frames.size()) + return false; + for (size_t cameraIndex = 0; cameraIndex < frames.timings.size(); ++cameraIndex) + { + FrameAlignmentSample sample; + sample.valid = frames.timings[cameraIndex].hasSensorSofTimestampTsc; + sample.sofTscNs = frames.timings[cameraIndex].sensorSofTimestampTscNs; + sample.framePeriodNs = capture->framePeriodNs(streamIndex); + samples.push_back(sample); + AlignmentFrameLocation location; + location.domainIndex = capture->domainFor(streamIndex, cameraIndex); + locations.push_back(location); + } + } + + size_t sampleIndex = 0; + finalDecision = chooseFrameToReacquire(samples, &sampleIndex, &finalSpanNs); + if (finalDecision == FRAME_ALIGNMENT_UNAVAILABLE) + { + if (!unavailableWarned || !*unavailableWarned) + { + std::fprintf( + stderr, + "Warning: global frame alignment skipped because a due frame lacks a valid SOF TSC timestamp\n"); + if (unavailableWarned) + *unavailableWarned = true; + } + return true; + } + if (finalDecision == FRAME_ALIGNMENT_KEEP || drops >= kMaxFrameAlignmentDrops) + { + break; + } + if (sampleIndex >= locations.size()) + return false; + const AlignmentFrameLocation& location = locations[sampleIndex]; + if (!capture->reacquireDomain(location.domainIndex, captured)) + { + return false; + } + ++drops; + } + + if (verboseLoggingEnabled() && (startup || drops > 0 || finalDecision == FRAME_ALIGNMENT_REACQUIRE)) + { + const uint64_t elapsedUs = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - alignStart).count()); + std::printf("argus_sender[alignment]: %s dropped=%u, tsc_span_us=%.3f, within_half_frame=%s, reacquire_us=%llu\n", + startup ? "startup" : "periodic", drops, static_cast(finalSpanNs) / 1000.0, + finalDecision == FRAME_ALIGNMENT_KEEP ? "yes" : "no", static_cast(elapsedUs)); + } + return true; +} + +bool captureTscSpanNs(const std::vector& captured, uint64_t* spanNs) +{ + if (!spanNs) + return false; + uint64_t earliest = 0; + uint64_t latest = 0; + size_t samples = 0; + for (size_t stream = 0; stream < captured.size(); ++stream) + { + for (size_t camera = 0; camera < captured[stream].timings.size(); ++camera) + { + const FrameTiming& timing = captured[stream].timings[camera]; + if (!timing.hasSensorSofTimestampTsc || timing.sensorSofTimestampTscNs == 0) + { + return false; + } + if (earliest == 0 || timing.sensorSofTimestampTscNs < earliest) + { + earliest = timing.sensorSofTimestampTscNs; + } + latest = std::max(latest, timing.sensorSofTimestampTscNs); + ++samples; + } + } + if (samples == 0) + return false; + *spanNs = latest - earliest; + return true; +} + +// The sender TSC "now" is only trusted when it lands shortly after the due +// captures' VI SOF stamps, which proves it is in the same Tegra TSC domain. +// Returns 0 when the check fails so the bundle leaves +// AVS1_BUNDLE_SENDER_TSC_VALID clear. Each verdict is logged once per process. +uint64_t validatedCaptureReadyTscNs(uint64_t tscNowNs, const std::vector& inputs) +{ + static bool enabledLogged = false; + static bool disabledLogged = false; + const uint64_t kMaxSofLagNs = 500000000ULL; + uint64_t latestSofTscNs = 0; + for (size_t stream = 0; stream < inputs.size(); ++stream) + { + if (!inputs[stream].present) + continue; + for (size_t camera = 0; camera < inputs[stream].timings.size(); ++camera) + { + const FrameTiming& timing = inputs[stream].timings[camera]; + if (timing.hasSensorSofTimestampTsc) + { + latestSofTscNs = std::max(latestSofTscNs, timing.sensorSofTimestampTscNs); + } + } + } + const bool valid = + tscNowNs != 0 && latestSofTscNs != 0 && tscNowNs > latestSofTscNs && tscNowNs - latestSofTscNs < kMaxSofLagNs; + if (!valid) + { + if (!disabledLogged) + { + disabledLogged = true; + std::fprintf( + stderr, + "argus_sender[coordinator]: sensor TSC domain check failed: now=%llu sof=%llu -> sensor SOF wall stamps disabled\n", + static_cast(tscNowNs), static_cast(latestSofTscNs)); + } + return 0; + } + if (!enabledLogged) + { + enabledLogged = true; + std::fprintf( + stderr, + "argus_sender[coordinator]: sensor SOF -> capture-ready lag %.1f ms; sensor SOF wall stamps enabled\n", + static_cast(tscNowNs - latestSofTscNs) / 1e6); + } + return tscNowNs; +} + +bool frameSetReferenceSofTscNs(const CapturedFrameSet& frames, + uint64_t toleranceNs, + uint64_t* referenceSofTscNs, + uint64_t* spanNs) +{ + if (!referenceSofTscNs || frames.timings.empty()) + return false; + std::vector timestamps; + timestamps.reserve(frames.timings.size()); + for (size_t i = 0; i < frames.timings.size(); ++i) + { + const FrameTiming& timing = frames.timings[i]; + if (!timing.hasSensorSofTimestampTsc || timing.sensorSofTimestampTscNs == 0) + { + return false; + } + timestamps.push_back(timing.sensorSofTimestampTscNs); + } + return computeSynchronizedTscReference(timestamps, toleranceNs, referenceSofTscNs, spanNs); +} + +} // namespace + +bool printCameraModes() +{ + ArgusSamples::EGLDisplayHolder display(true); + PROPAGATE_ERROR(display.initialize(EGL_DEFAULT_DISPLAY)); + Argus::UniqueObj provider(Argus::CameraProvider::create()); + Argus::ICameraProvider* iProvider = Argus::interface_cast(provider); + if (!iProvider) + ORIGINATE_ERROR("Failed to create CameraProvider"); + if (verboseLoggingEnabled()) + std::printf("Argus Version: %s\n", iProvider->getVersion().c_str()); + const bool ok = listCameraModes(provider.get()); + provider.reset(); + display.cleanup(); + return ok; +} + +bool runConfiguredCapture(const AppConfig& config, bool* receiverDisconnected, bool appendDump) +{ + if (receiverDisconnected) + *receiverDisconnected = false; + ArgusSamples::EGLDisplayHolder display(true); + PROPAGATE_ERROR(display.initialize(EGL_DEFAULT_DISPLAY)); + Argus::UniqueObj provider(Argus::CameraProvider::create()); + Argus::ICameraProvider* iProvider = Argus::interface_cast(provider); + if (!iProvider) + ORIGINATE_ERROR("Failed to create CameraProvider"); + std::printf("Argus Version: %s\n", iProvider->getVersion().c_str()); + + CaptureManager capture(config); + if (!capture.initialize(provider.get(), display.get())) + return false; + std::vector streamFps; + for (size_t i = 0; i < config.streams.size(); ++i) + { + const uint32_t fps = config.streams[i].output.fps ? config.streams[i].output.fps : capture.nominalFps(i); + if (fps == 0) + ORIGINATE_ERROR("Stream %s resolved to zero FPS", config.streams[i].id.c_str()); + streamFps.push_back(fps); + } + if (config.streams[0].output.fps == 0) + { + for (size_t i = 1; i < streamFps.size(); ++i) + { + if (streamFps[i] != streamFps[0]) + { + ORIGINATE_ERROR( + "Native-rate streams resolved to incompatible FPS values %u and %u; configure explicit FPS", + streamFps[0], streamFps[i]); + } + } + } + MixedRateScheduler scheduler(streamFps); + const size_t baseStreamIndex = selectPriorityStream(config, streamFps); + if (config.streams[baseStreamIndex].capture.mode != CAPTURE_MODE_MAILBOX) + { + std::fprintf( + stderr, + "Configuration error: priority stream %s is configured for %s capture; EGL priority capture requires mailbox mode, while direct-buffer HEVC priority capture has no FIFO mode\n", + config.streams[baseStreamIndex].id.c_str(), captureModeName(config.streams[baseStreamIndex].capture.mode)); + return false; + } + const size_t baseDomainIndex = capture.streamDomain(baseStreamIndex); + const uint64_t baseFramePeriodNs = capture.framePeriodNs(baseStreamIndex); + bool hasSecondaryMonoStream = false; + for (size_t i = 0; i < config.streams.size(); ++i) + { + if (i != baseStreamIndex && config.streams[i].type == STREAM_TYPE_MONO) + { + hasSecondaryMonoStream = true; + break; + } + } + const uint64_t secondaryReleaseDelayNs = + hasSecondaryMonoStream ? 0 : secondaryCaptureReleaseDelayNs(baseFramePeriodNs); + const uint32_t baseRateLimit = config.streams[baseStreamIndex].output.fps == 0 ? 0 : scheduler.baseFps(); + StereoOutputRateLimiter baseRateLimiter(baseRateLimit); + + std::printf("Configured AVS1 sender: %zu logical stream(s), base cadence %u FPS, priority=%s/domain%zu, logging=%s\n", + config.streams.size(), scheduler.baseFps(), config.streams[baseStreamIndex].id.c_str(), baseDomainIndex, + verboseLoggingEnabled() ? "verbose" : "concise"); + double estimatedRawGbps = 0.0; + for (size_t i = 0; i < config.streams.size(); ++i) + { + const StreamConfig& stream = config.streams[i]; + const ResolvedOutputGeometry& geometry = capture.geometry(i); + if (verboseLoggingEnabled()) + { + std::printf(" [%zu] %s: %s, %zu camera(s), capture=%s", i, stream.id.c_str(), streamTypeName(stream.type), + stream.cameras.size(), + stream.output.codec == CODEC_HEVC ? "direct-buffer" : captureModeName(stream.capture.mode)); + if (stream.output.codec != CODEC_HEVC && stream.capture.mode == CAPTURE_MODE_FIFO) + std::printf(":%u", stream.capture.fifoLength); + std::printf(", %s %ux%u @ %u FPS, resize=%s, receiver_rotation=%u", codecTypeName(stream.output.codec), + geometry.width, geometry.height, streamFps[i], resizeModeName(stream.output.resizeMode), + stream.output.rotation); + if (stream.output.codec == CODEC_HEVC) + std::printf(", bitrate=%u Mbps per camera", stream.output.bitrateMbps); + if (stream.output.remap.enabled()) + std::printf(", remap=%s (capture %ux%u)", stream.output.remap.file.c_str(), geometry.captureWidth, + geometry.captureHeight); + std::printf("\n"); + } + if (stream.output.codec == CODEC_RAW_NV12) + { + estimatedRawGbps += static_cast(geometry.width) * geometry.height * 1.5 * streamFps[i] * + stream.cameras.size() * 8.0 / 1.0e9; + } + } + if (estimatedRawGbps > 0.9) + { + std::fprintf(stderr, "Warning: configured raw NV12 payload rate is approximately %.2f Gbps before TCP overhead\n", + estimatedRawGbps); + } + if (verboseLoggingEnabled()) + { + std::printf("Listening for AVS1 receiver on %s:%u\n", config.network.bindHost.c_str(), config.network.port); + } + int fd = acceptTcpClient(config.network.bindHost, config.network.port, config.network.reconnectDelayMs); + if (fd < 0) + return false; + + TcpPacketSenderSettings transportSettings; + transportSettings.port = config.network.port; + transportSettings.dumpWirePath = config.network.dumpWirePath; + for (size_t i = 0; i < config.streams.size(); ++i) + { + transportSettings.streamLabels.push_back(config.streams[i].id); + for (size_t camera = 0; camera < config.streams[i].cameras.size(); ++camera) + { + if (config.streams[i].type == STREAM_TYPE_STEREO) + { + transportSettings.sampleLabels.push_back(config.streams[i].id + (camera == 0 ? ".left" : ".right")); + } + else + { + transportSettings.sampleLabels.push_back(config.streams[i].id); + } + } + } + transportSettings.kpi = config.runtime.kpi; + transportSettings.logIntervalSeconds = config.runtime.logIntervalSeconds; + TcpPacketSender tcpSender(&fd, transportSettings, kTcpQueueDepth, config.network.port, "avs1", appendDump); + if (!tcpSender.start()) + { + close(fd); + return false; + } + + const std::vector templates = buildStreamTemplates(config, capture, streamFps); + FrameBundleAssembler assembler(&tcpSender, createSessionId(), templates); + std::vector> processors; + for (size_t i = 0; i < config.streams.size(); ++i) + { + std::unique_ptr processor( + new StreamProcessor(config.streams[i], capture.geometry(i), i, config.runtime.kpi, + config.runtime.logIntervalSeconds, config.runtime.benchEncode)); + if (!processor->start(&assembler, streamFps[i])) + { + tcpSender.shutdown(false); + close(fd); + return false; + } + processors.push_back(std::move(processor)); + } + + bool ok = capture.startPriorityDomain(baseDomainIndex); + if (ok) + { + ok = capture.discardDomainWarmup(baseDomainIndex, config.alignment.warmupFrames); + } + if (ok) + { + ok = capture.waitForDomainSynchronization(baseDomainIndex, kPrioritySyncAttempts, kPrioritySyncConsecutiveFrames, + prioritySyncToleranceNs(baseFramePeriodNs)); + } + if (ok) + ok = capture.startSecondaryDomains(baseDomainIndex); + if (ok && capture.domainCount() > 1) + ok = capture.discardWarmup(config.alignment.warmupFrames); + if (ok && capture.domainCount() > 1) + { + ok = capture.waitForDomainSynchronization(baseDomainIndex, kPrioritySyncAttempts, kPrioritySyncConsecutiveFrames, + prioritySyncToleranceNs(baseFramePeriodNs)); + } + const bool continuousFifoSelection = ok && capture.hasContinuousFifoDomains(); + if (continuousFifoSelection && verboseLoggingEnabled()) + { + std::printf( + "argus_sender[alignment]: periodic reacquisition disabled while FIFO domains are active; alignment.interval_frames=%u is inactive and secondary FIFO domains select the closest candidate to every priority SOF TSC\n", + config.alignment.intervalFrames); + } + + uint64_t bundleSequence = 0; + std::vector streamSequences(config.streams.size(), 0); + std::vector intervalStreamFrames(config.streams.size(), 0); + uint64_t intervalBundles = 0; + uint64_t intervalBaseRateDrops = 0; + uint64_t totalBaseRateDrops = 0; + uint64_t intervalGlobalTscSamples = 0; + double intervalGlobalTscSpanSumNs = 0.0; + uint64_t intervalGlobalTscSpanMaxNs = 0; + bool alignmentTimestampUnavailableWarned = false; + std::chrono::steady_clock::time_point statsStart = std::chrono::steady_clock::now(); + while (ok && !stopRequested() && !tcpSender.hasError() && !assembler.hasError() && !capture.hasError() && + (config.runtime.frames == 0 || bundleSequence < config.runtime.frames)) + { + std::vector captured; + PendingCopyScope pendingCopyScope(&processors); + capture.prepareCaptured(&captured); + std::vector bundleInputs(config.streams.size()); + if (!capture.acquireDomain(baseDomainIndex, &captured)) + { + ok = false; + break; + } + uint64_t priorityReferenceSofTscNs = 0; + uint64_t priorityTscSpanNs = 0; + if (!frameSetReferenceSofTscNs(captured[baseStreamIndex], prioritySyncToleranceNs(baseFramePeriodNs), + &priorityReferenceSofTscNs, &priorityTscSpanNs)) + { + std::fprintf(stderr, + "Priority stream %s is not synchronized at runtime: tsc_span_us=%.3f, tolerance_us=%.3f\n", + config.streams[baseStreamIndex].id.c_str(), static_cast(priorityTscSpanNs) / 1000.0, + static_cast(prioritySyncToleranceNs(baseFramePeriodNs)) / 1000.0); + ok = false; + break; + } + if (!baseRateLimiter.shouldOutput(captured[baseStreamIndex].timings[0])) + { + ++intervalBaseRateDrops; + ++totalBaseRateDrops; + if (!releaseSecondaryCaptures(&capture, baseDomainIndex, 0)) + { + ok = false; + break; + } + for (size_t domainIndex = 0; domainIndex < capture.domainCount(); ++domainIndex) + { + if (domainIndex != baseDomainIndex && !capture.domainUsesContinuousFifo(domainIndex) && + !capture.acquireDomain(domainIndex, &captured)) + { + ok = false; + break; + } + } + if (!ok) + break; + for (size_t i = 0; i < config.streams.size(); ++i) + capture.recordCaptured(i, captured[i]); + continue; + } + + const std::vector due = scheduler.next(); + if (!due[baseStreamIndex]) + { + std::fprintf(stderr, "Internal error: AVS1 base stream is not due\n"); + ok = false; + break; + } + for (size_t i = 0; i < config.streams.size(); ++i) + { + if (!due[i]) + continue; + if (!processors[i]->waitForInputBuffers()) + { + ok = false; + break; + } + ++streamSequences[i]; + bundleInputs[i].present = true; + bundleInputs[i].sequence = streamSequences[i]; + } + if (!ok) + break; + + const uint64_t nextBundleSequence = bundleSequence + 1; + const bool alignmentCycle = config.alignment.intervalFrames > 0 && !continuousFifoSelection && + nextBundleSequence % config.alignment.intervalFrames == 0; + std::vector copiesSubmitted(config.streams.size(), false); + bool priorityCopyStarted = false; + + if (!alignmentCycle) + { + for (size_t i = 0; i < config.streams.size(); ++i) + { + if (due[i] && config.streams[i].output.codec == CODEC_HEVC && capture.streamDomain(i) == baseDomainIndex) + { + if (!processors[i]->submitCopies(&captured[i], nextBundleSequence)) + { + ok = false; + break; + } + copiesSubmitted[i] = true; + priorityCopyStarted = true; + } + } + } + if (ok && !releaseSecondaryCaptures(&capture, baseDomainIndex, priorityCopyStarted ? secondaryReleaseDelayNs : 0)) + { + ok = false; + } + + for (size_t domainIndex = 0; ok && domainIndex < capture.domainCount(); ++domainIndex) + { + if (domainIndex == baseDomainIndex) + continue; + const bool acquired = capture.domainUsesContinuousFifo(domainIndex) ? + capture.acquireDomainClosestTo(domainIndex, priorityReferenceSofTscNs, &captured) : + capture.acquireDomain(domainIndex, &captured); + if (!acquired) + { + ok = false; + break; + } + if (!alignmentCycle) + { + for (size_t i = 0; i < config.streams.size(); ++i) + { + if (due[i] && !copiesSubmitted[i] && config.streams[i].output.codec == CODEC_HEVC && + capture.streamDomain(i) == domainIndex) + { + if (!processors[i]->submitCopies(&captured[i], nextBundleSequence)) + { + ok = false; + break; + } + copiesSubmitted[i] = true; + } + } + } + } + if (!ok) + break; + + if (alignmentCycle && !alignCapturedFrames(&capture, &captured, false, &alignmentTimestampUnavailableWarned)) + { + ok = false; + break; + } + for (size_t i = 0; i < config.streams.size(); ++i) + { + capture.recordCaptured(i, captured[i]); + if (due[i]) + bundleInputs[i].timings = captured[i].timings; + } + uint64_t globalTscSpanNs = 0; + if (captureTscSpanNs(captured, &globalTscSpanNs)) + { + ++intervalGlobalTscSamples; + intervalGlobalTscSpanSumNs += static_cast(globalTscSpanNs); + intervalGlobalTscSpanMaxNs = std::max(intervalGlobalTscSpanMaxNs, globalTscSpanNs); + } + + uint32_t captureClockDomain = SENDER_CLOCK_UNKNOWN; + const uint64_t senderCaptureReadyNs = readSenderClockNs(&captureClockDomain); + // Same instant as senderCaptureReadyNs so receivers can place each + // sample's VI SOF TSC on the sender wall clock. + const uint64_t tscNowNs = readTscNowNs(); + const uint64_t senderCaptureReadyTscNs = validatedCaptureReadyTscNs(tscNowNs, bundleInputs); + uint32_t submitClockDomain = SENDER_CLOCK_UNKNOWN; + const uint64_t senderProcessSubmitNs = readSenderClockNs(&submitClockDomain); + if (captureClockDomain != submitClockDomain) + captureClockDomain = SENDER_CLOCK_UNKNOWN; + if (!assembler.beginBundle(nextBundleSequence, bundleInputs, senderCaptureReadyNs, senderProcessSubmitNs, + captureClockDomain, senderCaptureReadyTscNs)) + { + ok = false; + break; + } + // Submit every due stream's copies before waiting on any of them: the + // raw streams' whole pipeline (VIC copy + remap/pack) runs on their + // per-camera worker threads, so starting them here overlaps that work + // with the head VIC copies and NVENC queueing instead of serializing + // the two stages inside one bundle tick. + for (size_t i = 0; i < config.streams.size(); ++i) + { + if (due[i] && config.streams[i].output.codec == CODEC_HEVC && !copiesSubmitted[i] && + !processors[i]->submitCopies(&captured[i], nextBundleSequence)) + { + ok = false; + break; + } + } + for (size_t i = 0; ok && i < config.streams.size(); ++i) + { + if (due[i] && config.streams[i].output.codec == CODEC_RAW_NV12 && + !processors[i]->submitCopies(&captured[i], nextBundleSequence)) + { + ok = false; + } + } + for (size_t i = 0; ok && i < config.streams.size(); ++i) + { + if (due[i] && config.streams[i].output.codec == CODEC_HEVC && !processors[i]->finishSubmit()) + ok = false; + } + for (size_t i = 0; ok && i < config.streams.size(); ++i) + { + if (due[i] && config.streams[i].output.codec == CODEC_RAW_NV12 && !processors[i]->finishSubmit()) + { + ok = false; + } + } + if (!ok) + break; + bundleSequence = nextBundleSequence; + ++intervalBundles; + for (size_t i = 0; i < due.size(); ++i) + intervalStreamFrames[i] += due[i] ? 1 : 0; + + if (config.runtime.kpi && config.runtime.logIntervalSeconds > 0) + { + const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); + const double elapsed = std::chrono::duration(now - statsStart).count(); + if (elapsed >= static_cast(config.runtime.logIntervalSeconds)) + { + std::printf("argus_sender[coordinator]: %.2f bundle FPS, total=%llu, streams=", + static_cast(intervalBundles) / elapsed, + static_cast(bundleSequence)); + for (size_t i = 0; i < intervalStreamFrames.size(); ++i) + { + std::printf("%s%s:%.2f", i == 0 ? "" : ",", config.streams[i].id.c_str(), + static_cast(intervalStreamFrames[i]) / elapsed); + intervalStreamFrames[i] = 0; + } + if (intervalGlobalTscSamples > 0) + { + std::printf(", global_tsc_span_us(avg/max)=%.3f/%.3f", + intervalGlobalTscSpanSumNs / static_cast(intervalGlobalTscSamples) / 1000.0, + static_cast(intervalGlobalTscSpanMaxNs) / 1000.0); + } + else + { + std::printf(", global_tsc_span_us=unavailable"); + } + std::printf(", base_rate_drops(interval/total)=%llu/%llu\n", + static_cast(intervalBaseRateDrops), + static_cast(totalBaseRateDrops)); + capture.logFifoCandidateStats(); + intervalBundles = 0; + intervalBaseRateDrops = 0; + intervalGlobalTscSamples = 0; + intervalGlobalTscSpanSumNs = 0.0; + intervalGlobalTscSpanMaxNs = 0; + statsStart = now; + } + } + } + + if (!capture.stop()) + ok = false; + const bool disconnected = tcpSender.disconnected() && !stopRequested(); + if (!disconnected) + { + for (size_t i = 0; i < processors.size(); ++i) + { + if (!processors[i]->drain()) + ok = false; + } + for (size_t i = 0; i < processors.size(); ++i) + processors[i]->sendEos(); + } + bool processorError = false; + for (size_t i = 0; i < processors.size(); ++i) + { + processorError = processorError || processors[i]->hasError(); + processors[i]->shutdown(); + } + tcpSender.shutdown(ok && !processorError && !stopRequested()); + if (fd >= 0) + close(fd); + processors.clear(); + capture.releaseResources(); + provider.reset(); + display.cleanup(); + if (receiverDisconnected) + *receiverDisconnected = disconnected; + return disconnected || (ok && !processorError && !assembler.hasError()); +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/CaptureCoordinator.h b/examples/camera_streamer/argus_sender/src/CaptureCoordinator.h new file mode 100644 index 0000000000..bbc3b291c5 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/CaptureCoordinator.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_CAPTURE_COORDINATOR_H +#define ARGUS_SENDER_CAPTURE_COORDINATOR_H + +#include + +namespace argus_sender +{ + +struct AppConfig; + +bool runConfiguredCapture(const AppConfig& config, bool* receiverDisconnected, bool appendDump); +bool printCameraModes(); + +} // namespace argus_sender + +#endif // ARGUS_SENDER_CAPTURE_COORDINATOR_H diff --git a/examples/camera_streamer/argus_sender/src/CaptureManager.cpp b/examples/camera_streamer/argus_sender/src/CaptureManager.cpp new file mode 100644 index 0000000000..54754161b9 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/CaptureManager.cpp @@ -0,0 +1,2310 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "CaptureManager.h" + +#include "CameraCatalog.h" +#include "CaptureDomainPlan.h" +#include "CapturePriorityPolicy.h" +#include "Error.h" +#include "FrameAlignmentPolicy.h" +#include "HevcEncoder.h" +#include "PriorityFrameHandoff.h" +#include "Runtime.h" +#include "StereoAlignmentPolicy.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace argus_sender +{ + +CapturedDmaBuffer::CapturedDmaBuffer() : m_buffer(NULL) +{ +} + +CapturedDmaBuffer::~CapturedDmaBuffer() +{ + reset(); +} + +CapturedDmaBuffer::CapturedDmaBuffer(CapturedDmaBuffer&& other) noexcept : m_buffer(other.release()) +{ +} + +CapturedDmaBuffer& CapturedDmaBuffer::operator=(CapturedDmaBuffer&& other) noexcept +{ + if (this != &other) + { + reset(); + m_buffer = other.release(); + } + return *this; +} + +DmaBuffer* CapturedDmaBuffer::get() const +{ + return m_buffer; +} + +DmaBuffer* CapturedDmaBuffer::release() +{ + DmaBuffer* buffer = m_buffer; + m_buffer = NULL; + return buffer; +} + +void CapturedDmaBuffer::reset(DmaBuffer* buffer) +{ + if (m_buffer && m_buffer != buffer && !m_buffer->releaseToArgus()) + { + std::fprintf(stderr, "Failed to release unused direct-capture DMABUF to Argus\n"); + } + m_buffer = buffer; +} + +namespace +{ + +const uint64_t kCaptureWaitTimeoutNs = 100000000ULL; +const uint64_t kIdleWaitTimeoutNs = 5000000000ULL; +const uint32_t kWaitPollMs = 100; +const size_t kFifoCandidateWindowLength = 2; +// Use the same direct-output pool size for head and wrist cameras. Continuous +// acquisition and latest-frame selection keep excess filled buffers drained. +const uint32_t kDirectCaptureBufferCount = 10; + +uint64_t steadyNowNs() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +int64_t deltaNs(uint64_t right, uint64_t left) +{ + if (right >= left) + { + const uint64_t value = right - left; + return value > static_cast(std::numeric_limits::max()) ? std::numeric_limits::max() : + static_cast(value); + } + const uint64_t value = left - right; + return value > static_cast(std::numeric_limits::max()) ? std::numeric_limits::min() : + -static_cast(value); +} + +uint64_t absoluteDeltaNs(int64_t value) +{ + if (value >= 0) + return static_cast(value); + return static_cast(-(value + 1)) + 1ULL; +} + +void updateAtomicMaximum(std::atomic* target, uint64_t value) +{ + uint64_t current = target->load(); + while (current < value && !target->compare_exchange_weak(current, value)) + { + } +} + +} // namespace + +class CaptureManager::Impl +{ +public: + struct StreamState + { + StreamConfig config; + ResolvedOutputGeometry geometry; + std::vector cameraIndices; + uint32_t nominalFps = 0; + uint64_t framePeriodNs = 0; + std::chrono::steady_clock::time_point intervalStart; + uint64_t intervalFrames = 0; + uint64_t intervalTscSamples = 0; + double intervalTscSumNs = 0.0; + int64_t intervalTscMinNs = 0; + int64_t intervalTscMaxNs = 0; + uint64_t totalFrames = 0; + std::vector acquireSamples; + std::vector acquireSumNs; + std::vector acquireMaxNs; + std::vector lastSelectedFrameNumbers; + std::vector selectedFrameGapsInterval; + std::vector hasLastSelectedFrameNumber; + }; + + struct CameraState + { + size_t streamIndex = 0; + size_t cameraIndex = 0; + size_t domainIndex = 0; + CameraConfig config; + std::string label; + Argus::CameraDevice* device = NULL; + Argus::SensorMode* sensorMode = NULL; + Argus::ISensorMode* iSensorMode = NULL; + Argus::UniqueObj outputStream; + std::vector> dmaBuffers; + std::vector> argusBuffers; + Argus::IBufferOutputStream* iBufferStream = NULL; + Argus::UniqueObj consumer; + EGLStream::IFrameConsumer* iConsumer = NULL; + uint64_t lastFrameNumber = 0; + bool hasLastFrameNumber = false; + }; + + struct PendingFrame + { + explicit PendingFrame(size_t index) : cameraFlatIndex(index) + { + } + + size_t cameraFlatIndex; + Argus::UniqueObj frame; + CapturedDmaBuffer dmaBuffer; + FrameTiming timing; + bool duplicate = false; + }; + + struct CaptureSessionState + { + std::vector cameraIndices; + Argus::UniqueObj session; + Argus::ICaptureSession* iSession = NULL; + Argus::IEventProvider* iEventProvider = NULL; + Argus::UniqueObj eventQueue; + Argus::IEventQueue* iEventQueue = NULL; + Argus::UniqueObj request; + uint64_t captureCompleteCredits = 0; + bool repeating = false; + }; + + struct DomainState + { + size_t index = 0; + std::vector cameraIndices; + std::vector> sessions; + std::thread worker; + std::thread eventWorker; + std::array cameraWorkers; + std::array, 2> headFrames; + mutable std::mutex mutex; + std::condition_variable slotAvailable; + std::condition_variable framesAvailable; + std::vector> pending; + std::deque>> candidates; + bool hasPending = false; + bool stop = false; + bool error = false; + bool demandDriven = false; + bool fifoMode = false; + uint32_t fifoLength = 0; + bool continuousFifo = false; + bool captureRequested = false; + bool selectionTimestampUnavailableWarned = false; + bool priorityStereo = false; + size_t priorityStereoStreamIndex = std::numeric_limits::max(); + size_t stereoLeftFlatIndex = std::numeric_limits::max(); + size_t stereoRightFlatIndex = std::numeric_limits::max(); + std::atomic stoppingRepeat{ false }; + std::atomic candidateOverflowDiscardsInterval{ 0 }; + std::atomic candidateOverflowDiscardsTotal{ 0 }; + std::atomic candidateSelectionDiscardsInterval{ 0 }; + std::atomic candidateSelectionDiscardsTotal{ 0 }; + std::atomic candidateSelectionsInterval{ 0 }; + std::atomic candidateSelectionsTotal{ 0 }; + std::atomic candidateDeltaSumNsInterval{ 0 }; + std::atomic candidateDeltaMaxNsInterval{ 0 }; + std::atomic candidatePastSelectionsInterval{ 0 }; + std::atomic candidateFutureSelectionsInterval{ 0 }; + std::atomic candidateDepthPeakInterval{ 0 }; + std::atomic fifoEventBacklog{ 0 }; + std::atomic fifoEventBacklogPeakInterval{ 0 }; + std::atomic stereoPairsInterval{ 0 }; + std::atomic stereoPairsTotal{ 0 }; + std::atomic stereoRepairPairsInterval{ 0 }; + std::atomic stereoRepairPairsTotal{ 0 }; + std::atomic stereoLeftDropsInterval{ 0 }; + std::atomic stereoLeftDropsTotal{ 0 }; + std::atomic stereoRightDropsInterval{ 0 }; + std::atomic stereoRightDropsTotal{ 0 }; + std::atomic stereoReacquiresInterval{ 0 }; + std::atomic stereoReacquiresTotal{ 0 }; + std::atomic stereoReacquireWaitSumNsInterval{ 0 }; + std::atomic stereoReacquireWaitMaxNsInterval{ 0 }; + std::atomic stereoSelectedAgeSumNsInterval{ 0 }; + std::atomic stereoSelectedAgeMaxNsInterval{ 0 }; + std::atomic stereoInitialDeltaSumNsInterval{ 0 }; + std::atomic stereoInitialDeltaMaxNsInterval{ 0 }; + std::atomic stereoFinalDeltaSumNsInterval{ 0 }; + std::atomic stereoFinalDeltaMaxNsInterval{ 0 }; + std::atomic headSlotOverwritesInterval{ 0 }; + std::atomic headSlotOverwritesTotal{ 0 }; + }; + + explicit Impl(const AppConfig& config) : m_config(config), m_started(false), m_error(false) + { + } + + ~Impl() + { + stop(); + cleanup(); + } + + bool initialize(Argus::CameraProvider* provider, EGLDisplay display) + { + if (!provider || display == EGL_NO_DISPLAY) + ORIGINATE_ERROR("Capture manager got invalid Argus/EGL context"); + + Argus::ICameraProvider* iProvider = Argus::interface_cast(provider); + if (!iProvider) + ORIGINATE_ERROR("Failed to get CameraProvider interface"); + + std::vector endpoints; + m_streams.resize(m_config.streams.size()); + m_cameraLookup.resize(m_config.streams.size()); + for (size_t streamIndex = 0; streamIndex < m_config.streams.size(); ++streamIndex) + { + m_streams[streamIndex].reset(new StreamState()); + StreamState& stream = *m_streams[streamIndex]; + stream.config = m_config.streams[streamIndex]; + stream.intervalStart = std::chrono::steady_clock::now(); + m_cameraLookup[streamIndex].resize(stream.config.cameras.size()); + std::vector sourceSizes; + for (size_t cameraIndex = 0; cameraIndex < stream.config.cameras.size(); ++cameraIndex) + { + std::unique_ptr camera(new CameraState()); + camera->streamIndex = streamIndex; + camera->cameraIndex = cameraIndex; + camera->config = stream.config.cameras[cameraIndex]; + std::ostringstream label; + label << stream.config.id << "."; + if (stream.config.type == STREAM_TYPE_STEREO) + label << (cameraIndex == 0 ? "left" : "right"); + else + label << "mono"; + camera->label = label.str(); + camera->device = selectCameraDevice(provider, camera->config.device); + if (!camera->device) + { + ORIGINATE_ERROR( + "Stream %s selected unavailable device %u", stream.config.id.c_str(), camera->config.device); + } + camera->sensorMode = selectSensorMode(camera->device, camera->config.mode); + camera->iSensorMode = Argus::interface_cast(camera->sensorMode); + if (!camera->iSensorMode) + { + ORIGINATE_ERROR("Stream %s selected unavailable mode %u on d%u", stream.config.id.c_str(), + camera->config.mode, camera->config.device); + } + const Argus::Size2D resolution = camera->iSensorMode->getResolution(); + const uint64_t period = camera->iSensorMode->getFrameDurationRange().min(); + if (period == 0) + { + ORIGINATE_ERROR("Stream %s sensor mode has zero frame duration", stream.config.id.c_str()); + } + + ImageSize sourceSize; + sourceSize.width = resolution.width(); + sourceSize.height = resolution.height(); + sourceSizes.push_back(sourceSize); + stream.framePeriodNs = std::max(stream.framePeriodNs, period); + + CaptureEndpointDescriptor endpoint; + endpoint.streamIndex = streamIndex; + endpoint.cameraIndex = cameraIndex; + endpoint.streamType = stream.config.type; + endpoint.modeIndex = camera->config.mode; + endpoint.sourceWidth = resolution.width(); + endpoint.sourceHeight = resolution.height(); + endpoint.framePeriodNs = period; + endpoints.push_back(endpoint); + + const size_t flatIndex = m_cameras.size(); + m_cameraLookup[streamIndex][cameraIndex] = flatIndex; + stream.cameraIndices.push_back(flatIndex); + m_cameras.push_back(std::move(camera)); + } + + std::string geometryError; + if (!resolveOutputGeometry(stream.config.output, sourceSizes, &stream.geometry, &geometryError)) + { + ORIGINATE_ERROR("Stream %s geometry error: %s", stream.config.id.c_str(), geometryError.c_str()); + } + stream.nominalFps = roundedFrameRateFromDurationNs(stream.framePeriodNs); + stream.acquireSamples.resize(stream.cameraIndices.size(), 0); + stream.lastSelectedFrameNumbers.resize(stream.cameraIndices.size(), 0); + stream.selectedFrameGapsInterval.resize(stream.cameraIndices.size(), 0); + stream.hasLastSelectedFrameNumber.resize(stream.cameraIndices.size(), false); + stream.acquireSumNs.resize(stream.cameraIndices.size(), 0.0); + stream.acquireMaxNs.resize(stream.cameraIndices.size(), 0); + } + + CaptureDomainPlan plan; + std::string planError; + if (!buildCaptureDomainPlan(endpoints, &plan, &planError)) + ORIGINATE_ERROR("Capture-domain plan failed: %s", planError.c_str()); + + std::vector streamFps; + for (size_t streamIndex = 0; streamIndex < m_streams.size(); ++streamIndex) + { + streamFps.push_back(m_config.streams[streamIndex].output.fps ? m_config.streams[streamIndex].output.fps : + m_streams[streamIndex]->nominalFps); + } + const size_t priorityStreamIndex = selectPriorityStream(m_config, streamFps); + + m_domains.clear(); + m_domains.reserve(plan.domains.size()); + for (size_t domainIndex = 0; domainIndex < plan.domains.size(); ++domainIndex) + { + m_domains.emplace_back(new DomainState()); + DomainState& domain = *m_domains.back(); + domain.index = domainIndex; + for (size_t member = 0; member < plan.domains[domainIndex].size(); ++member) + { + const size_t cameraFlatIndex = plan.domains[domainIndex][member]; + CameraState& camera = *m_cameras[cameraFlatIndex]; + camera.domainIndex = domainIndex; + domain.cameraIndices.push_back(cameraFlatIndex); + } + + std::vector> sessionCameraGroups; + const bool independentPriorityStereo = + priorityStreamIndex < m_streams.size() && + m_streams[priorityStreamIndex]->config.type == STREAM_TYPE_STEREO && domain.cameraIndices.size() == 2 && + m_cameras[domain.cameraIndices[0]]->streamIndex == priorityStreamIndex && + m_cameras[domain.cameraIndices[1]]->streamIndex == priorityStreamIndex; + if (independentPriorityStereo) + { + for (size_t member = 0; member < domain.cameraIndices.size(); ++member) + { + sessionCameraGroups.push_back(std::vector(1, domain.cameraIndices[member])); + } + } + else + { + sessionCameraGroups.push_back(domain.cameraIndices); + } + + for (size_t sessionIndex = 0; sessionIndex < sessionCameraGroups.size(); ++sessionIndex) + { + std::unique_ptr captureSession(new CaptureSessionState()); + captureSession->cameraIndices = sessionCameraGroups[sessionIndex]; + std::vector devices; + for (size_t member = 0; member < captureSession->cameraIndices.size(); ++member) + { + devices.push_back(m_cameras[captureSession->cameraIndices[member]]->device); + } + + Argus::Status sessionStatus = Argus::STATUS_OK; + captureSession->session.reset(iProvider->createCaptureSession(devices, &sessionStatus)); + captureSession->iSession = Argus::interface_cast(captureSession->session); + captureSession->iEventProvider = Argus::interface_cast(captureSession->session); + if (!captureSession->iSession || !captureSession->iEventProvider) + { + ORIGINATE_ERROR("Failed to create capture domain %zu session %zu (status %d)", domainIndex, + sessionIndex, sessionStatus); + } + + std::vector eventTypes; + if (captureSession->iEventProvider->getAvailableEventTypes(&eventTypes) != Argus::STATUS_OK || + eventTypes.empty()) + { + ORIGINATE_ERROR( + "Failed to enumerate events for capture domain %zu session %zu", domainIndex, sessionIndex); + } + captureSession->eventQueue.reset(captureSession->iEventProvider->createEventQueue(eventTypes)); + captureSession->iEventQueue = Argus::interface_cast(captureSession->eventQueue); + if (!captureSession->eventQueue || !captureSession->iEventQueue) + { + ORIGINATE_ERROR( + "Failed to create event queue for capture domain %zu session %zu", domainIndex, sessionIndex); + } + + for (size_t member = 0; member < captureSession->cameraIndices.size(); ++member) + { + CameraState& camera = *m_cameras[captureSession->cameraIndices[member]]; + const StreamState& stream = *m_streams[camera.streamIndex]; + const bool directHevc = stream.config.output.codec == CODEC_HEVC; + Argus::UniqueObj settings( + captureSession->iSession->createOutputStreamSettings(directHevc ? Argus::STREAM_TYPE_BUFFER : + Argus::STREAM_TYPE_EGL)); + Argus::IOutputStreamSettings* iSettings = + Argus::interface_cast(settings); + if (!iSettings) + { + ORIGINATE_ERROR("Failed to create output settings for %s", camera.label.c_str()); + } + iSettings->setCameraDevice(camera.device); + const bool fifoMode = stream.config.capture.mode == CAPTURE_MODE_FIFO; + if (domain.sessions.empty() && member == 0) + { + domain.fifoMode = fifoMode; + domain.fifoLength = fifoMode ? stream.config.capture.fifoLength : 0; + } + else if (domain.fifoMode != fifoMode || + (fifoMode && domain.fifoLength != stream.config.capture.fifoLength)) + { + ORIGINATE_ERROR("Capture domain %zu mixes incompatible EGL stream modes", domainIndex); + } + if (directHevc) + { + Argus::IBufferOutputStreamSettings* iBufferSettings = + Argus::interface_cast(settings); + if (!iBufferSettings || + iBufferSettings->setBufferType(Argus::BUFFER_TYPE_EGL_IMAGE) != Argus::STATUS_OK) + { + ORIGINATE_ERROR("Failed to configure direct DMABUF capture for %s", camera.label.c_str()); + } + iBufferSettings->setMetadataEnable(true); + } + else + { + Argus::IEGLOutputStreamSettings* iEglSettings = + Argus::interface_cast(settings); + if (!iEglSettings) + { + ORIGINATE_ERROR("Failed to create EGL settings for %s", camera.label.c_str()); + } + iEglSettings->setPixelFormat(Argus::PIXEL_FMT_YCbCr_420_888); + iEglSettings->setResolution( + Argus::Size2D(stream.geometry.captureWidth, stream.geometry.captureHeight)); + iEglSettings->setEGLDisplay(display); + if (iEglSettings->setMode(fifoMode ? Argus::EGL_STREAM_MODE_FIFO : + Argus::EGL_STREAM_MODE_MAILBOX) != Argus::STATUS_OK) + { + ORIGINATE_ERROR( + "Failed to set %s mode for %s", fifoMode ? "FIFO" : "mailbox", camera.label.c_str()); + } + if (fifoMode && (domain.fifoLength == 0 || + iEglSettings->setFifoLength(domain.fifoLength) != Argus::STATUS_OK)) + { + ORIGINATE_ERROR( + "Failed to set FIFO length %u for %s", domain.fifoLength, camera.label.c_str()); + } + iEglSettings->setMetadataEnable(true); + } + camera.outputStream.reset(captureSession->iSession->createOutputStream(settings.get())); + if (!camera.outputStream) + ORIGINATE_ERROR("Failed to create output stream %s", camera.label.c_str()); + if (directHevc) + { + camera.iBufferStream = Argus::interface_cast(camera.outputStream); + if (!camera.iBufferStream) + { + ORIGINATE_ERROR("Failed to create direct buffer stream %s", camera.label.c_str()); + } + Argus::UniqueObj bufferSettings( + camera.iBufferStream->createBufferSettings()); + Argus::IEGLImageBufferSettings* iEglImageSettings = + Argus::interface_cast(bufferSettings); + if (!bufferSettings || !iEglImageSettings || + iEglImageSettings->setEGLDisplay(display) != Argus::STATUS_OK) + { + ORIGINATE_ERROR("Failed to create direct buffer settings for %s", camera.label.c_str()); + } + camera.dmaBuffers.reserve(kDirectCaptureBufferCount); + camera.argusBuffers.resize(kDirectCaptureBufferCount); + const Argus::Size2D directSize(stream.geometry.width, stream.geometry.height); + for (uint32_t bufferIndex = 0; bufferIndex < kDirectCaptureBufferCount; ++bufferIndex) + { + std::unique_ptr dmaBuffer( + DmaBuffer::create(directSize, NVBUF_COLOR_FORMAT_NV12, NVBUF_LAYOUT_BLOCK_LINEAR)); + if (!dmaBuffer || !dmaBuffer->mapEglImage() || + iEglImageSettings->setEGLImage(dmaBuffer->getEglImage()) != Argus::STATUS_OK) + { + ORIGINATE_ERROR("Failed to allocate direct capture DMABUF %u for %s", bufferIndex, + camera.label.c_str()); + } + camera.argusBuffers[bufferIndex].reset( + camera.iBufferStream->createBuffer(bufferSettings.get())); + Argus::IBuffer* iBuffer = + Argus::interface_cast(camera.argusBuffers[bufferIndex]); + if (!iBuffer) + { + ORIGINATE_ERROR("Failed to create Argus direct buffer %u for %s", bufferIndex, + camera.label.c_str()); + } + iBuffer->setClientData(dmaBuffer.get()); + if (!dmaBuffer->bindArgusBuffer(camera.iBufferStream, camera.argusBuffers[bufferIndex].get()) || + !dmaBuffer->releaseToArgus()) + { + ORIGINATE_ERROR("Failed to release direct buffer %u for initial %s capture", + bufferIndex, camera.label.c_str()); + } + camera.dmaBuffers.push_back(std::move(dmaBuffer)); + } + } + else + { + camera.consumer.reset(EGLStream::FrameConsumer::create(camera.outputStream.get())); + camera.iConsumer = Argus::interface_cast(camera.consumer); + if (!camera.iConsumer) + ORIGINATE_ERROR("Failed to create consumer %s", camera.label.c_str()); + } + } + + captureSession->request.reset(captureSession->iSession->createRequest(Argus::CAPTURE_INTENT_PREVIEW)); + Argus::IRequest* iRequest = Argus::interface_cast(captureSession->request); + if (!iRequest) + ORIGINATE_ERROR("Failed to create request for domain %zu session %zu", domainIndex, sessionIndex); + Argus::IAutoControlSettings* iAutoControl = + Argus::interface_cast(iRequest->getAutoControlSettings()); + if (!iAutoControl || + iAutoControl->setAeAntibandingMode(Argus::AE_ANTIBANDING_MODE_OFF) != Argus::STATUS_OK) + { + ORIGINATE_ERROR("Failed to disable AE antibanding for capture domain %zu session %zu", domainIndex, + sessionIndex); + } + for (size_t member = 0; member < captureSession->cameraIndices.size(); ++member) + { + CameraState& camera = *m_cameras[captureSession->cameraIndices[member]]; + const StreamState& stream = *m_streams[camera.streamIndex]; + if (iRequest->enableOutputStream(camera.outputStream.get()) != Argus::STATUS_OK) + { + ORIGINATE_ERROR("Failed to enable output %s", camera.label.c_str()); + } + Argus::InterfaceProvider* streamSettings = iRequest->getStreamSettings(camera.outputStream.get()); + Argus::IStreamSettings* iStreamSettings = + Argus::interface_cast(streamSettings); + if (!iStreamSettings) + { + ORIGINATE_ERROR("Failed to get stream crop settings for %s", camera.label.c_str()); + } + const NormalizedClip& clip = stream.geometry.clips[camera.cameraIndex]; + if (iStreamSettings->setSourceClipRect( + Argus::Rectangle(clip.left, clip.top, clip.right, clip.bottom)) != Argus::STATUS_OK) + { + ORIGINATE_ERROR("Failed to set source crop for %s", camera.label.c_str()); + } + } + Argus::ISourceSettings* sourceSettings = + Argus::interface_cast(iRequest->getSourceSettings()); + if (!sourceSettings) + { + ORIGINATE_ERROR( + "Failed to get source settings for domain %zu session %zu", domainIndex, sessionIndex); + } + CameraState& firstCamera = *m_cameras[captureSession->cameraIndices.front()]; + if (sourceSettings->setSensorMode(firstCamera.sensorMode) != Argus::STATUS_OK) + { + std::fprintf(stderr, + "Warning: capture domain %zu session %zu rejected explicit mode; using Argus default\n", + domainIndex, sessionIndex); + } + // Pin the frame duration to the sensor mode's fastest rate. + // Without this, auto-exposure is free to stretch the exposure + // (and with it the frame period) in dimmer light — the whole + // rig then follows the trigger down (measured: 20 ms exposure + // -> 43.7 fps on every camera under evening lab lighting). A + // fixed-rate robot camera must hold cadence and let AE + // compensate with gain instead. + const uint64_t pinnedFrameDurationNs = firstCamera.iSensorMode->getFrameDurationRange().min(); + if (sourceSettings->setFrameDurationRange( + Argus::Range(pinnedFrameDurationNs, pinnedFrameDurationNs)) != Argus::STATUS_OK) + { + std::fprintf(stderr, "Warning: capture domain %zu session %zu rejected the pinned frame duration\n", + domainIndex, sessionIndex); + } + domain.sessions.push_back(std::move(captureSession)); + } + + if (verboseLoggingEnabled()) + { + std::printf("Capture domain %zu:", domainIndex); + for (size_t member = 0; member < domain.cameraIndices.size(); ++member) + { + const CameraState& camera = *m_cameras[domain.cameraIndices[member]]; + const StreamState& stream = *m_streams[camera.streamIndex]; + const bool fifoMode = stream.config.capture.mode == CAPTURE_MODE_FIFO; + std::printf("%s %s(d%u/m%u/%s,capture-mode=%s", member == 0 ? "" : ",", camera.label.c_str(), + camera.config.device, camera.config.mode, + stream.config.output.codec == CODEC_HEVC ? "direct-buffer" : "egl-stream", + fifoMode ? "fifo" : "mailbox"); + if (fifoMode && stream.config.output.codec != CODEC_HEVC) + std::printf(",egl-fifo-length=%u", stream.config.capture.fifoLength); + if (independentPriorityStereo) + std::printf("/independent-session"); + std::printf(")"); + } + std::printf("\n"); + } + } + return true; + } + + bool startPriorityDomain(size_t priorityDomainIndex) + { + if (m_started || priorityDomainIndex >= m_domains.size()) + return false; + if (m_domains[priorityDomainIndex]->fifoMode) + { + std::fprintf( + stderr, + "Priority capture domain %zu is configured for FIFO consumption; priority capture requires mailbox policy for both direct-buffer and EGL streams\n", + priorityDomainIndex); + return false; + } + m_error = false; + for (size_t i = 0; i < m_domains.size(); ++i) + { + DomainState& domain = *m_domains[i]; + std::lock_guard lock(domain.mutex); + domain.stop = false; + domain.error = false; + domain.hasPending = false; + domain.pending.clear(); + domain.candidates.clear(); + for (size_t eye = 0; eye < domain.headFrames.size(); ++eye) + domain.headFrames[eye].reset(); + domain.continuousFifo = i != priorityDomainIndex && domain.fifoMode; + domain.demandDriven = i != priorityDomainIndex && !domain.continuousFifo; + domain.captureRequested = false; + domain.selectionTimestampUnavailableWarned = false; + for (size_t sessionIndex = 0; sessionIndex < domain.sessions.size(); ++sessionIndex) + { + domain.sessions[sessionIndex]->captureCompleteCredits = 0; + domain.sessions[sessionIndex]->repeating = false; + } + domain.priorityStereo = false; + domain.priorityStereoStreamIndex = std::numeric_limits::max(); + domain.stereoLeftFlatIndex = std::numeric_limits::max(); + domain.stereoRightFlatIndex = std::numeric_limits::max(); + if (i == priorityDomainIndex && domain.cameraIndices.size() == 2) + { + const size_t firstFlatIndex = domain.cameraIndices[0]; + const size_t secondFlatIndex = domain.cameraIndices[1]; + const CameraState& first = *m_cameras[firstFlatIndex]; + const CameraState& second = *m_cameras[secondFlatIndex]; + if (first.streamIndex == second.streamIndex && + m_streams[first.streamIndex]->config.type == STREAM_TYPE_STEREO && + first.cameraIndex != second.cameraIndex && first.cameraIndex < 2 && second.cameraIndex < 2) + { + domain.priorityStereo = true; + domain.priorityStereoStreamIndex = first.streamIndex; + domain.stereoLeftFlatIndex = first.cameraIndex == 0 ? firstFlatIndex : secondFlatIndex; + domain.stereoRightFlatIndex = first.cameraIndex == 1 ? firstFlatIndex : secondFlatIndex; + } + } + domain.stoppingRepeat = false; + domain.candidateOverflowDiscardsInterval = 0; + domain.candidateOverflowDiscardsTotal = 0; + domain.candidateSelectionDiscardsInterval = 0; + domain.candidateSelectionDiscardsTotal = 0; + domain.candidateSelectionsInterval = 0; + domain.candidateSelectionsTotal = 0; + domain.candidateDeltaSumNsInterval = 0; + domain.candidateDeltaMaxNsInterval = 0; + domain.candidatePastSelectionsInterval = 0; + domain.candidateFutureSelectionsInterval = 0; + domain.candidateDepthPeakInterval = 0; + domain.fifoEventBacklog = 0; + domain.fifoEventBacklogPeakInterval = 0; + domain.stereoPairsInterval = 0; + domain.stereoPairsTotal = 0; + domain.stereoRepairPairsInterval = 0; + domain.stereoRepairPairsTotal = 0; + domain.stereoLeftDropsInterval = 0; + domain.stereoLeftDropsTotal = 0; + domain.stereoRightDropsInterval = 0; + domain.stereoRightDropsTotal = 0; + domain.stereoReacquiresInterval = 0; + domain.stereoReacquiresTotal = 0; + domain.stereoReacquireWaitSumNsInterval = 0; + domain.stereoReacquireWaitMaxNsInterval = 0; + domain.stereoSelectedAgeSumNsInterval = 0; + domain.stereoSelectedAgeMaxNsInterval = 0; + domain.stereoInitialDeltaSumNsInterval = 0; + domain.stereoInitialDeltaMaxNsInterval = 0; + domain.stereoFinalDeltaSumNsInterval = 0; + domain.stereoFinalDeltaMaxNsInterval = 0; + domain.headSlotOverwritesInterval = 0; + domain.headSlotOverwritesTotal = 0; + } + if (!startDomain(priorityDomainIndex)) + { + stop(); + return false; + } + m_started = true; + if (verboseLoggingEnabled()) + { + std::printf( + "argus_sender[capture]: priority domain %zu started; secondary domains are held until priority warm-up completes\n", + priorityDomainIndex); + } + return true; + } + + bool startSecondaryDomains(size_t priorityDomainIndex) + { + if (!m_started || priorityDomainIndex >= m_domains.size()) + return false; + for (size_t i = 0; i < m_domains.size(); ++i) + { + if (i == priorityDomainIndex) + continue; + if (!startDomain(i)) + { + stop(); + return false; + } + } + if (verboseLoggingEnabled() && hasContinuousFifoDomains()) + { + std::printf( + "argus_sender[capture]: secondary domains started; FIFO domains consume continuously with a %zu-frame candidate window\n", + kFifoCandidateWindowLength); + } + else if (verboseLoggingEnabled()) + { + std::printf("argus_sender[capture]: secondary domains started in demand-driven capture mode\n"); + } + return true; + } + + bool discardDomainWarmup(size_t domainIndex, uint32_t frameCount) + { + if (domainIndex >= m_domains.size()) + return false; + for (uint32_t frame = 0; frame < frameCount; ++frame) + { + if (!requestDomainCapture(domainIndex)) + return false; + std::vector> discarded; + if (!takeDomain(domainIndex, &discarded)) + return false; + } + if (frameCount > 0 && verboseLoggingEnabled()) + { + std::printf("argus_sender[capture-domain:%zu]: warm-up complete, discarded %u frame set(s)\n", domainIndex, + frameCount); + } + return true; + } + + bool discardWarmup(uint32_t frameCount) + { + if (frameCount == 0) + return true; + std::vector> warmups; + for (size_t domainIndex = 0; domainIndex < m_domains.size(); ++domainIndex) + { + warmups.push_back(std::async(std::launch::async, [this, domainIndex, frameCount]() + { return discardDomainWarmup(domainIndex, frameCount); })); + } + bool ok = true; + for (size_t i = 0; i < warmups.size(); ++i) + ok = warmups[i].get() && ok; + return ok; + } + + bool waitForDomainSynchronization(size_t domainIndex, + uint32_t maxFrameSets, + uint32_t requiredConsecutive, + uint64_t toleranceNs) + { + if (domainIndex >= m_domains.size() || maxFrameSets == 0 || requiredConsecutive == 0) + { + return false; + } + DomainState& domain = *m_domains[domainIndex]; + if (domain.cameraIndices.size() < 2) + return true; + + uint32_t consecutive = 0; + uint64_t lastSpanNs = 0; + for (uint32_t attempt = 1; attempt <= maxFrameSets; ++attempt) + { + if (!requestDomainCapture(domainIndex)) + return false; + std::vector> pending; + if (!takeDomain(domainIndex, &pending)) + return false; + uint64_t earliest = std::numeric_limits::max(); + uint64_t latest = 0; + for (size_t i = 0; i < pending.size(); ++i) + { + if (!pending[i]->timing.hasSensorSofTimestampTsc || pending[i]->timing.sensorSofTimestampTscNs == 0) + { + std::fprintf(stderr, + "Priority capture domain %zu has no SOF TSC timestamp; refusing unlocked startup\n", + domainIndex); + return false; + } + earliest = std::min(earliest, pending[i]->timing.sensorSofTimestampTscNs); + latest = std::max(latest, pending[i]->timing.sensorSofTimestampTscNs); + } + lastSpanNs = latest - earliest; + if (lastSpanNs <= toleranceNs) + ++consecutive; + else + consecutive = 0; + if (consecutive >= requiredConsecutive) + { + std::printf( + "argus_sender[capture-domain:%zu]: priority synchronization locked after %u frame set(s), tsc_span_us=%.3f\n", + domainIndex, attempt, static_cast(lastSpanNs) / 1000.0); + return true; + } + } + std::fprintf( + stderr, + "Priority capture domain %zu did not reach %u consecutive frame sets within %.3f us after %u attempts; refusing unlocked startup with tsc_span_us=%.3f\n", + domainIndex, requiredConsecutive, static_cast(toleranceNs) / 1000.0, maxFrameSets, + static_cast(lastSpanNs) / 1000.0); + return false; + } + + bool requestDomainCapture(size_t domainIndex) + { + if (domainIndex >= m_domains.size()) + return false; + DomainState& domain = *m_domains[domainIndex]; + { + std::lock_guard lock(domain.mutex); + if (domain.error || domain.stop) + return false; + if (domain.continuousFifo) + return true; + if (!domain.demandDriven) + return true; + domain.captureRequested = true; + } + domain.slotAvailable.notify_one(); + return true; + } + + void prepareCaptured(std::vector* captured) const + { + if (!captured) + return; + captured->clear(); + captured->resize(m_streams.size()); + for (size_t streamIndex = 0; streamIndex < m_streams.size(); ++streamIndex) + { + (*captured)[streamIndex].frames.resize(m_streams[streamIndex]->cameraIndices.size()); + (*captured)[streamIndex].dmaBuffers.resize(m_streams[streamIndex]->cameraIndices.size()); + (*captured)[streamIndex].timings.resize(m_streams[streamIndex]->cameraIndices.size()); + } + } + + bool acquireDomain(size_t domainIndex, std::vector* captured) + { + if (!captured || captured->size() != m_streams.size()) + return false; + std::vector> pending; + if (!takeDomain(domainIndex, &pending)) + return false; + return movePendingToCaptured(&pending, captured); + } + + bool acquireDomainClosestTo(size_t domainIndex, uint64_t referenceSofTscNs, std::vector* captured) + { + if (!captured || captured->size() != m_streams.size() || domainIndex >= m_domains.size()) + { + return false; + } + DomainState& domain = *m_domains[domainIndex]; + if (!domain.continuousFifo) + return acquireDomain(domainIndex, captured); + + const CameraState& firstCamera = *m_cameras[domain.cameraIndices.front()]; + const uint64_t framePeriodNs = m_streams[firstCamera.streamIndex]->framePeriodNs; + const uint64_t selectionWaitNs = framePeriodNs > 0 ? framePeriodNs * 2ULL : kCaptureWaitTimeoutNs; + const std::chrono::steady_clock::time_point deadline = + std::chrono::steady_clock::now() + std::chrono::nanoseconds(selectionWaitNs); + + std::vector> selected; + int64_t selectedDeltaNs = 0; + size_t selectionDiscards = 0; + { + std::unique_lock lock(domain.mutex); + while (!candidateReadyLocked(domain, referenceSofTscNs, framePeriodNs) && !domain.error && !domain.stop && + !stopRequested()) + { + if (domain.framesAvailable.wait_until(lock, deadline) == std::cv_status::timeout) + { + break; + } + } + if (domain.candidates.empty() || domain.error || domain.stop || stopRequested()) + { + return false; + } + + std::vector candidateTimestamps; + candidateTimestamps.reserve(domain.candidates.size()); + for (size_t i = 0; i < domain.candidates.size(); ++i) + { + candidateTimestamps.push_back(candidateSofTscNs(domain.candidates[i])); + } + size_t selectedIndex = domain.candidates.size() - 1; + if (!chooseClosestTscCandidate(candidateTimestamps, referenceSofTscNs, &selectedIndex, &selectedDeltaNs)) + { + selectedDeltaNs = 0; + if (!domain.selectionTimestampUnavailableWarned) + { + std::fprintf( + stderr, + "Warning: FIFO candidate selection for domain %zu lacks valid SOF TSC; selecting newest candidate\n", + domain.index); + domain.selectionTimestampUnavailableWarned = true; + } + } + + selectionDiscards = selectedIndex; + while (selectedIndex > 0) + { + domain.candidates.pop_front(); + --selectedIndex; + } + selected.swap(domain.candidates.front()); + domain.candidates.pop_front(); + } + + if (selectionDiscards > 0) + { + domain.candidateSelectionDiscardsInterval.fetch_add(selectionDiscards); + domain.candidateSelectionDiscardsTotal.fetch_add(selectionDiscards); + } + domain.candidateSelectionsInterval.fetch_add(1); + domain.candidateSelectionsTotal.fetch_add(1); + const uint64_t absoluteDelta = absoluteDeltaNs(selectedDeltaNs); + domain.candidateDeltaSumNsInterval.fetch_add(absoluteDelta); + updateAtomicMaximum(&domain.candidateDeltaMaxNsInterval, absoluteDelta); + if (selectedDeltaNs < 0) + domain.candidatePastSelectionsInterval.fetch_add(1); + else if (selectedDeltaNs > 0) + domain.candidateFutureSelectionsInterval.fetch_add(1); + + return movePendingToCaptured(&selected, captured); + } + + bool reacquireDomain(size_t domainIndex, std::vector* captured) + { + if (domainIndex >= m_domains.size() || !captured) + return false; + const DomainState& domain = *m_domains[domainIndex]; + for (size_t i = 0; i < domain.cameraIndices.size(); ++i) + { + const CameraState& camera = *m_cameras[domain.cameraIndices[i]]; + (*captured)[camera.streamIndex].frames[camera.cameraIndex].reset(); + (*captured)[camera.streamIndex].dmaBuffers[camera.cameraIndex].reset(); + (*captured)[camera.streamIndex].timings[camera.cameraIndex] = FrameTiming(); + } + if (!requestDomainCapture(domainIndex)) + return false; + return acquireDomain(domainIndex, captured); + } + + void recordCaptured(size_t streamIndex, const CapturedFrameSet& frames) + { + if (streamIndex >= m_streams.size() || frames.frames.size() != m_streams[streamIndex]->cameraIndices.size() || + frames.dmaBuffers.size() != frames.frames.size()) + { + return; + } + for (size_t i = 0; i < frames.frames.size(); ++i) + { + // The HEVC processor can take ownership of a direct DMABUF + // before this periodic stats hook. Successful acquisition leaves + // a steady-clock marker even when SOF TSC is unavailable. + if (!frames.frames[i].get() && !frames.dmaBuffers[i].get() && + (i >= frames.timings.size() || frames.timings[i].acquiredSteadyNs == 0)) + return; + } + recordStats(m_streams[streamIndex].get(), frames); + } + + bool stop() + { + bool ok = true; + std::vector> waitForIdle(m_domains.size()); + for (size_t i = 0; i < m_domains.size(); ++i) + { + if (!m_domains[i]) + continue; + DomainState& domain = *m_domains[i]; + waitForIdle[i].assign(domain.sessions.size(), false); + bool repeating = false; + for (size_t sessionIndex = 0; sessionIndex < domain.sessions.size(); ++sessionIndex) + { + repeating = repeating || domain.sessions[sessionIndex]->repeating; + } + { + std::lock_guard lock(domain.mutex); + if (domain.continuousFifo && repeating) + { + domain.stoppingRepeat = true; + domain.candidates.clear(); + } + else + { + domain.stop = true; + } + domain.captureRequested = false; + } + domain.slotAvailable.notify_all(); + domain.framesAvailable.notify_all(); + } + for (size_t i = 0; i < m_domains.size(); ++i) + { + if (!m_domains[i]) + continue; + DomainState& domain = *m_domains[i]; + for (size_t sessionIndex = 0; sessionIndex < domain.sessions.size(); ++sessionIndex) + { + CaptureSessionState& session = *domain.sessions[sessionIndex]; + if (session.iSession && session.repeating) + { + waitForIdle[i][sessionIndex] = true; + session.iSession->stopRepeat(); + if (session.iSession->isRepeating()) + { + std::fprintf(stderr, "Capture domain %zu session %zu remained in repeat mode after stopRepeat\n", + i, sessionIndex); + ok = false; + } + } + session.repeating = false; + } + } + // Keep continuous FIFO consumers alive until all repeated requests + // retire. During this phase they drain and immediately release frames + // instead of placing them back into the candidate window. + for (size_t i = 0; i < m_domains.size(); ++i) + { + if (!m_domains[i]) + continue; + DomainState& domain = *m_domains[i]; + if (!domain.continuousFifo) + continue; + for (size_t sessionIndex = 0; sessionIndex < domain.sessions.size(); ++sessionIndex) + { + CaptureSessionState& session = *domain.sessions[sessionIndex]; + if (session.iSession && waitForIdle[i][sessionIndex]) + { + const Argus::Status status = session.iSession->waitForIdle(kIdleWaitTimeoutNs); + if (status != Argus::STATUS_OK) + { + std::fprintf( + stderr, + "Capture domain %zu session %zu failed to become idle while draining FIFO (status %d)\n", i, + sessionIndex, status); + ok = false; + } + } + } + } + for (size_t i = 0; i < m_domains.size(); ++i) + { + if (!m_domains[i]) + continue; + DomainState& domain = *m_domains[i]; + { + std::lock_guard lock(domain.mutex); + domain.stop = true; + domain.pending.clear(); + domain.hasPending = false; + domain.candidates.clear(); + for (size_t eye = 0; eye < domain.headFrames.size(); ++eye) + domain.headFrames[eye].reset(); + } + domain.slotAvailable.notify_all(); + domain.framesAvailable.notify_all(); + } + for (size_t i = 0; i < m_domains.size(); ++i) + { + if (!m_domains[i]) + continue; + DomainState& domain = *m_domains[i]; + if (domain.worker.joinable()) + domain.worker.join(); + if (domain.eventWorker.joinable()) + domain.eventWorker.join(); + for (size_t worker = 0; worker < domain.cameraWorkers.size(); ++worker) + { + if (domain.cameraWorkers[worker].joinable()) + domain.cameraWorkers[worker].join(); + } + } + for (size_t i = 0; i < m_domains.size(); ++i) + { + if (!m_domains[i]) + continue; + DomainState& domain = *m_domains[i]; + { + std::lock_guard lock(domain.mutex); + domain.pending.clear(); + domain.hasPending = false; + domain.candidates.clear(); + for (size_t eye = 0; eye < domain.headFrames.size(); ++eye) + domain.headFrames[eye].reset(); + } + if (domain.continuousFifo) + continue; + for (size_t sessionIndex = 0; sessionIndex < domain.sessions.size(); ++sessionIndex) + { + CaptureSessionState& session = *domain.sessions[sessionIndex]; + if (session.iSession && waitForIdle[i][sessionIndex]) + { + const Argus::Status status = session.iSession->waitForIdle(kIdleWaitTimeoutNs); + if (status != Argus::STATUS_OK) + { + std::fprintf(stderr, "Capture domain %zu session %zu failed to become idle (status %d)\n", i, + sessionIndex, status); + ok = false; + } + } + } + } + m_started = false; + if (!ok) + m_error = true; + return ok; + } + + bool hasError() const + { + if (m_error.load()) + return true; + for (size_t i = 0; i < m_domains.size(); ++i) + { + std::lock_guard lock(m_domains[i]->mutex); + if (m_domains[i]->error) + return true; + } + return false; + } + + bool domainUsesContinuousFifo(size_t domainIndex) const + { + if (domainIndex >= m_domains.size()) + return false; + std::lock_guard lock(m_domains[domainIndex]->mutex); + return m_domains[domainIndex]->continuousFifo; + } + + bool hasContinuousFifoDomains() const + { + for (size_t i = 0; i < m_domains.size(); ++i) + { + std::lock_guard lock(m_domains[i]->mutex); + if (m_domains[i]->continuousFifo) + return true; + } + return false; + } + + void logFifoCandidateStats() + { + for (size_t domainIndex = 0; domainIndex < m_domains.size(); ++domainIndex) + { + DomainState& domain = *m_domains[domainIndex]; + size_t candidateDepth = 0; + bool continuousFifo = false; + { + std::lock_guard lock(domain.mutex); + continuousFifo = domain.continuousFifo; + candidateDepth = domain.candidates.size(); + } + if (domain.priorityStereo && domain.priorityStereoStreamIndex < m_streams.size()) + { + const uint64_t pairs = domain.stereoPairsInterval.exchange(0); + const uint64_t pairsTotal = domain.stereoPairsTotal.load(); + const uint64_t repairs = domain.stereoRepairPairsInterval.exchange(0); + const uint64_t repairsTotal = domain.stereoRepairPairsTotal.load(); + const uint64_t leftDrops = domain.stereoLeftDropsInterval.exchange(0); + const uint64_t leftDropsTotal = domain.stereoLeftDropsTotal.load(); + const uint64_t rightDrops = domain.stereoRightDropsInterval.exchange(0); + const uint64_t rightDropsTotal = domain.stereoRightDropsTotal.load(); + const uint64_t reacquires = domain.stereoReacquiresInterval.exchange(0); + const uint64_t reacquiresTotal = domain.stereoReacquiresTotal.load(); + const uint64_t waitSumNs = domain.stereoReacquireWaitSumNsInterval.exchange(0); + const uint64_t waitMaxNs = domain.stereoReacquireWaitMaxNsInterval.exchange(0); + const uint64_t selectedAgeSumNs = domain.stereoSelectedAgeSumNsInterval.exchange(0); + const uint64_t selectedAgeMaxNs = domain.stereoSelectedAgeMaxNsInterval.exchange(0); + const uint64_t initialDeltaSumNs = domain.stereoInitialDeltaSumNsInterval.exchange(0); + const uint64_t initialDeltaMaxNs = domain.stereoInitialDeltaMaxNsInterval.exchange(0); + const uint64_t finalDeltaSumNs = domain.stereoFinalDeltaSumNsInterval.exchange(0); + const uint64_t finalDeltaMaxNs = domain.stereoFinalDeltaMaxNsInterval.exchange(0); + const uint64_t slotOverwrites = domain.headSlotOverwritesInterval.exchange(0); + const uint64_t slotOverwritesTotal = domain.headSlotOverwritesTotal.load(); + if (!verboseLoggingEnabled()) + { + std::printf( + "argus_sender[head-handoff:%s]: slot_overwrites(interval/total)=%llu/%llu, alignment_drops(left/right,interval)=%llu/%llu\n", + m_streams[domain.priorityStereoStreamIndex]->config.id.c_str(), + static_cast(slotOverwrites), + static_cast(slotOverwritesTotal), + static_cast(leftDrops), static_cast(rightDrops)); + } + if (pairs > 0) + { + if (verboseLoggingEnabled()) + std::printf( + "argus_sender[head-align:%s]: pairs(interval/total)=%llu/%llu, repaired_pairs(interval/total)=%llu/%llu, intentional_drops(left/right,interval)=%llu/%llu, intentional_drops(left/right,total)=%llu/%llu, alignment_reacquires(interval/total)=%llu/%llu, slot_overwrites(interval/total)=%llu/%llu, reacquire_wait_us(avg/max)=%.3f/%.3f, selected_age_us(avg/max)=%.3f/%.3f, abs_tsc_delta_us(initial_avg/max)=%.3f/%.3f, abs_tsc_delta_us(final_avg/max)=%.3f/%.3f\n", + m_streams[domain.priorityStereoStreamIndex]->config.id.c_str(), + static_cast(pairs), static_cast(pairsTotal), + static_cast(repairs), static_cast(repairsTotal), + static_cast(leftDrops), static_cast(rightDrops), + static_cast(leftDropsTotal), + static_cast(rightDropsTotal), + static_cast(reacquires), static_cast(reacquiresTotal), + static_cast(slotOverwrites), + static_cast(slotOverwritesTotal), + reacquires > 0 ? static_cast(waitSumNs) / static_cast(reacquires) / 1000.0 : + 0.0, + static_cast(waitMaxNs) / 1000.0, + static_cast(selectedAgeSumNs) / static_cast(pairs) / 1000.0, + static_cast(selectedAgeMaxNs) / 1000.0, + static_cast(initialDeltaSumNs) / static_cast(pairs) / 1000.0, + static_cast(initialDeltaMaxNs) / 1000.0, + static_cast(finalDeltaSumNs) / static_cast(pairs) / 1000.0, + static_cast(finalDeltaMaxNs) / 1000.0); + } + } + if (!verboseLoggingEnabled()) + continue; + if (!continuousFifo || domain.cameraIndices.empty()) + continue; + + const CameraState& camera = *m_cameras[domain.cameraIndices.front()]; + const uint64_t overflowInterval = domain.candidateOverflowDiscardsInterval.exchange(0); + const uint64_t selectionInterval = domain.candidateSelectionDiscardsInterval.exchange(0); + const uint64_t overflowTotal = domain.candidateOverflowDiscardsTotal.load(); + const uint64_t selectionTotal = domain.candidateSelectionDiscardsTotal.load(); + const uint64_t selections = domain.candidateSelectionsInterval.exchange(0); + const uint64_t selectionsTotal = domain.candidateSelectionsTotal.load(); + const uint64_t deltaSumNs = domain.candidateDeltaSumNsInterval.exchange(0); + const uint64_t deltaMaxNs = domain.candidateDeltaMaxNsInterval.exchange(0); + const uint64_t pastSelections = domain.candidatePastSelectionsInterval.exchange(0); + const uint64_t futureSelections = domain.candidateFutureSelectionsInterval.exchange(0); + const uint64_t depthPeak = domain.candidateDepthPeakInterval.exchange(candidateDepth); + const uint64_t eventBacklogPeak = + domain.fifoEventBacklogPeakInterval.exchange(domain.fifoEventBacklog.load()); + const double deltaAverageUs = + selections > 0 ? static_cast(deltaSumNs) / static_cast(selections) / 1000.0 : 0.0; + + std::printf( + "argus_sender[fifo:%s]: fifo_length=%u, candidate_depth(current/peak/capacity)=%zu/%llu/%zu, fifo_event_backlog(current/peak)=%llu/%llu, intentional_discards(interval/total)=%llu/%llu (overflow=%llu/%llu, selection=%llu/%llu), selected(interval/total)=%llu/%llu, selected_abs_tsc_delta_us(avg/max)=%.3f/%.3f, selected_side(past/future/exact)=%llu/%llu/%llu\n", + camera.label.c_str(), domain.fifoLength, candidateDepth, static_cast(depthPeak), + kFifoCandidateWindowLength, static_cast(domain.fifoEventBacklog.load()), + static_cast(eventBacklogPeak), + static_cast(overflowInterval + selectionInterval), + static_cast(overflowTotal + selectionTotal), + static_cast(overflowInterval), static_cast(overflowTotal), + static_cast(selectionInterval), static_cast(selectionTotal), + static_cast(selections), static_cast(selectionsTotal), + deltaAverageUs, static_cast(deltaMaxNs) / 1000.0, + static_cast(pastSelections), static_cast(futureSelections), + static_cast(selections - pastSelections - futureSelections)); + } + } + + size_t streamCount() const + { + return m_streams.size(); + } + size_t domainCount() const + { + return m_domains.size(); + } + + size_t streamDomain(size_t streamIndex) const + { + if (streamIndex >= m_streams.size() || m_streams[streamIndex]->cameraIndices.empty()) + { + return std::numeric_limits::max(); + } + return m_cameras[m_streams[streamIndex]->cameraIndices.front()]->domainIndex; + } + + size_t domainFor(size_t streamIndex, size_t cameraIndex) const + { + if (streamIndex >= m_cameraLookup.size() || cameraIndex >= m_cameraLookup[streamIndex].size()) + { + return std::numeric_limits::max(); + } + return m_cameras[m_cameraLookup[streamIndex][cameraIndex]]->domainIndex; + } + + const ResolvedOutputGeometry& geometry(size_t streamIndex) const + { + return m_streams[streamIndex]->geometry; + } + + uint32_t nominalFps(size_t streamIndex) const + { + return m_streams[streamIndex]->nominalFps; + } + + uint64_t framePeriodNs(size_t streamIndex) const + { + return m_streams[streamIndex]->framePeriodNs; + } + + void releaseResources() + { + cleanup(); + } + +private: + bool startDomain(size_t domainIndex) + { + DomainState& domain = *m_domains[domainIndex]; + if (domain.priorityStereo && verboseLoggingEnabled()) + { + std::printf( + "argus_sender[head-align:%s]: independent left/right Argus sessions and continuous acquisition; acquired surfaces are retained without copying\n", + m_streams[domain.priorityStereoStreamIndex]->config.id.c_str()); + } + try + { + domain.worker = std::thread(&Impl::domainThreadMain, this, &domain); + if (domain.priorityStereo) + { + domain.eventWorker = std::thread(&Impl::priorityEventThreadMain, this, &domain); + domain.cameraWorkers[0] = + std::thread(&Impl::priorityCameraThreadMain, this, &domain, domain.stereoLeftFlatIndex, 0); + domain.cameraWorkers[1] = + std::thread(&Impl::priorityCameraThreadMain, this, &domain, domain.stereoRightFlatIndex, 1); + } + } + catch (...) + { + return failDomain(&domain, "failed to create capture worker"); + } + for (size_t sessionIndex = 0; sessionIndex < domain.sessions.size(); ++sessionIndex) + { + CaptureSessionState& session = *domain.sessions[sessionIndex]; + if (session.iSession->repeat(session.request.get()) != Argus::STATUS_OK) + { + return failDomain(&domain, "failed to start repeating capture"); + } + session.repeating = true; + } + return true; + } + + bool stopping(DomainState* domain) const + { + std::lock_guard lock(domain->mutex); + return domain->stop || domain->error || stopRequested(); + } + + bool failDomain(DomainState* domain, const char* message, Argus::Status status = Argus::STATUS_OK) + { + if (status == Argus::STATUS_OK) + { + std::fprintf(stderr, "capture domain %zu: %s\n", domain->index, message); + } + else + { + std::fprintf(stderr, "capture domain %zu: %s (status %d)\n", domain->index, message, status); + } + { + std::lock_guard lock(domain->mutex); + domain->error = true; + } + m_error = true; + domain->framesAvailable.notify_all(); + domain->slotAvailable.notify_all(); + return false; + } + + bool waitForSessionCaptureComplete(DomainState* domain, + CaptureSessionState* session, + const std::atomic* expectedStop) + { + if (session->captureCompleteCredits > 0) + { + --session->captureCompleteCredits; + return true; + } + while (!stopping(domain)) + { + const Argus::Status status = + session->iEventProvider->waitForEvents(session->eventQueue.get(), kCaptureWaitTimeoutNs); + if (status == Argus::STATUS_TIMEOUT) + continue; + if (status != Argus::STATUS_OK) + { + if (expectedStop && expectedStop->load()) + return false; + return failDomain(domain, "failed waiting for capture event", status); + } + uint64_t completeCount = 0; + const Argus::Event* event = NULL; + while ((event = session->iEventQueue->getNextEvent()) != NULL) + { + const Argus::IEvent* iEvent = Argus::interface_cast(event); + if (!iEvent) + continue; + const Argus::EventType eventType = iEvent->getEventType(); + if (eventType == Argus::EVENT_TYPE_CAPTURE_COMPLETE) + { + ++completeCount; + continue; + } + const Argus::IEventError* iError = Argus::interface_cast(event); + if (!iError) + continue; + if (expectedStop && expectedStop->load()) + return false; + const char* eventName = eventType.getName(); + const bool correctable = eventName && std::strstr(eventName, "CORR") && !std::strstr(eventName, "UNCORR"); + if (correctable) + { + std::fprintf( + stderr, + "Warning: capture domain %zu received correctable Argus event %s (status %d); continuing\n", + domain->index, eventName, iError->getStatus()); + continue; + } + return failDomain(domain, "received Argus capture error", iError->getStatus()); + } + if (completeCount > 0) + { + session->captureCompleteCredits += domain->continuousFifo ? completeCount : 1ULL; + --session->captureCompleteCredits; + return true; + } + } + return false; + } + + bool waitForCaptureComplete(DomainState* domain, const std::atomic* expectedStop = NULL) + { + for (size_t sessionIndex = 0; sessionIndex < domain->sessions.size(); ++sessionIndex) + { + if (!waitForSessionCaptureComplete(domain, domain->sessions[sessionIndex].get(), expectedStop)) + { + return false; + } + } + uint64_t backlog = 0; + for (size_t sessionIndex = 0; sessionIndex < domain->sessions.size(); ++sessionIndex) + { + backlog += domain->sessions[sessionIndex]->captureCompleteCredits; + } + updateAtomicMaximum(&domain->fifoEventBacklogPeakInterval, backlog); + domain->fifoEventBacklog = backlog; + return true; + } + + void priorityEventThreadMain(DomainState* domain) + { + // Error events still need a consumer, but capture-complete events must + // not gate direct-buffer acquisition. + while (!stopping(domain) && waitForCaptureComplete(domain)) + { + } + } + + std::unique_ptr acquireCameraFrame(DomainState* domain, + size_t flatIndex, + const std::atomic* expectedStop, + bool continuousAcquire = false) + { + CameraState& camera = *m_cameras[flatIndex]; + Argus::Status status = Argus::STATUS_OK; + std::unique_ptr sample(new PendingFrame(flatIndex)); + const uint64_t droughtStartNs = steadyNowNs(); + uint64_t acquiredTscNs = 0; + do + { + status = Argus::STATUS_OK; + if (camera.iBufferStream) + { + Argus::Buffer* argusBuffer = camera.iBufferStream->acquireBuffer(kCaptureWaitTimeoutNs, &status); + acquiredTscNs = readTscNowNs(); + if (argusBuffer) + { + Argus::IBuffer* iBuffer = Argus::interface_cast(argusBuffer); + DmaBuffer* dmaBuffer = + iBuffer ? const_cast(static_cast(iBuffer->getClientData())) : NULL; + if (!dmaBuffer || !dmaBuffer->markArgusAcquired(argusBuffer)) + { + camera.iBufferStream->releaseBuffer(argusBuffer); + failDomain(domain, "failed to resolve acquired direct DMABUF"); + return std::unique_ptr(); + } + sample->dmaBuffer.reset(dmaBuffer); + } + } + else + { + EGLStream::Frame* frame = camera.iConsumer->acquireFrame(kCaptureWaitTimeoutNs, &status); + acquiredTscNs = readTscNowNs(); + sample->frame.reset(frame); + } + if (status == Argus::STATUS_TIMEOUT && continuousAcquire && !stopping(domain)) + { + const uint64_t limitNs = camera.hasLastFrameNumber ? kPrioritySteadyDroughtNs : kPriorityStartupDroughtNs; + if (priorityFrameDroughtExpired(droughtStartNs, steadyNowNs(), camera.hasLastFrameNumber)) + { + std::fprintf(stderr, "Priority camera %s produced no frame for %.1f s (%s)\n", camera.label.c_str(), + static_cast(limitNs) / 1e9, + camera.hasLastFrameNumber ? "steady-state drought" : "startup timeout"); + failDomain(domain, "priority camera frame drought"); + return std::unique_ptr(); + } + continue; + } + break; + } while (!stopping(domain)); + if ((!sample->frame && !sample->dmaBuffer.get()) || status != Argus::STATUS_OK) + { + if (stopping(domain) || (expectedStop && expectedStop->load())) + { + return std::unique_ptr(); + } + failDomain(domain, camera.iBufferStream ? "failed to acquire direct buffer" : "failed to acquire EGL frame", + status); + return std::unique_ptr(); + } + const uint64_t acquiredNs = steadyNowNs(); + const bool timingRead = + camera.iBufferStream ? + readFrameTiming(sample->dmaBuffer.get()->getArgusBuffer(), camera.label.c_str(), &sample->timing) : + readFrameTiming(sample->frame.get(), camera.label.c_str(), &sample->timing); + if (!timingRead) + { + failDomain(domain, "failed to read frame metadata"); + return std::unique_ptr(); + } + sample->timing.acquiredSteadyNs = acquiredNs; + sample->timing.acquiredTscNs = acquiredTscNs; + sample->duplicate = camera.hasLastFrameNumber && sample->timing.frameNumber == camera.lastFrameNumber; + if (camera.hasLastFrameNumber && sample->timing.frameNumber > camera.lastFrameNumber + 1) + { + sample->timing.frameNumberGap = sample->timing.frameNumber - camera.lastFrameNumber - 1; + } + camera.lastFrameNumber = sample->timing.frameNumber; + camera.hasLastFrameNumber = true; + return sample; + } + + void priorityCameraThreadMain(DomainState* domain, size_t flatIndex, size_t eye) + { + PriorityFrameProgress progress(steadyNowNs()); + while (!stopping(domain)) + { + std::unique_ptr sample = acquireCameraFrame(domain, flatIndex, NULL, true); + if (!sample) + break; + if (sample->duplicate) + { + const uint64_t frameNumber = sample->timing.frameNumber; + // Return the acquired surface before rate-limiting retries + // of the same frame number. + sample.reset(); + if (progress.droughtExpired(steadyNowNs())) + { + std::fprintf( + stderr, "Priority camera %s produced no new frame for %.1f s (repeated frame number %llu)\n", + m_cameras[flatIndex]->label.c_str(), static_cast(progress.droughtLimitNs()) / 1e9, + static_cast(frameNumber)); + failDomain(domain, "priority camera new-frame drought"); + break; + } + std::this_thread::sleep_for(std::chrono::microseconds(100)); + continue; + } + progress.noteFreshFrame(steadyNowNs()); + std::unique_ptr stale; + { + std::lock_guard lock(domain->mutex); + if (domain->stop || domain->error || stopRequested()) + break; + LatestFrameSlot& slot = domain->headFrames[eye]; + if (slot.occupied()) + { + domain->headSlotOverwritesInterval.fetch_add(1); + domain->headSlotOverwritesTotal.fetch_add(1); + } + stale = slot.replace(std::move(sample)); + } + // Recycle superseded Argus buffers outside the queue lock. + stale.reset(); + domain->framesAvailable.notify_all(); + } + } + + std::unique_ptr takePriorityCameraFrame(DomainState* domain, size_t eye) + { + std::unique_lock lock(domain->mutex); + while (!domain->headFrames[eye].occupied() && !domain->error && !domain->stop && !stopRequested()) + { + domain->framesAvailable.wait_for(lock, std::chrono::milliseconds(kWaitPollMs)); + } + if (domain->error || domain->stop || stopRequested()) + return std::unique_ptr(); + std::unique_ptr sample = domain->headFrames[eye].take(); + return sample; + } + + bool acquireDomainSet(DomainState* domain, + std::vector>* pending, + const std::atomic* expectedStop = NULL) + { + pending->clear(); + for (size_t member = 0; member < domain->cameraIndices.size(); ++member) + { + std::unique_ptr sample = + acquireCameraFrame(domain, domain->cameraIndices[member], expectedStop); + if (!sample) + return false; + pending->push_back(std::move(sample)); + } + for (size_t member = 0; member < pending->size(); ++member) + { + if ((*pending)[member]->duplicate) + { + pending->clear(); + break; + } + } + return true; + } + + bool replacePriorityCameraFrame(DomainState* domain, PendingFrame* sample) + { + const size_t eye = sample->cameraFlatIndex == domain->stereoLeftFlatIndex ? 0 : 1; + // Release the older surface before waiting for a replacement so it + // cannot starve the producer's finite DMABUF pool. + sample->frame.reset(); + sample->dmaBuffer.reset(); + std::unique_ptr replacement = takePriorityCameraFrame(domain, eye); + if (!replacement) + return false; + sample->frame.reset(replacement->frame.release()); + sample->dmaBuffer.reset(replacement->dmaBuffer.release()); + sample->timing = replacement->timing; + sample->duplicate = false; + return true; + } + + bool acquirePriorityStereoSet(DomainState* domain, std::vector>* pending) + { + if (!domain || !pending || domain->priorityStereoStreamIndex >= m_streams.size()) + { + return false; + } + pending->clear(); + std::unique_ptr leftSample = takePriorityCameraFrame(domain, 0); + if (!leftSample) + return false; + std::unique_ptr rightSample = takePriorityCameraFrame(domain, 1); + if (!rightSample) + return false; + pending->push_back(std::move(leftSample)); + pending->push_back(std::move(rightSample)); + + PendingFrame* left = NULL; + PendingFrame* right = NULL; + for (size_t i = 0; i < pending->size(); ++i) + { + if ((*pending)[i]->cameraFlatIndex == domain->stereoLeftFlatIndex) + { + left = (*pending)[i].get(); + } + else if ((*pending)[i]->cameraFlatIndex == domain->stereoRightFlatIndex) + { + right = (*pending)[i].get(); + } + } + if (!left || !right) + return failDomain(domain, "priority stereo pair is incomplete"); + + const uint64_t framePeriodNs = m_streams[domain->priorityStereoStreamIndex]->framePeriodNs; + int64_t delta = 0; + if (!stereoAlignmentDeltaNs(left->timing, right->timing, &delta)) + { + return failDomain(domain, "head stereo alignment requires comparable SOF timestamps"); + } + const int64_t initialDelta = delta; + uint32_t leftDrops = 0; + uint32_t rightDrops = 0; + uint32_t reacquires = 0; + uint64_t reacquireWaitNs = 0; + while (reacquires < kStereoMaxAlignmentDrops) + { + const StereoAlignmentAction action = + chooseStereoAlignmentAction(left->timing, right->timing, framePeriodNs, reacquires, &delta); + PendingFrame* older = NULL; + if (action == STEREO_ALIGNMENT_DROP_LEFT) + { + older = left; + ++leftDrops; + } + else if (action == STEREO_ALIGNMENT_DROP_RIGHT) + { + older = right; + ++rightDrops; + } + else + { + break; + } + const uint64_t reacquireStartNs = steadyNowNs(); + if (!replacePriorityCameraFrame(domain, older)) + return false; + reacquireWaitNs += steadyNowNs() - reacquireStartNs; + ++reacquires; + if (!stereoAlignmentDeltaNs(left->timing, right->timing, &delta)) + { + return failDomain(domain, "selectively reacquired head frame has no comparable SOF timestamp"); + } + } + const bool aligned = framePeriodNs != 0 && absoluteDeltaNs(delta) <= framePeriodNs / 2ULL; + const uint64_t readyNs = steadyNowNs(); + const uint64_t oldestAcquiredNs = std::min(left->timing.acquiredSteadyNs, right->timing.acquiredSteadyNs); + const uint64_t selectedAgeNs = + oldestAcquiredNs != 0 && readyNs > oldestAcquiredNs ? readyNs - oldestAcquiredNs : 0; + + domain->stereoPairsInterval.fetch_add(1); + domain->stereoPairsTotal.fetch_add(1); + const uint64_t drops = leftDrops + rightDrops; + if (drops > 0) + { + domain->stereoRepairPairsInterval.fetch_add(1); + domain->stereoRepairPairsTotal.fetch_add(1); + } + domain->stereoLeftDropsInterval.fetch_add(leftDrops); + domain->stereoLeftDropsTotal.fetch_add(leftDrops); + domain->stereoRightDropsInterval.fetch_add(rightDrops); + domain->stereoRightDropsTotal.fetch_add(rightDrops); + domain->stereoReacquiresInterval.fetch_add(reacquires); + domain->stereoReacquiresTotal.fetch_add(reacquires); + domain->stereoReacquireWaitSumNsInterval.fetch_add(reacquireWaitNs); + updateAtomicMaximum(&domain->stereoReacquireWaitMaxNsInterval, reacquireWaitNs); + domain->stereoSelectedAgeSumNsInterval.fetch_add(selectedAgeNs); + updateAtomicMaximum(&domain->stereoSelectedAgeMaxNsInterval, selectedAgeNs); + const uint64_t initialDeltaNs = absoluteDeltaNs(initialDelta); + const uint64_t finalDeltaNs = absoluteDeltaNs(delta); + domain->stereoInitialDeltaSumNsInterval.fetch_add(initialDeltaNs); + updateAtomicMaximum(&domain->stereoInitialDeltaMaxNsInterval, initialDeltaNs); + domain->stereoFinalDeltaSumNsInterval.fetch_add(finalDeltaNs); + updateAtomicMaximum(&domain->stereoFinalDeltaMaxNsInterval, finalDeltaNs); + + if (!aligned) + { + std::fprintf( + stderr, + "Head stereo pair could not be associated within half a %.3f us frame period after %u selective reacquisition(s): initial_delta_us=%.3f, final_delta_us=%.3f, drops(left/right)=%u/%u\n", + static_cast(framePeriodNs) / 1000.0, reacquires, static_cast(initialDelta) / 1000.0, + static_cast(delta) / 1000.0, leftDrops, rightDrops); + return failDomain(domain, "selective stereo frame association exhausted its retry budget"); + } + + return true; + } + + void domainThreadMain(DomainState* domain) + { + while (!stopping(domain)) + { + bool demandGranted = false; + if (!domain->continuousFifo) + { + std::unique_lock lock(domain->mutex); + domain->slotAvailable.wait( + lock, + [domain]() + { + return domain->stop || stopRequested() || + (!domain->hasPending && (!domain->demandDriven || domain->captureRequested)); + }); + if (domain->stop || stopRequested()) + break; + demandGranted = domain->demandDriven; + if (demandGranted) + domain->captureRequested = false; + } + if (!domain->priorityStereo && + !waitForCaptureComplete(domain, domain->continuousFifo ? &domain->stoppingRepeat : NULL)) + break; + std::vector> pending; + const bool acquired = + domain->priorityStereo ? + acquirePriorityStereoSet(domain, &pending) : + acquireDomainSet(domain, &pending, domain->continuousFifo ? &domain->stoppingRepeat : NULL); + if (!acquired) + break; + if (domain->continuousFifo && domain->stoppingRepeat.load()) + { + pending.clear(); + continue; + } + if (pending.empty()) + { + if (demandGranted) + { + std::lock_guard lock(domain->mutex); + domain->captureRequested = true; + domain->slotAvailable.notify_one(); + } + continue; + } + if (domain->continuousFifo) + { + size_t overflowDiscards = 0; + size_t candidateDepth = 0; + { + std::lock_guard lock(domain->mutex); + if (domain->stop || stopRequested()) + break; + while (domain->candidates.size() >= kFifoCandidateWindowLength) + { + domain->candidates.pop_front(); + ++overflowDiscards; + } + domain->candidates.push_back(std::move(pending)); + candidateDepth = domain->candidates.size(); + } + if (overflowDiscards > 0) + { + domain->candidateOverflowDiscardsInterval.fetch_add(overflowDiscards); + domain->candidateOverflowDiscardsTotal.fetch_add(overflowDiscards); + } + updateAtomicMaximum(&domain->candidateDepthPeakInterval, candidateDepth); + domain->framesAvailable.notify_all(); + continue; + } + { + std::lock_guard lock(domain->mutex); + if (domain->stop || stopRequested()) + break; + domain->pending.swap(pending); + domain->hasPending = true; + } + domain->framesAvailable.notify_one(); + } + } + + bool takeDomain(size_t domainIndex, std::vector>* pending) + { + if (!pending || domainIndex >= m_domains.size()) + return false; + DomainState& domain = *m_domains[domainIndex]; + std::unique_lock lock(domain.mutex); + while (!(domain.continuousFifo ? !domain.candidates.empty() : domain.hasPending) && !domain.error && + !domain.stop && !stopRequested()) + { + domain.framesAvailable.wait_for(lock, std::chrono::milliseconds(kWaitPollMs)); + } + if (domain.continuousFifo) + { + if (domain.candidates.empty()) + return false; + pending->clear(); + pending->swap(domain.candidates.front()); + domain.candidates.pop_front(); + return true; + } + if (!domain.hasPending) + return false; + pending->clear(); + pending->swap(domain.pending); + domain.hasPending = false; + lock.unlock(); + domain.slotAvailable.notify_one(); + return true; + } + + uint64_t candidateSofTscNs(const std::vector>& pending) const + { + if (pending.empty()) + return 0; + uint64_t earliest = std::numeric_limits::max(); + uint64_t latest = 0; + for (size_t i = 0; i < pending.size(); ++i) + { + if (!pending[i] || !pending[i]->timing.hasSensorSofTimestampTsc || + pending[i]->timing.sensorSofTimestampTscNs == 0) + { + return 0; + } + earliest = std::min(earliest, pending[i]->timing.sensorSofTimestampTscNs); + latest = std::max(latest, pending[i]->timing.sensorSofTimestampTscNs); + } + return earliest + (latest - earliest) / 2ULL; + } + + bool candidateReadyLocked(const DomainState& domain, uint64_t referenceSofTscNs, uint64_t framePeriodNs) const + { + if (domain.candidates.empty()) + return false; + if (referenceSofTscNs == 0) + return true; + const uint64_t newestTscNs = candidateSofTscNs(domain.candidates.back()); + if (newestTscNs == 0 || newestTscNs >= referenceSofTscNs || framePeriodNs == 0) + { + return true; + } + return referenceSofTscNs - newestTscNs <= framePeriodNs / 2ULL; + } + + bool movePendingToCaptured(std::vector>* pending, + std::vector* captured) + { + if (!pending || !captured || captured->size() != m_streams.size()) + return false; + for (size_t i = 0; i < pending->size(); ++i) + { + PendingFrame& sample = *(*pending)[i]; + CameraState& camera = *m_cameras[sample.cameraFlatIndex]; + CapturedFrameSet& streamFrames = (*captured)[camera.streamIndex]; + if (camera.cameraIndex >= streamFrames.frames.size() || + camera.cameraIndex >= streamFrames.dmaBuffers.size() || camera.cameraIndex >= streamFrames.timings.size()) + { + return false; + } + streamFrames.frames[camera.cameraIndex].reset(sample.frame.release()); + streamFrames.dmaBuffers[camera.cameraIndex].reset(sample.dmaBuffer.release()); + streamFrames.timings[camera.cameraIndex] = sample.timing; + } + return true; + } + + void recordStats(StreamState* stream, const CapturedFrameSet& frames) + { + if (m_config.runtime.logIntervalSeconds == 0) + return; + const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); + if (stream->intervalFrames == 0) + stream->intervalStart = now; + ++stream->intervalFrames; + ++stream->totalFrames; + for (size_t cameraIndex = 0; cameraIndex < frames.timings.size() && cameraIndex < stream->acquireSamples.size(); + ++cameraIndex) + { + const FrameTiming& timing = frames.timings[cameraIndex]; + if (stream->hasLastSelectedFrameNumber[cameraIndex] && + timing.frameNumber > stream->lastSelectedFrameNumbers[cameraIndex] + 1) + { + stream->selectedFrameGapsInterval[cameraIndex] += + timing.frameNumber - stream->lastSelectedFrameNumbers[cameraIndex] - 1; + } + stream->lastSelectedFrameNumbers[cameraIndex] = timing.frameNumber; + stream->hasLastSelectedFrameNumber[cameraIndex] = true; + if (!timing.hasSensorSofTimestampTsc || timing.acquiredTscNs < timing.sensorSofTimestampTscNs) + { + continue; + } + const uint64_t delayNs = timing.acquiredTscNs - timing.sensorSofTimestampTscNs; + ++stream->acquireSamples[cameraIndex]; + stream->acquireSumNs[cameraIndex] += static_cast(delayNs); + stream->acquireMaxNs[cameraIndex] = std::max(stream->acquireMaxNs[cameraIndex], delayNs); + } + if (frames.timings.size() == 2 && frames.timings[0].hasSensorSofTimestampTsc && + frames.timings[1].hasSensorSofTimestampTsc) + { + const int64_t value = + deltaNs(frames.timings[1].sensorSofTimestampTscNs, frames.timings[0].sensorSofTimestampTscNs); + if (stream->intervalTscSamples == 0) + { + stream->intervalTscMinNs = value; + stream->intervalTscMaxNs = value; + } + else + { + stream->intervalTscMinNs = std::min(stream->intervalTscMinNs, value); + stream->intervalTscMaxNs = std::max(stream->intervalTscMaxNs, value); + } + stream->intervalTscSumNs += static_cast(value); + ++stream->intervalTscSamples; + } + const double elapsed = std::chrono::duration(now - stream->intervalStart).count(); + if (elapsed < static_cast(m_config.runtime.logIntervalSeconds)) + return; + if (verboseLoggingEnabled() || stream->config.type == STREAM_TYPE_STEREO) + { + std::printf("argus_sender[capture:%s]: %.2f FPS, total=%llu", stream->config.id.c_str(), + static_cast(stream->intervalFrames) / elapsed, + static_cast(stream->totalFrames)); + if (stream->intervalTscSamples > 0) + { + std::printf(", tsc_delta_us(avg/min/max,right-left)=%.3f/%.3f/%.3f", + stream->intervalTscSumNs / static_cast(stream->intervalTscSamples) / 1000.0, + static_cast(stream->intervalTscMinNs) / 1000.0, + static_cast(stream->intervalTscMaxNs) / 1000.0); + } + if (verboseLoggingEnabled()) + { + std::printf(", camera_settings(latest exposure_us/analog_gain/isp_digital_gain)="); + for (size_t cameraIndex = 0; cameraIndex < frames.timings.size(); ++cameraIndex) + { + const FrameTiming& timing = frames.timings[cameraIndex]; + const CameraState& camera = *m_cameras[stream->cameraIndices[cameraIndex]]; + std::printf("%s%s:", cameraIndex == 0 ? "" : ",", camera.label.c_str()); + if (timing.hasSensorExposureTime) + { + std::printf("%.3f", static_cast(timing.sensorExposureTimeNs) / 1000.0); + } + else + std::printf("n/a"); + std::printf("/"); + if (timing.hasSensorAnalogGain) + std::printf("%.3f", timing.sensorAnalogGain); + else + std::printf("n/a"); + std::printf("/"); + if (timing.hasIspDigitalGain) + std::printf("%.3f", timing.ispDigitalGain); + else + std::printf("n/a"); + } + std::printf(", awb(latest mode/cct_K/gain_r:g_even:g_odd:b)="); + for (size_t cameraIndex = 0; cameraIndex < frames.timings.size(); ++cameraIndex) + { + const FrameTiming& timing = frames.timings[cameraIndex]; + const CameraState& camera = *m_cameras[stream->cameraIndices[cameraIndex]]; + std::printf("%s%s:", cameraIndex == 0 ? "" : ",", camera.label.c_str()); + if (timing.hasAwbMode) + std::printf("%s", timing.awbModeName); + else + std::printf("n/a"); + std::printf("/"); + if (timing.hasAwbCct) + std::printf("%u", timing.awbCct); + else + std::printf("n/a"); + std::printf("/"); + if (timing.hasAwbGains) + { + std::printf("%.3f:%.3f:%.3f:%.3f", timing.awbGainR, timing.awbGainGEven, timing.awbGainGOdd, + timing.awbGainB); + } + else + std::printf("n/a"); + } + } + std::printf("\n"); + } + std::printf("argus_sender[sof-acquire:%s]: ", stream->config.id.c_str()); + for (size_t cameraIndex = 0; cameraIndex < stream->acquireSamples.size(); ++cameraIndex) + { + const CameraState& camera = *m_cameras[stream->cameraIndices[cameraIndex]]; + std::printf("%s%s=", cameraIndex == 0 ? "" : ",", camera.label.c_str()); + if (stream->acquireSamples[cameraIndex] > 0) + { + std::printf( + "%.3f/%.3fms(avg/max,n=%llu)", + stream->acquireSumNs[cameraIndex] / static_cast(stream->acquireSamples[cameraIndex]) / 1.0e6, + static_cast(stream->acquireMaxNs[cameraIndex]) / 1.0e6, + static_cast(stream->acquireSamples[cameraIndex])); + } + else + { + std::printf("n/a"); + } + std::printf("[selected_frame_gaps=%llu]", + static_cast(stream->selectedFrameGapsInterval[cameraIndex])); + stream->selectedFrameGapsInterval[cameraIndex] = 0; + stream->acquireSamples[cameraIndex] = 0; + stream->acquireSumNs[cameraIndex] = 0.0; + stream->acquireMaxNs[cameraIndex] = 0; + } + std::printf("\n"); + stream->intervalStart = now; + stream->intervalFrames = 0; + stream->intervalTscSamples = 0; + stream->intervalTscSumNs = 0.0; + } + + void cleanup() + { + for (size_t i = 0; i < m_domains.size(); ++i) + { + if (!m_domains[i]) + continue; + m_domains[i]->pending.clear(); + m_domains[i]->candidates.clear(); + for (size_t eye = 0; eye < m_domains[i]->headFrames.size(); ++eye) + { + m_domains[i]->headFrames[eye].reset(); + } + for (size_t sessionIndex = 0; sessionIndex < m_domains[i]->sessions.size(); ++sessionIndex) + { + CaptureSessionState& session = *m_domains[i]->sessions[sessionIndex]; + session.request.reset(); + session.eventQueue.reset(); + session.iEventQueue = NULL; + session.iEventProvider = NULL; + } + } + for (size_t i = 0; i < m_cameras.size(); ++i) + { + m_cameras[i]->consumer.reset(); + m_cameras[i]->iConsumer = NULL; + m_cameras[i]->argusBuffers.clear(); + m_cameras[i]->iBufferStream = NULL; + m_cameras[i]->dmaBuffers.clear(); + m_cameras[i]->outputStream.reset(); + } + for (size_t i = 0; i < m_domains.size(); ++i) + { + if (!m_domains[i]) + continue; + for (size_t sessionIndex = 0; sessionIndex < m_domains[i]->sessions.size(); ++sessionIndex) + { + m_domains[i]->sessions[sessionIndex]->session.reset(); + m_domains[i]->sessions[sessionIndex]->iSession = NULL; + } + m_domains[i]->sessions.clear(); + } + } + + AppConfig m_config; + std::vector> m_streams; + std::vector> m_cameras; + std::vector> m_cameraLookup; + std::vector> m_domains; + bool m_started; + std::atomic m_error; +}; + +CaptureManager::CaptureManager(const AppConfig& config) : m_impl(new Impl(config)) +{ +} + +CaptureManager::~CaptureManager() +{ +} +bool CaptureManager::initialize(Argus::CameraProvider* provider, EGLDisplay display) +{ + return m_impl->initialize(provider, display); +} +bool CaptureManager::startPriorityDomain(size_t priorityDomainIndex) +{ + return m_impl->startPriorityDomain(priorityDomainIndex); +} +bool CaptureManager::startSecondaryDomains(size_t priorityDomainIndex) +{ + return m_impl->startSecondaryDomains(priorityDomainIndex); +} +bool CaptureManager::discardDomainWarmup(size_t domainIndex, uint32_t frameCount) +{ + return m_impl->discardDomainWarmup(domainIndex, frameCount); +} +bool CaptureManager::discardWarmup(uint32_t frameCount) +{ + return m_impl->discardWarmup(frameCount); +} +bool CaptureManager::waitForDomainSynchronization(size_t domainIndex, + uint32_t maxFrameSets, + uint32_t requiredConsecutive, + uint64_t toleranceNs) +{ + return m_impl->waitForDomainSynchronization(domainIndex, maxFrameSets, requiredConsecutive, toleranceNs); +} +bool CaptureManager::requestDomainCapture(size_t domainIndex) +{ + return m_impl->requestDomainCapture(domainIndex); +} +void CaptureManager::prepareCaptured(std::vector* captured) const +{ + m_impl->prepareCaptured(captured); +} +bool CaptureManager::acquireDomain(size_t domainIndex, std::vector* captured) +{ + return m_impl->acquireDomain(domainIndex, captured); +} +bool CaptureManager::acquireDomainClosestTo(size_t domainIndex, + uint64_t referenceSofTscNs, + std::vector* captured) +{ + return m_impl->acquireDomainClosestTo(domainIndex, referenceSofTscNs, captured); +} +bool CaptureManager::reacquireDomain(size_t domainIndex, std::vector* captured) +{ + return m_impl->reacquireDomain(domainIndex, captured); +} +void CaptureManager::recordCaptured(size_t streamIndex, const CapturedFrameSet& frames) +{ + m_impl->recordCaptured(streamIndex, frames); +} +bool CaptureManager::stop() +{ + return m_impl->stop(); +} +void CaptureManager::releaseResources() +{ + m_impl->releaseResources(); +} +bool CaptureManager::hasError() const +{ + return m_impl->hasError(); +} +size_t CaptureManager::streamCount() const +{ + return m_impl->streamCount(); +} +size_t CaptureManager::domainCount() const +{ + return m_impl->domainCount(); +} +size_t CaptureManager::streamDomain(size_t streamIndex) const +{ + return m_impl->streamDomain(streamIndex); +} +size_t CaptureManager::domainFor(size_t streamIndex, size_t cameraIndex) const +{ + return m_impl->domainFor(streamIndex, cameraIndex); +} +bool CaptureManager::domainUsesContinuousFifo(size_t domainIndex) const +{ + return m_impl->domainUsesContinuousFifo(domainIndex); +} +bool CaptureManager::hasContinuousFifoDomains() const +{ + return m_impl->hasContinuousFifoDomains(); +} +void CaptureManager::logFifoCandidateStats() +{ + m_impl->logFifoCandidateStats(); +} +const ResolvedOutputGeometry& CaptureManager::geometry(size_t streamIndex) const +{ + return m_impl->geometry(streamIndex); +} +uint32_t CaptureManager::nominalFps(size_t streamIndex) const +{ + return m_impl->nominalFps(streamIndex); +} +uint64_t CaptureManager::framePeriodNs(size_t streamIndex) const +{ + return m_impl->framePeriodNs(streamIndex); +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/CaptureManager.h b/examples/camera_streamer/argus_sender/src/CaptureManager.h new file mode 100644 index 0000000000..487089b22f --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/CaptureManager.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_CAPTURE_MANAGER_H +#define ARGUS_SENDER_CAPTURE_MANAGER_H + +#include "Config.h" +#include "OutputGeometry.h" +#include "StereoTiming.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace argus_sender +{ + +class DmaBuffer; + +class CapturedDmaBuffer +{ +public: + CapturedDmaBuffer(); + ~CapturedDmaBuffer(); + CapturedDmaBuffer(CapturedDmaBuffer&& other) noexcept; + CapturedDmaBuffer& operator=(CapturedDmaBuffer&& other) noexcept; + + DmaBuffer* get() const; + DmaBuffer* release(); + void reset(DmaBuffer* buffer = NULL); + +private: + CapturedDmaBuffer(const CapturedDmaBuffer&); + CapturedDmaBuffer& operator=(const CapturedDmaBuffer&); + + DmaBuffer* m_buffer; +}; + +struct CapturedFrameSet +{ + std::vector> frames; + // HEVC streams capture directly into client-owned encoder DMABUFs. Each + // lease returns its buffer to Argus unless ownership is transferred to + // NVENC by StreamProcessor. + std::vector dmaBuffers; + std::vector timings; +}; + +class CaptureManager +{ +public: + explicit CaptureManager(const AppConfig& config); + ~CaptureManager(); + + bool initialize(Argus::CameraProvider* provider, EGLDisplay display); + bool startPriorityDomain(size_t priorityDomainIndex); + bool startSecondaryDomains(size_t priorityDomainIndex); + bool discardDomainWarmup(size_t domainIndex, uint32_t frameCount); + bool discardWarmup(uint32_t frameCount); + bool waitForDomainSynchronization(size_t domainIndex, + uint32_t maxFrameSets, + uint32_t requiredConsecutive, + uint64_t toleranceNs); + bool requestDomainCapture(size_t domainIndex); + void prepareCaptured(std::vector* captured) const; + bool acquireDomain(size_t domainIndex, std::vector* captured); + bool acquireDomainClosestTo(size_t domainIndex, uint64_t referenceSofTscNs, std::vector* captured); + bool reacquireDomain(size_t domainIndex, std::vector* captured); + void recordCaptured(size_t streamIndex, const CapturedFrameSet& frames); + bool stop(); + void releaseResources(); + bool hasError() const; + + size_t streamCount() const; + size_t domainCount() const; + size_t streamDomain(size_t streamIndex) const; + size_t domainFor(size_t streamIndex, size_t cameraIndex) const; + bool domainUsesContinuousFifo(size_t domainIndex) const; + bool hasContinuousFifoDomains() const; + void logFifoCandidateStats(); + const ResolvedOutputGeometry& geometry(size_t streamIndex) const; + uint32_t nominalFps(size_t streamIndex) const; + uint64_t framePeriodNs(size_t streamIndex) const; + +private: + class Impl; + std::unique_ptr m_impl; +}; + +} // namespace argus_sender + +#endif // ARGUS_SENDER_CAPTURE_MANAGER_H diff --git a/examples/camera_streamer/argus_sender/src/EGLGlobal.cpp b/examples/camera_streamer/argus_sender/src/EGLGlobal.cpp new file mode 100644 index 0000000000..569768b0e8 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/EGLGlobal.cpp @@ -0,0 +1,200 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Minimal headless EGL helper for argus_sender, derived from NVIDIA's + * Argus sample EGLGlobal utility. + */ + +#include "EGLGlobal.h" + +#include "Error.h" + +#include + +namespace ArgusSamples +{ + +#define EGL_EXTN_FUNC(_type, _name) _type _name = NULL; +EGL_EXTN_FUNC_LIST +#undef EGL_EXTN_FUNC + +EGLDisplayHolder::EGLDisplayHolder(bool disableWindow) : m_display(EGL_NO_DISPLAY), m_disableWindow(disableWindow) +{ +} + +EGLDisplayHolder::~EGLDisplayHolder() +{ + cleanup(); +} + +bool EGLDisplayHolder::initialize(EGLNativeDisplayType native) +{ + m_display = eglGetDisplay(native); + +#ifdef EGL_EXT_platform_device + if (m_display == EGL_NO_DISPLAY) + { + const char* eglClientExtensions = eglQueryString(EGL_NO_DISPLAY, EGL_EXTENSIONS); + if (eglClientExtensions) + { + if (!std::strstr(eglClientExtensions, "EGL_EXT_client_extensions") || + !std::strstr(eglClientExtensions, "EGL_EXT_device_base") || + !std::strstr(eglClientExtensions, "EGL_EXT_platform_base") || + !std::strstr(eglClientExtensions, "EGL_EXT_platform_device")) + { + ORIGINATE_ERROR("EXT_platform_device extension missing"); + } + +# define EGL_EXTN(_type, _name) _type _name = (_type)eglGetProcAddress(# _name); + EGL_EXTN(PFNEGLQUERYDEVICESEXTPROC, eglQueryDevicesEXT); + EGL_EXTN(PFNEGLGETPLATFORMDISPLAYEXTPROC, eglGetPlatformDisplayEXT); +# undef EGL_EXTN + + EGLDeviceEXT device; + EGLint numDevices; + if (!eglQueryDevicesEXT(1, &device, &numDevices) || numDevices != 1) + ORIGINATE_ERROR("No EGL device available"); + + m_display = eglGetPlatformDisplayEXT(EGL_PLATFORM_DEVICE_EXT, device, NULL); + } + } +#endif + + if (m_display == EGL_NO_DISPLAY) + ORIGINATE_ERROR("Could not get EGL display"); + + if (!eglInitialize(m_display, 0, 0)) + ORIGINATE_ERROR("Could not initialize EGL display"); + + const char* eglExtensions = eglQueryString(m_display, EGL_EXTENSIONS); + if (!eglExtensions) + ORIGINATE_ERROR("Could not query EGL extensions"); + if (!std::strstr(eglExtensions, "EGL_KHR_stream")) + ORIGINATE_ERROR("EGL_KHR_stream not supported"); + if (!std::strstr(eglExtensions, "EGL_KHR_stream_consumer_gltexture")) + ORIGINATE_ERROR("EGL_KHR_stream_consumer_gltexture not supported"); + if (!std::strstr(eglExtensions, "EGL_KHR_stream_producer_eglsurface")) + ORIGINATE_ERROR("EGL_KHR_stream_producer_eglsurface not supported"); + if (!std::strstr(eglExtensions, "EGL_KHR_fence_sync")) + ORIGINATE_ERROR("EGL_KHR_fence_sync not supported"); + if (!std::strstr(eglExtensions, "EGL_KHR_reusable_sync")) + ORIGINATE_ERROR("EGL_KHR_reusable_sync not supported"); + +#define EGL_EXTN_FUNC(_type, _name) \ + _name = (_type)eglGetProcAddress(#_name); \ + if (!_name) \ + ORIGINATE_ERROR("Failed to get EGL function: " #_name); + EGL_EXTN_FUNC_LIST +#undef EGL_EXTN_FUNC + + return true; +} + +bool EGLDisplayHolder::cleanup() +{ + if (m_display != EGL_NO_DISPLAY) + { + if (eglTerminate(m_display) != EGL_TRUE) + REPORT_ERROR("eglTerminate failed (error 0x%04x)", eglGetError()); + m_display = EGL_NO_DISPLAY; + } + return true; +} + +EGLDisplay EGLDisplayHolder::get() const +{ + return m_display; +} + +EGLStreamHolder::EGLStreamHolder() : m_display(EGL_NO_DISPLAY), m_stream(EGL_NO_STREAM_KHR) +{ +} + +EGLStreamHolder::~EGLStreamHolder() +{ + destroy(); +} + +bool EGLStreamHolder::create(EGLDisplay display) +{ + if (m_stream != EGL_NO_STREAM_KHR) + ORIGINATE_ERROR("Already created"); + + m_display = display; + m_stream = eglCreateStreamKHR(m_display, NULL); + if (m_stream == EGL_NO_STREAM_KHR) + ORIGINATE_ERROR("Unable to create stream"); + + CHECK_STREAM_STATE(*this, CREATED); + return true; +} + +bool EGLStreamHolder::destroy() +{ + if (m_stream != EGL_NO_STREAM_KHR) + { + if (!eglDestroyStreamKHR(m_display, m_stream)) + ORIGINATE_ERROR("Error destroying EGLStream"); + m_stream = EGL_NO_STREAM_KHR; + } + return true; +} + +EGLStreamKHR EGLStreamHolder::get() const +{ + return m_stream; +} + +EGLint EGLStreamHolder::getState() const +{ + EGLint streamState = 0; + eglQueryStreamKHR(m_display, m_stream, EGL_STREAM_STATE_KHR, &streamState); + return streamState; +} + +EGLImageHolder::EGLImageHolder(EGLDisplay eglDisplay, EGLImageKHR eglImage) : m_display(eglDisplay), m_image(eglImage) +{ +} + +EGLImageHolder::~EGLImageHolder() +{ + if (m_display != EGL_NO_DISPLAY && m_image != EGL_NO_IMAGE_KHR) + eglDestroyImageKHR(m_display, m_image); +} + +EGLImageKHR EGLImageHolder::getImage() const +{ + return m_image; +} + +} // namespace ArgusSamples diff --git a/examples/camera_streamer/argus_sender/src/EGLGlobal.h b/examples/camera_streamer/argus_sender/src/EGLGlobal.h new file mode 100644 index 0000000000..fdab944f18 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/EGLGlobal.h @@ -0,0 +1,186 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef EGLGLOBAL_H +#define EGLGLOBAL_H + +#include +#include + +namespace ArgusSamples +{ + +#ifdef EGL_KHR_stream +# define EGL_KHR_STREAM_FUNCS \ + EGL_EXTN_FUNC(PFNEGLCREATESTREAMKHRPROC, eglCreateStreamKHR) \ + EGL_EXTN_FUNC(PFNEGLDESTROYSTREAMKHRPROC, eglDestroyStreamKHR) \ + EGL_EXTN_FUNC(PFNEGLSTREAMATTRIBKHRPROC, eglStreamAttribKHR) \ + EGL_EXTN_FUNC(PFNEGLQUERYSTREAMKHRPROC, eglQueryStreamKHR) +#else +# define EGL_KHR_STREAM_FUNCS +#endif + +#ifdef EGL_KHR_stream_consumer_gltexture +# define EGL_KHR_STREAM_CONSUMER_GLTEXTURE_FUNCS \ + EGL_EXTN_FUNC(PFNEGLSTREAMCONSUMERGLTEXTUREEXTERNALKHRPROC, eglStreamConsumerGLTextureExternalKHR) \ + EGL_EXTN_FUNC(PFNEGLSTREAMCONSUMERACQUIREKHRPROC, eglStreamConsumerAcquireKHR) \ + EGL_EXTN_FUNC(PFNEGLSTREAMCONSUMERRELEASEKHRPROC, eglStreamConsumerReleaseKHR) +#else +# define EGL_KHR_STREAM_CONSUMER_GLTEXTURE_FUNCS +#endif + +#ifdef EGL_KHR_stream_producer_eglsurface +# define EGL_KHR_STREAM_PRODUCER_EGLSURFACE_FUNCS \ + EGL_EXTN_FUNC(PFNEGLCREATESTREAMPRODUCERSURFACEKHRPROC, eglCreateStreamProducerSurfaceKHR) +#else +# define EGL_KHR_STREAM_PRODUCER_EGLSURFACE_FUNCS +#endif + +#ifdef EGL_KHR_fence_sync +# define EGL_KHR_FENCE_SYNC_FUNCS \ + EGL_EXTN_FUNC(PFNEGLCREATESYNCKHRPROC, eglCreateSyncKHR) \ + EGL_EXTN_FUNC(PFNEGLDESTROYSYNCKHRPROC, eglDestroySyncKHR) \ + EGL_EXTN_FUNC(PFNEGLCLIENTWAITSYNCKHRPROC, eglClientWaitSyncKHR) \ + EGL_EXTN_FUNC(PFNEGLGETSYNCATTRIBKHRPROC, eglGetSyncAttribKHR) +#else +# define EGL_KHR_FENCE_SYNC_FUNCS +#endif + +#ifdef EGL_NV_stream_sync +# define EGL_NV_STREAM_SYNC_FUNCS EGL_EXTN_FUNC(PFNEGLCREATESTREAMSYNCNVPROC, eglCreateStreamSyncNV) +#else +# define EGL_NV_STREAM_SYNC_FUNCS +#endif + +#ifdef EGL_KHR_reusable_sync +# define EGL_KHR_REUSABLE_SYNC_FUNCS EGL_EXTN_FUNC(PFNEGLSIGNALSYNCKHRPROC, eglSignalSyncKHR) +#else +# define EGL_KHR_REUSABLE_SYNC_FUNCS +#endif + +#ifdef EGL_KHR_image +# define EGL_KHR_IMAGE_FUNCS \ + EGL_EXTN_FUNC(PFNEGLCREATEIMAGEKHRPROC, eglCreateImageKHR) \ + EGL_EXTN_FUNC(PFNEGLDESTROYIMAGEKHRPROC, eglDestroyImageKHR) +#else +# define EGL_KHR_IMAGE_FUNCS +#endif + +#ifdef EGL_KHR_wait_sync +# define EGL_KHR_WAIT_SYNC_FUNCS EGL_EXTN_FUNC(PFNEGLWAITSYNCKHRPROC, eglWaitSyncKHR) +#else +# define EGL_KHR_WAIT_SYNC_FUNCS +#endif + + +#define EGL_EXTN_FUNC_LIST \ + EGL_KHR_STREAM_FUNCS \ + EGL_KHR_STREAM_CONSUMER_GLTEXTURE_FUNCS \ + EGL_KHR_STREAM_PRODUCER_EGLSURFACE_FUNCS \ + EGL_KHR_FENCE_SYNC_FUNCS \ + EGL_NV_STREAM_SYNC_FUNCS \ + EGL_KHR_REUSABLE_SYNC_FUNCS \ + EGL_KHR_IMAGE_FUNCS \ + EGL_KHR_WAIT_SYNC_FUNCS + +#define EGL_EXTN_FUNC(_type, _name) extern _type _name; +EGL_EXTN_FUNC_LIST +#undef EGL_EXTN_FUNC + +/*! + * A class to initialize and cleanup an EGL display. + */ +class EGLDisplayHolder +{ +public: + EGLDisplayHolder(bool disableWindow = false); + ~EGLDisplayHolder(); + + bool initialize(EGLNativeDisplayType native = EGL_DEFAULT_DISPLAY); + bool cleanup(); + + EGLDisplay get() const; + +private: + EGLDisplay m_display; + bool m_disableWindow; +}; + +/*! + * A class to wrap and destroy an EGL image. + */ +class EGLImageHolder +{ +public: + EGLImageHolder(EGLDisplay eglDisplay, EGLImageKHR eglImage); + ~EGLImageHolder(); + + EGLImageKHR getImage() const; + +private: + EGLDisplay m_display; + EGLImageKHR m_image; +}; + +/*! + * A class to create and destroy an EGL stream. + */ +class EGLStreamHolder +{ +public: + EGLStreamHolder(); + ~EGLStreamHolder(); + + bool create(EGLDisplay display); + bool destroy(); + + EGLStreamKHR get() const; + + EGLint getState() const; + +private: + EGLDisplay m_display; + EGLStreamKHR m_stream; +}; + +// Helper macro to ensure the stream is in the given state. +#define CHECK_STREAM_STATE(stream, state) \ + do \ + { \ + EGLint state = (stream).getState(); \ + if (state != EGL_STREAM_STATE_##state##_KHR) \ + ORIGINATE_ERROR("Stream not in " #state " state but in state 0x%x", state); \ + } while (0) + +} // namespace ArgusSamples + +#endif // EGLGLOBAL_H diff --git a/examples/camera_streamer/argus_sender/src/EncodedFrameSink.h b/examples/camera_streamer/argus_sender/src/EncodedFrameSink.h new file mode 100644 index 0000000000..3a413b2f4c --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/EncodedFrameSink.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_ENCODED_FRAME_SINK_H +#define ARGUS_SENDER_ENCODED_FRAME_SINK_H + +#include +#include + +namespace argus_sender +{ + +class EncodedFrameSink +{ +public: + virtual ~EncodedFrameSink() + { + } + virtual bool enqueueEncoded( + const uint8_t* data, size_t size, uint64_t frameSeq, uint64_t encodeDoneNs, uint32_t senderClockDomain) = 0; + virtual bool hasError() const = 0; +}; + +} // namespace argus_sender + +#endif // ARGUS_SENDER_ENCODED_FRAME_SINK_H diff --git a/examples/camera_streamer/argus_sender/src/Error.h b/examples/camera_streamer/argus_sender/src/Error.h new file mode 100644 index 0000000000..e7c866c1ab --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/Error.h @@ -0,0 +1,80 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of NVIDIA CORPORATION nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef ERROR_H +#define ERROR_H + +#include + +#define LOG_ERROR(_file, _func, _line, _str, ...) \ + do \ + { \ + fprintf(stderr, "Error generated. %s, %s:%d ", _file, _func, _line); \ + fprintf(stderr, _str, ##__VA_ARGS__); \ + fprintf(stderr, "\n"); \ + } while (0) + +/** + * Simply report an error. + */ +#define REPORT_ERROR(_str, ...) \ + do \ + { \ + LOG_ERROR(__FILE__, __FUNCTION__, __LINE__, (_str), ##__VA_ARGS__); \ + } while (0) + +/** + * Report and return an error that was first detected in the current method. + */ +#define ORIGINATE_ERROR(_str, ...) \ + do \ + { \ + LOG_ERROR(__FILE__, __FUNCTION__, __LINE__, (_str), ##__VA_ARGS__); \ + return false; \ + } while (0) + +/** + * Report and return an error that was first detected in some method + * called by the current method. + */ +#define PROPAGATE_ERROR(_err) \ + do \ + { \ + bool peResult = (_err); \ + if (peResult != true) \ + { \ + LOG_ERROR(__FILE__, __FUNCTION__, __LINE__, "(propagating)"); \ + return false; \ + } \ + } while (0) + +#endif // ERROR_H diff --git a/examples/camera_streamer/argus_sender/src/FrameBundleAssembler.cpp b/examples/camera_streamer/argus_sender/src/FrameBundleAssembler.cpp new file mode 100644 index 0000000000..ea3555c19e --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/FrameBundleAssembler.cpp @@ -0,0 +1,517 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "FrameBundleAssembler.h" + +#include "EncodedFrameSink.h" +#include "PacketSink.h" +#include "Runtime.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace argus_sender +{ + +#ifdef ARGUS_SENDER_TEST_SUBMISSION_HOOK +void observeSubmissionLock(std::mutex& mutex, uint64_t bundleSequence); +#endif + +namespace +{ + +uint64_t steadyNowNs() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +} // namespace + +class FrameBundleAssembler::Impl +{ +public: + class SampleSink : public EncodedFrameSink + { + public: + SampleSink(Impl* parent, size_t streamIndex, size_t cameraIndex) + : m_parent(parent), m_streamIndex(streamIndex), m_cameraIndex(cameraIndex) + { + } + + bool enqueueEncoded(const uint8_t* data, + size_t size, + uint64_t frameSeq, + uint64_t encodeDoneNs, + uint32_t senderClockDomain) override + { + return m_parent && m_parent->submitEncoded( + m_streamIndex, m_cameraIndex, frameSeq, data, size, encodeDoneNs, senderClockDomain); + } + + bool hasError() const override + { + return !m_parent || m_parent->hasError(); + } + + private: + Impl* m_parent; + size_t m_streamIndex; + size_t m_cameraIndex; + }; + + Impl(PacketSink* packetSink, uint64_t sessionId, const std::vector& streamTemplates) + : m_packetSink(packetSink), + m_sessionId(sessionId), + m_streamTemplates(streamTemplates), + m_nextSendSequence(1), + m_error(false) + { + m_sinks.resize(streamTemplates.size()); + for (size_t stream = 0; stream < streamTemplates.size(); ++stream) + { + for (size_t camera = 0; camera < streamTemplates[stream].samples.size(); ++camera) + { + m_sinks[stream].push_back(std::unique_ptr(new SampleSink(this, stream, camera))); + } + } + } + + EncodedFrameSink* encodedSink(size_t streamIndex, size_t cameraIndex) + { + if (streamIndex >= m_sinks.size() || cameraIndex >= m_sinks[streamIndex].size()) + { + return NULL; + } + return m_sinks[streamIndex][cameraIndex].get(); + } + + bool beginBundle(uint64_t bundleSequence, + const std::vector& inputs, + uint64_t senderCaptureReadyNs, + uint64_t senderProcessSubmitNs, + uint32_t senderClockDomain, + uint64_t senderCaptureReadyTscNs) + { + if (bundleSequence == 0 || inputs.size() != m_streamTemplates.size()) + return setError("invalid AVS1 bundle registration"); + + PendingBundle pending; + pending.captureReadySteadyNs = steadyNowNs(); + pending.timing.captureStartSteadyNs = pending.captureReadySteadyNs; + pending.timing.streams.resize(inputs.size()); + uint64_t earliestAcquiredSteadyNs = 0; + uint64_t latestAcquiredSteadyNs = 0; + pending.bundle.sequence = bundleSequence; + pending.bundle.sessionId = m_sessionId; + pending.bundle.senderCaptureReadyNs = senderCaptureReadyNs; + if (senderClockDomain != SENDER_CLOCK_UNKNOWN && senderCaptureReadyNs != 0 && senderProcessSubmitNs != 0) + { + pending.bundle.flags |= AVS1_BUNDLE_SENDER_CLOCK_VALID; + if (senderClockDomain == SENDER_CLOCK_TAI) + pending.bundle.flags |= AVS1_BUNDLE_SENDER_CLOCK_TAI; + pending.senderClockDomain = senderClockDomain; + if (senderCaptureReadyTscNs != 0) + { + pending.bundle.flags |= AVS1_BUNDLE_SENDER_TSC_VALID; + pending.bundle.senderCaptureReadyTscNs = senderCaptureReadyTscNs; + } + } + pending.bundle.streams = m_streamTemplates; + for (size_t streamIndex = 0; streamIndex < inputs.size(); ++streamIndex) + { + Avs1StreamRecord& stream = pending.bundle.streams[streamIndex]; + const BundleStreamInput& input = inputs[streamIndex]; + PacketStreamTiming& streamTiming = pending.timing.streams[streamIndex]; + streamTiming.present = input.present; + streamTiming.encoded = stream.codec == AVS1_CODEC_HEVC; + uint64_t streamEarliestAcquiredSteadyNs = 0; + uint64_t streamLatestAcquiredSteadyNs = 0; + stream.flags &= ~AVS1_STREAM_PRESENT; + stream.sequence = input.present ? input.sequence : 0; + if (input.present) + stream.flags |= AVS1_STREAM_PRESENT; + if (input.present && input.timings.size() != stream.samples.size()) + return setError("AVS1 present stream timing count mismatch"); + for (size_t cameraIndex = 0; cameraIndex < stream.samples.size(); ++cameraIndex) + { + PacketSampleTiming sampleTiming; + Avs1SampleRecord& sample = stream.samples[cameraIndex]; + sample.flags = 0; + sample.payload.clear(); + sample.frameNumber = 0; + sample.sofTscNs = 0; + sample.frameTimestampNs = 0; + sample.senderCaptureReadyNs = 0; + sample.senderProcessSubmitNs = 0; + sample.senderProcessDoneNs = 0; + if (!input.present) + { + pending.timing.samples.push_back(sampleTiming); + continue; + } + const FrameTiming& timing = input.timings[cameraIndex]; + if (timing.acquiredSteadyNs != 0) + { + if (earliestAcquiredSteadyNs == 0 || timing.acquiredSteadyNs < earliestAcquiredSteadyNs) + { + earliestAcquiredSteadyNs = timing.acquiredSteadyNs; + } + if (timing.acquiredSteadyNs > latestAcquiredSteadyNs) + latestAcquiredSteadyNs = timing.acquiredSteadyNs; + if (streamEarliestAcquiredSteadyNs == 0 || timing.acquiredSteadyNs < streamEarliestAcquiredSteadyNs) + { + streamEarliestAcquiredSteadyNs = timing.acquiredSteadyNs; + } + if (timing.acquiredSteadyNs > streamLatestAcquiredSteadyNs) + { + streamLatestAcquiredSteadyNs = timing.acquiredSteadyNs; + } + } + sample.flags |= AVS1_SAMPLE_PRESENT; + sample.frameNumber = timing.frameNumber; + if (timing.hasSensorSofTimestampTsc) + { + sample.flags |= AVS1_SAMPLE_SOF_TSC_VALID; + sample.sofTscNs = timing.sensorSofTimestampTscNs; + sampleTiming.present = true; + sampleTiming.sensorSofTscNs = timing.sensorSofTimestampTscNs; + } + pending.timing.samples.push_back(sampleTiming); + if (timing.frameTimeNs != 0) + { + sample.flags |= AVS1_SAMPLE_FRAME_TIMESTAMP_VALID; + sample.frameTimestampNs = timing.frameTimeNs; + } + if (pending.bundle.flags & AVS1_BUNDLE_SENDER_CLOCK_VALID) + { + sample.flags |= AVS1_SAMPLE_SENDER_TIMING_VALID; + sample.senderCaptureReadyNs = senderCaptureReadyNs; + sample.senderProcessSubmitNs = senderProcessSubmitNs; + } + ++pending.expectedPayloads; + } + if (input.present) + { + streamTiming.captureStartSteadyNs = + streamEarliestAcquiredSteadyNs != 0 ? streamEarliestAcquiredSteadyNs : pending.captureReadySteadyNs; + if (streamEarliestAcquiredSteadyNs != 0) + { + streamTiming.acquireSpreadNs = streamLatestAcquiredSteadyNs - streamEarliestAcquiredSteadyNs; + } + } + } + if (earliestAcquiredSteadyNs != 0) + { + pending.timing.captureStartSteadyNs = earliestAcquiredSteadyNs; + pending.timing.acquireSpreadNs = latestAcquiredSteadyNs - earliestAcquiredSteadyNs; + } + + std::vector ready; + std::lock_guard sendLock(m_sendMutex); + { + std::lock_guard lock(m_mutex); + if (m_pending.count(bundleSequence) != 0) + return setError("duplicate AVS1 bundle registration"); + m_pending[bundleSequence] = std::move(pending); + collectReadyLocked(&ready); + } + return sendReady(&ready); + } + + bool recordCopyDone(size_t streamIndex, uint64_t bundleSequence, uint64_t copyDoneSteadyNs) + { + if (bundleSequence == 0 || copyDoneSteadyNs == 0) + return setError("invalid AVS1 copy-done timestamp"); + std::lock_guard lock(m_mutex); + std::map::iterator it = m_pending.find(bundleSequence); + if (it == m_pending.end()) + return setError("AVS1 copy done has no registered bundle"); + if (streamIndex >= it->second.timing.streams.size()) + return setError("AVS1 copy done stream is out of range"); + PacketStreamTiming& stream = it->second.timing.streams[streamIndex]; + if (!stream.present || copyDoneSteadyNs < stream.captureStartSteadyNs) + { + return setError("invalid AVS1 per-stream copy-done timestamp"); + } + stream.copyDoneSteadyNs = std::max(stream.copyDoneSteadyNs, copyDoneSteadyNs); + return true; + } + + bool recordProcessStart(size_t streamIndex, uint64_t bundleSequence, uint64_t processStartSteadyNs) + { + if (bundleSequence == 0 || processStartSteadyNs == 0) + return setError("invalid AVS1 process-start timestamp"); + std::lock_guard lock(m_mutex); + std::map::iterator it = m_pending.find(bundleSequence); + if (it == m_pending.end()) + return setError("AVS1 process start has no registered bundle"); + PacketTiming& timing = it->second.timing; + if (streamIndex >= timing.streams.size()) + return setError("AVS1 process start stream is out of range"); + PacketStreamTiming& stream = timing.streams[streamIndex]; + if (!stream.present || processStartSteadyNs < stream.captureStartSteadyNs || + processStartSteadyNs < timing.captureStartSteadyNs) + { + return setError("invalid AVS1 per-stream process-start timestamp"); + } + if (stream.processStartSteadyNs == 0 || processStartSteadyNs < stream.processStartSteadyNs) + { + stream.processStartSteadyNs = processStartSteadyNs; + } + const uint64_t latencyNs = processStartSteadyNs - timing.captureStartSteadyNs; + if (timing.captureToEncodeStartNs == 0 || latencyNs < timing.captureToEncodeStartNs) + { + timing.captureToEncodeStartNs = latencyNs; + } + return true; + } + + bool submitRaw(size_t streamIndex, + size_t cameraIndex, + uint64_t bundleSequence, + std::vector&& payload, + uint64_t processDoneNs, + uint32_t senderClockDomain) + { + return submitOwned( + streamIndex, cameraIndex, bundleSequence, std::move(payload), processDoneNs, senderClockDomain); + } + + bool submitEncoded(size_t streamIndex, + size_t cameraIndex, + uint64_t bundleSequence, + const uint8_t* data, + size_t size, + uint64_t processDoneNs, + uint32_t senderClockDomain) + { + if (!data || size == 0) + return setError("empty AVS1 encoded payload"); + std::vector owned(data, data + size); + return submitOwned(streamIndex, cameraIndex, bundleSequence, std::move(owned), processDoneNs, senderClockDomain); + } + + bool hasError() const + { + return m_error.load() || !m_packetSink || m_packetSink->hasError(); + } + +private: + struct ReadyBundle + { + Avs1Bundle bundle; + PacketTiming timing; + }; + + struct PendingBundle + { + Avs1Bundle bundle; + PacketTiming timing; + uint64_t captureReadySteadyNs = 0; + uint32_t senderClockDomain = SENDER_CLOCK_UNKNOWN; + size_t expectedPayloads = 0; + size_t receivedPayloads = 0; + }; + + bool submitOwned(size_t streamIndex, + size_t cameraIndex, + uint64_t bundleSequence, + std::vector&& payload, + uint64_t processDoneNs, + uint32_t senderClockDomain) + { + if (payload.empty()) + return setError("empty AVS1 sample payload"); + const uint64_t payloadDoneSteadyNs = steadyNowNs(); + std::vector ready; +#ifdef ARGUS_SENDER_TEST_SUBMISSION_HOOK + observeSubmissionLock(m_sendMutex, bundleSequence); +#endif + std::lock_guard sendLock(m_sendMutex); + { + std::lock_guard lock(m_mutex); + std::map::iterator it = m_pending.find(bundleSequence); + if (it == m_pending.end()) + return setError("AVS1 payload has no registered bundle"); + PendingBundle& pending = it->second; + if (streamIndex >= pending.bundle.streams.size() || + cameraIndex >= pending.bundle.streams[streamIndex].samples.size()) + { + return setError("AVS1 payload index is out of range"); + } + if (streamIndex >= pending.timing.streams.size()) + return setError("AVS1 payload timing stream is out of range"); + PacketStreamTiming& streamTiming = pending.timing.streams[streamIndex]; + if (!streamTiming.present || payloadDoneSteadyNs < streamTiming.captureStartSteadyNs) + { + return setError("invalid AVS1 per-stream payload-ready timestamp"); + } + streamTiming.processDoneSteadyNs = std::max(streamTiming.processDoneSteadyNs, payloadDoneSteadyNs); + Avs1SampleRecord& sample = pending.bundle.streams[streamIndex].samples[cameraIndex]; + if (!(sample.flags & AVS1_SAMPLE_PRESENT) || !sample.payload.empty()) + return setError("duplicate or unexpected AVS1 sample payload"); + sample.payload = std::move(payload); + sample.senderProcessDoneNs = processDoneNs; + uint64_t latencyNs = 0; + if (senderClockDomain == pending.senderClockDomain && pending.bundle.senderCaptureReadyNs != 0 && + processDoneNs >= pending.bundle.senderCaptureReadyNs && + pending.captureReadySteadyNs >= pending.timing.captureStartSteadyNs) + { + latencyNs = pending.captureReadySteadyNs - pending.timing.captureStartSteadyNs + processDoneNs - + pending.bundle.senderCaptureReadyNs; + } + else if (payloadDoneSteadyNs >= pending.timing.captureStartSteadyNs) + { + latencyNs = payloadDoneSteadyNs - pending.timing.captureStartSteadyNs; + } + if (latencyNs > pending.timing.captureToProcessDoneNs) + pending.timing.captureToProcessDoneNs = latencyNs; + if (senderClockDomain != pending.senderClockDomain || processDoneNs == 0) + { + pending.bundle.flags &= ~AVS1_BUNDLE_SENDER_CLOCK_VALID; + pending.bundle.flags &= ~AVS1_BUNDLE_SENDER_CLOCK_TAI; + pending.bundle.flags &= ~AVS1_BUNDLE_SENDER_TSC_VALID; + pending.bundle.senderCaptureReadyTscNs = 0; + for (size_t s = 0; s < pending.bundle.streams.size(); ++s) + { + for (size_t c = 0; c < pending.bundle.streams[s].samples.size(); ++c) + { + pending.bundle.streams[s].samples[c].flags &= ~AVS1_SAMPLE_SENDER_TIMING_VALID; + } + } + } + ++pending.receivedPayloads; + collectReadyLocked(&ready); + } + return sendReady(&ready); + } + + void collectReadyLocked(std::vector* ready) + { + while (true) + { + std::map::iterator it = m_pending.find(m_nextSendSequence); + if (it == m_pending.end() || it->second.receivedPayloads != it->second.expectedPayloads) + { + break; + } + ReadyBundle completed; + completed.bundle = std::move(it->second.bundle); + completed.timing = it->second.timing; + ready->push_back(std::move(completed)); + m_pending.erase(it); + ++m_nextSendSequence; + } + } + + bool sendReady(std::vector* ready) + { + for (size_t i = 0; i < ready->size(); ++i) + { + uint32_t clockDomain = SENDER_CLOCK_UNKNOWN; + (*ready)[i].bundle.senderTcpEnqueueNs = readSenderClockNs(&clockDomain); + if (clockDomain == SENDER_CLOCK_UNKNOWN || (*ready)[i].bundle.senderTcpEnqueueNs == 0) + { + (*ready)[i].bundle.flags &= ~AVS1_BUNDLE_SENDER_CLOCK_VALID; + (*ready)[i].bundle.flags &= ~AVS1_BUNDLE_SENDER_CLOCK_TAI; + (*ready)[i].bundle.flags &= ~AVS1_BUNDLE_SENDER_TSC_VALID; + (*ready)[i].bundle.senderCaptureReadyTscNs = 0; + } + std::vector packet; + const char* wireError = NULL; + if (!buildAvs1WireRecord((*ready)[i].bundle, &packet, &wireError) || !m_packetSink || + !m_packetSink->enqueueFramedWithTiming(std::move(packet), (*ready)[i].timing)) + { + if (!stopRequested()) + { + std::fprintf(stderr, "Failed to emit AVS1 bundle: %s\n", wireError ? wireError : "packet sink error"); + m_error = true; + } + return false; + } + } + return true; + } + + bool setError(const char* message) + { + std::fprintf(stderr, "Frame bundle assembler: %s\n", message); + m_error = true; + return false; + } + + PacketSink* m_packetSink; + uint64_t m_sessionId; + std::vector m_streamTemplates; + std::vector>> m_sinks; + // Hold across ordered collection and enqueue. Lock order: send, then state. + std::mutex m_sendMutex; + mutable std::mutex m_mutex; + std::map m_pending; + uint64_t m_nextSendSequence; + std::atomic m_error; +}; + +FrameBundleAssembler::FrameBundleAssembler(PacketSink* packetSink, + uint64_t sessionId, + const std::vector& streamTemplates) + : m_impl(new Impl(packetSink, sessionId, streamTemplates)) +{ +} + +FrameBundleAssembler::~FrameBundleAssembler() +{ +} + +EncodedFrameSink* FrameBundleAssembler::encodedSink(size_t streamIndex, size_t cameraIndex) +{ + return m_impl->encodedSink(streamIndex, cameraIndex); +} + +bool FrameBundleAssembler::beginBundle(uint64_t bundleSequence, + const std::vector& streams, + uint64_t senderCaptureReadyNs, + uint64_t senderProcessSubmitNs, + uint32_t senderClockDomain, + uint64_t senderCaptureReadyTscNs) +{ + return m_impl->beginBundle(bundleSequence, streams, senderCaptureReadyNs, senderProcessSubmitNs, senderClockDomain, + senderCaptureReadyTscNs); +} + +bool FrameBundleAssembler::recordCopyDone(size_t streamIndex, uint64_t bundleSequence, uint64_t copyDoneSteadyNs) +{ + return m_impl->recordCopyDone(streamIndex, bundleSequence, copyDoneSteadyNs); +} + +bool FrameBundleAssembler::recordProcessStart(size_t streamIndex, uint64_t bundleSequence, uint64_t processStartSteadyNs) +{ + return m_impl->recordProcessStart(streamIndex, bundleSequence, processStartSteadyNs); +} + +bool FrameBundleAssembler::submitRaw(size_t streamIndex, + size_t cameraIndex, + uint64_t bundleSequence, + std::vector&& payload, + uint64_t processDoneNs, + uint32_t senderClockDomain) +{ + return m_impl->submitRaw( + streamIndex, cameraIndex, bundleSequence, std::move(payload), processDoneNs, senderClockDomain); +} + +bool FrameBundleAssembler::hasError() const +{ + return m_impl->hasError(); +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/FrameBundleAssembler.h b/examples/camera_streamer/argus_sender/src/FrameBundleAssembler.h new file mode 100644 index 0000000000..8800f4555c --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/FrameBundleAssembler.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_FRAME_BUNDLE_ASSEMBLER_H +#define ARGUS_SENDER_FRAME_BUNDLE_ASSEMBLER_H + +#include "Avs1Wire.h" +#include "StereoTiming.h" + +#include +#include +#include +#include + +namespace argus_sender +{ + +class EncodedFrameSink; +class PacketSink; + +struct BundleStreamInput +{ + bool present = false; + uint64_t sequence = 0; + std::vector timings; +}; + +class FrameBundleAssembler +{ +public: + FrameBundleAssembler(PacketSink* packetSink, uint64_t sessionId, const std::vector& streamTemplates); + ~FrameBundleAssembler(); + + EncodedFrameSink* encodedSink(size_t streamIndex, size_t cameraIndex); + bool beginBundle(uint64_t bundleSequence, + const std::vector& streams, + uint64_t senderCaptureReadyNs, + uint64_t senderProcessSubmitNs, + uint32_t senderClockDomain, + uint64_t senderCaptureReadyTscNs); + bool recordCopyDone(size_t streamIndex, uint64_t bundleSequence, uint64_t copyDoneSteadyNs); + bool recordProcessStart(size_t streamIndex, uint64_t bundleSequence, uint64_t processStartSteadyNs); + bool submitRaw(size_t streamIndex, + size_t cameraIndex, + uint64_t bundleSequence, + std::vector&& payload, + uint64_t processDoneNs, + uint32_t senderClockDomain); + bool hasError() const; + +private: + class Impl; + std::unique_ptr m_impl; +}; + +} // namespace argus_sender + +#endif // ARGUS_SENDER_FRAME_BUNDLE_ASSEMBLER_H diff --git a/examples/camera_streamer/argus_sender/src/FrameCopyWorker.cpp b/examples/camera_streamer/argus_sender/src/FrameCopyWorker.cpp new file mode 100644 index 0000000000..07b0510877 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/FrameCopyWorker.cpp @@ -0,0 +1,356 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "FrameCopyWorker.h" + +#include "HevcEncoder.h" +#include "NvRemap.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace argus_sender +{ +namespace +{ + +bool copyFrameToBuffer(EGLStream::Frame* frame, DmaBuffer* buffer, const char* label) +{ + EGLStream::IFrame* iFrame = Argus::interface_cast(frame); + if (!iFrame) + { + std::fprintf(stderr, "Failed to get %s EGLStream IFrame interface\n", label); + return false; + } + + EGLStream::NV::IImageNativeBuffer* iNativeBuffer = + Argus::interface_cast(iFrame->getImage()); + if (!iNativeBuffer) + { + std::fprintf(stderr, "%s IImageNativeBuffer not supported by Argus EGL image\n", label); + return false; + } + + if (iNativeBuffer->copyToNvBuffer(buffer->getFd(), EGLStream::NV::ROTATION_0) != Argus::STATUS_OK) + { + std::fprintf(stderr, "Failed to copy %s Argus EGL frame to raw processing DMABUF\n", label); + return false; + } + return true; +} + +// Produces the tight raw NV12 wire payload from the pitch-linear DMABUF: +// either a plain row pack of the whole buffer, or the LUT remap when the +// job carries a table. Runs on the worker thread. +bool packRawNv12(DmaBuffer* buffer, + const RawPackJob& pack, + std::vector* scratchY, + std::vector* scratchUV, + const char* label) +{ + if (!buffer || !pack.payload) + return false; + NvBufSurface* surface = NULL; + if (NvBufSurfaceFromFd(buffer->getFd(), reinterpret_cast(&surface)) != 0 || !surface) + { + std::fprintf(stderr, "Failed to resolve %s raw NV12 NvBufSurface\n", label); + return false; + } + if (NvBufSurfaceMap(surface, 0, -1, NVBUF_MAP_READ) != 0) + { + std::fprintf(stderr, "Failed to map %s raw NV12 NvBufSurface\n", label); + return false; + } + bool ok = true; + if (NvBufSurfaceSyncForCpu(surface, 0, -1) != 0) + { + std::fprintf(stderr, "Failed to synchronize %s raw NV12 for CPU\n", label); + ok = false; + } + const NvBufSurfacePlaneParams& planes = surface->surfaceList[0].planeParams; + if (ok && planes.num_planes < 2) + { + std::fprintf(stderr, "%s raw NV12 surface has %u plane(s), expected at least 2\n", label, planes.num_planes); + ok = false; + } + const uint8_t* sourceY = NULL; + const uint8_t* sourceUV = NULL; + if (ok) + { + sourceY = static_cast(surface->surfaceList[0].mappedAddr.addr[0]); + sourceUV = static_cast(surface->surfaceList[0].mappedAddr.addr[1]); + if (!sourceY || !sourceUV) + { + std::fprintf(stderr, "%s raw NV12 planes are not CPU-mapped\n", label); + ok = false; + } + } + if (ok && pack.remap) + { + const NvRemapTable& table = *pack.remap; + if (planes.width[0] < table.srcWidth || planes.height[0] < table.srcHeight) + { + std::fprintf(stderr, "%s remap expects a %ux%u source but the buffer is %ux%u\n", label, table.srcWidth, + table.srcHeight, planes.width[0], planes.height[0]); + ok = false; + } + else + { + // NvBufSurface CPU mappings are uncached/write-combined: random + // gathers there are an order of magnitude slower than cached + // reads. Stream-copy just the source rows the taps use into + // cached scratch memory first (sequential reads are fast), then + // gather from the scratch. + const uint32_t lumaRows = table.lumaRowLast - table.lumaRowFirst + 1; + const uint32_t chromaRows = table.chromaRowLast - table.chromaRowFirst + 1; + scratchY->resize(static_cast(table.srcWidth) * lumaRows); + scratchUV->resize(static_cast(table.srcWidth) * chromaRows); + for (uint32_t row = 0; row < lumaRows; ++row) + { + std::memcpy(scratchY->data() + static_cast(row) * table.srcWidth, + sourceY + static_cast(table.lumaRowFirst + row) * planes.pitch[0], table.srcWidth); + } + for (uint32_t row = 0; row < chromaRows; ++row) + { + std::memcpy(scratchUV->data() + static_cast(row) * table.srcWidth, + sourceUV + static_cast(table.chromaRowFirst + row) * planes.pitch[1], table.srcWidth); + } + pack.payload->resize(static_cast(table.outWidth) * table.outHeight * 3U / 2U); + nvRemapNv12(table, scratchY->data(), table.srcWidth, scratchUV->data(), table.srcWidth, + pack.payload->data(), table.lumaRowFirst, table.chromaRowFirst); + } + } + else if (ok) + { + pack.payload->resize(static_cast(pack.width) * pack.height * 3U / 2U); + size_t destinationOffset = 0; + for (uint32_t plane = 0; plane < 2 && ok; ++plane) + { + const uint32_t rows = plane == 0 ? pack.height : pack.height / 2; + const uint32_t rowBytes = pack.width; + const uint32_t availableRowBytes = planes.width[plane] * planes.bytesPerPix[plane]; + const uint8_t* source = plane == 0 ? sourceY : sourceUV; + if (planes.height[plane] < rows || availableRowBytes < rowBytes || planes.pitch[plane] < rowBytes) + { + std::fprintf(stderr, "%s raw NV12 plane %u layout cannot provide tight %ux%u data\n", label, plane, + rowBytes, rows); + ok = false; + break; + } + for (uint32_t row = 0; row < rows; ++row) + { + std::memcpy(pack.payload->data() + destinationOffset, + source + static_cast(row) * planes.pitch[plane], rowBytes); + destinationOffset += rowBytes; + } + } + } + NvBufSurfaceUnMap(surface, 0, -1); + if (!ok) + pack.payload->clear(); + return ok; +} + +} // namespace + +class FrameCopyWorker::Impl +{ +public: + explicit Impl(const char* label) + : m_label(label ? label : ""), + m_stop(false), + m_hasJob(false), + m_completed(false), + m_result(false), + m_completionSteadyNs(0), + m_frame(NULL), + m_buffer(NULL), + m_hasPack(false) + { + } + + ~Impl() + { + shutdown(); + } + + bool start() + { + if (m_thread.joinable()) + return true; + + { + std::lock_guard lock(m_mutex); + m_stop = false; + m_hasJob = false; + m_completed = false; + m_result = false; + m_completionSteadyNs = 0; + m_frame = NULL; + m_buffer = NULL; + m_hasPack = false; + } + try + { + m_thread = std::thread(&Impl::threadMain, this); + } + catch (...) + { + std::fprintf(stderr, "Failed to start %s frame-copy worker\n", m_label.c_str()); + return false; + } + return true; + } + + void shutdown() + { + { + std::lock_guard lock(m_mutex); + m_stop = true; + } + m_jobAvailable.notify_all(); + m_jobCompleted.notify_all(); + if (m_thread.joinable()) + m_thread.join(); + } + + bool submit(EGLStream::Frame* frame, DmaBuffer* buffer, const RawPackJob* pack) + { + if (!frame || !buffer || (pack && !pack->payload)) + return false; + + { + std::lock_guard lock(m_mutex); + if (m_stop || !m_thread.joinable() || m_hasJob || m_completed) + { + std::fprintf(stderr, "%s frame-copy worker is not ready for a new job\n", m_label.c_str()); + return false; + } + m_frame = frame; + m_buffer = buffer; + m_hasPack = pack != nullptr; + if (pack) + m_pack = *pack; + m_hasJob = true; + } + m_jobAvailable.notify_one(); + return true; + } + + bool wait(uint64_t* completionSteadyNs) + { + std::unique_lock lock(m_mutex); + m_jobCompleted.wait(lock, [this]() { return m_completed || m_stop; }); + if (!m_completed) + return false; + + const bool result = m_result; + if (completionSteadyNs) + *completionSteadyNs = m_completionSteadyNs; + m_completed = false; + m_result = false; + m_completionSteadyNs = 0; + m_frame = NULL; + m_buffer = NULL; + m_hasPack = false; + return result; + } + +private: + void threadMain() + { + while (true) + { + EGLStream::Frame* frame = NULL; + DmaBuffer* buffer = NULL; + bool hasPack = false; + RawPackJob pack; + { + std::unique_lock lock(m_mutex); + m_jobAvailable.wait(lock, [this]() { return m_hasJob || m_stop; }); + if (m_stop) + break; + frame = m_frame; + buffer = m_buffer; + hasPack = m_hasPack; + pack = m_pack; + } + + bool result = copyFrameToBuffer(frame, buffer, m_label.c_str()); + if (result && hasPack) + result = packRawNv12(buffer, pack, &m_scratchY, &m_scratchUV, m_label.c_str()); + const uint64_t completionSteadyNs = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count()); + { + std::lock_guard lock(m_mutex); + m_result = result; + m_completionSteadyNs = completionSteadyNs; + m_hasJob = false; + m_completed = true; + } + m_jobCompleted.notify_one(); + } + } + + const std::string m_label; + std::thread m_thread; + std::mutex m_mutex; + std::condition_variable m_jobAvailable; + std::condition_variable m_jobCompleted; + bool m_stop; + bool m_hasJob; + bool m_completed; + bool m_result; + uint64_t m_completionSteadyNs; + EGLStream::Frame* m_frame; + DmaBuffer* m_buffer; + bool m_hasPack; + RawPackJob m_pack; + // Persistent cached staging for the remap source rows (worker thread only). + std::vector m_scratchY; + std::vector m_scratchUV; +}; + +FrameCopyWorker::FrameCopyWorker(const char* label) : m_impl(new Impl(label)) +{ +} + +FrameCopyWorker::~FrameCopyWorker() +{ +} + +bool FrameCopyWorker::start() +{ + return m_impl->start(); +} + +void FrameCopyWorker::shutdown() +{ + m_impl->shutdown(); +} + +bool FrameCopyWorker::submit(EGLStream::Frame* frame, DmaBuffer* buffer, const RawPackJob* pack) +{ + return m_impl->submit(frame, buffer, pack); +} + +bool FrameCopyWorker::wait(uint64_t* completionSteadyNs) +{ + return m_impl->wait(completionSteadyNs); +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/FrameCopyWorker.h b/examples/camera_streamer/argus_sender/src/FrameCopyWorker.h new file mode 100644 index 0000000000..4dd1dfb6cc --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/FrameCopyWorker.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_FRAME_COPY_WORKER_H +#define ARGUS_SENDER_FRAME_COPY_WORKER_H + +#include +#include +#include + +namespace EGLStream +{ +class Frame; +} + +namespace argus_sender +{ + +class DmaBuffer; +struct NvRemapTable; + +// Optional raw-payload production performed on the worker thread right after +// the VIC copy: either a tight NV12 pack of the whole buffer, or — when +// `remap` is set — a lookup-table resample into outWidth x outHeight (see +// NvRemap.h). Running this on the per-camera worker keeps the two wrist +// remaps parallel and off the coordinator thread. +struct RawPackJob +{ + uint32_t width = 0; + uint32_t height = 0; + const NvRemapTable* remap = nullptr; + std::vector* payload = nullptr; +}; + +class FrameCopyWorker +{ +public: + explicit FrameCopyWorker(const char* label); + ~FrameCopyWorker(); + + bool start(); + void shutdown(); + bool submit(EGLStream::Frame* frame, DmaBuffer* buffer, const RawPackJob* pack = nullptr); + bool wait(uint64_t* completionSteadyNs = NULL); + +private: + class Impl; + std::unique_ptr m_impl; +}; + +} // namespace argus_sender + +#endif // ARGUS_SENDER_FRAME_COPY_WORKER_H diff --git a/examples/camera_streamer/argus_sender/src/HevcEncoder.cpp b/examples/camera_streamer/argus_sender/src/HevcEncoder.cpp new file mode 100644 index 0000000000..d2bce4f9a3 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/HevcEncoder.cpp @@ -0,0 +1,518 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "HevcEncoder.h" + +#include "EncodedFrameSink.h" +#include "Error.h" +#include "Runtime.h" +#include "StereoTiming.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace argus_sender +{ +namespace +{ + +const uint32_t kEncoderCapturePlaneBuffers = 4; +const uint32_t kEncodedBufferBytes = 16 * 1024 * 1024; + +} // namespace + +DmaBuffer* DmaBuffer::create(const Argus::Size2D& size, + NvBufSurfaceColorFormat colorFormat, + NvBufSurfaceLayout layout) +{ + DmaBuffer* buffer = new DmaBuffer(); + if (!buffer) + return NULL; + + NvBufSurf::NvCommonAllocateParams params; + std::memset(¶ms, 0, sizeof(params)); + params.memtag = NvBufSurfaceTag_CAMERA; + params.width = size.width(); + params.height = size.height(); + params.colorFormat = colorFormat; + params.layout = layout; + params.memType = NVBUF_MEM_SURFACE_ARRAY; + + if (NvBufSurf::NvAllocate(¶ms, 1, &buffer->m_fd) != 0) + { + delete buffer; + return NULL; + } + + if (NvBufSurfaceFromFd(buffer->m_fd, reinterpret_cast(&buffer->m_surface)) != 0 || !buffer->m_surface) + { + delete buffer; + return NULL; + } + + buffer->planes[0].fd = buffer->m_fd; + buffer->planes[0].bytesused = 1; + return buffer; +} + +int DmaBuffer::getFd() const +{ + return m_fd; +} + +DmaBuffer::DmaBuffer() + : NvBuffer(0, 0), + m_fd(-1), + m_surface(NULL), + m_eglImageMapped(false), + m_argusStream(NULL), + m_argusBuffer(NULL), + m_argusAcquired(false) +{ +} + +DmaBuffer::~DmaBuffer() +{ + if (m_fd < 0) + return; + if (m_eglImageMapped && m_surface) + NvBufSurfaceUnMapEglImage(m_surface, 0); + if (m_surface) + NvBufSurfaceDestroy(m_surface); + m_surface = NULL; + m_fd = -1; +} + +bool DmaBuffer::mapEglImage() +{ + if (!m_surface) + return false; + if (m_eglImageMapped) + return getEglImage() != EGL_NO_IMAGE_KHR; + if (NvBufSurfaceMapEglImage(m_surface, 0) != 0) + return false; + m_eglImageMapped = true; + return getEglImage() != EGL_NO_IMAGE_KHR; +} + +EGLImageKHR DmaBuffer::getEglImage() const +{ + if (!m_surface || !m_eglImageMapped) + return EGL_NO_IMAGE_KHR; + return m_surface->surfaceList[0].mappedAddr.eglImage; +} + +bool DmaBuffer::bindArgusBuffer(Argus::IBufferOutputStream* stream, Argus::Buffer* buffer) +{ + if (!stream || !buffer || m_argusStream || m_argusBuffer) + return false; + m_argusStream = stream; + m_argusBuffer = buffer; + m_argusAcquired.store(true); + return true; +} + +bool DmaBuffer::markArgusAcquired(Argus::Buffer* buffer) +{ + if (!m_argusStream || buffer != m_argusBuffer) + return false; + bool expected = false; + return m_argusAcquired.compare_exchange_strong(expected, true); +} + +bool DmaBuffer::releaseToArgus() +{ + if (!m_argusStream || !m_argusBuffer) + return false; + bool expected = true; + if (!m_argusAcquired.compare_exchange_strong(expected, false)) + return false; + if (m_argusStream->releaseBuffer(m_argusBuffer) != Argus::STATUS_OK) + { + m_argusAcquired.store(true); + return false; + } + return true; +} + +bool DmaBuffer::isArgusBound() const +{ + return m_argusStream && m_argusBuffer; +} + +Argus::Buffer* DmaBuffer::getArgusBuffer() const +{ + return m_argusBuffer; +} + +class DirectHevcEncoder::Impl +{ +public: + Impl(EncodedFrameSink* sink, const HevcEncoderSettings& settings) + : m_sink(sink), + m_settings(settings), + m_encoder(NULL), + m_totalFrames(0), + m_intervalFrames(0), + m_intervalBytes(0), + m_benchSamples(0), + m_benchSumMs(0.0), + m_benchMinMs(std::numeric_limits::max()), + m_benchMaxMs(0.0), + m_benchSummaryPrinted(false), + m_error(false) + { + } + + ~Impl() + { + shutdown(); + } + + bool initialize(uint32_t width, uint32_t height, uint32_t fps, uint32_t gop) + { + m_statsStart = std::chrono::steady_clock::now(); + m_encoder = NvVideoEncoder::createVideoEncoder("argus_sender_hevc"); + if (!m_encoder) + ORIGINATE_ERROR("Could not create HEVC NvVideoEncoder"); + + if (m_encoder->setCapturePlaneFormat(V4L2_PIX_FMT_H265, width, height, kEncodedBufferBytes) < 0) + ORIGINATE_ERROR("Could not set HEVC encoder capture plane format"); + if (m_encoder->setOutputPlaneFormat(V4L2_PIX_FMT_NV12M, width, height, V4L2_COLORSPACE_REC709) < 0) + ORIGINATE_ERROR("Could not set HEVC encoder output plane format"); + if (m_encoder->setBitrate(m_settings.bitrateMbps * 1000000U) < 0) + ORIGINATE_ERROR("Could not set HEVC bitrate"); + if (m_encoder->setProfile(V4L2_MPEG_VIDEO_H265_PROFILE_MAIN) < 0) + ORIGINATE_ERROR("Could not set HEVC profile"); + if (m_encoder->setRateControlMode(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR) < 0) + ORIGINATE_ERROR("Could not set HEVC CBR mode"); + if (m_encoder->setIFrameInterval(gop) < 0) + ORIGINATE_ERROR("Could not set HEVC I-frame interval"); + if (m_encoder->setFrameRate(fps, 1) < 0) + ORIGINATE_ERROR("Could not set HEVC encoder frame rate"); + + setOptionalControl("max performance mode", m_encoder->setMaxPerfMode(1)); + setOptionalControl("ultrafast hardware preset", m_encoder->setHWPresetType(V4L2_ENC_HW_PRESET_ULTRAFAST)); + setOptionalControl("IDR interval", m_encoder->setIDRInterval(gop)); + setOptionalControl("disable B-frames", m_encoder->setNumBFrames(0)); + setOptionalControl("single reference frame", m_encoder->setNumReferenceFrames(1)); + setOptionalControl("insert SPS/PPS at IDR", m_encoder->setInsertSpsPpsAtIdrEnabled(true)); + setOptionalControl("insert AUD", m_encoder->setInsertAudEnabled(true)); + + if (m_encoder->output_plane.setupPlane(V4L2_MEMORY_DMABUF, kEncoderOutputPlaneBuffers, true, false) < 0) + ORIGINATE_ERROR("Could not setup HEVC encoder output plane"); + if (m_encoder->capture_plane.setupPlane(V4L2_MEMORY_MMAP, kEncoderCapturePlaneBuffers, true, false) < 0) + ORIGINATE_ERROR("Could not setup HEVC encoder capture plane"); + + if (m_encoder->output_plane.setStreamStatus(true) < 0) + ORIGINATE_ERROR("Failed to stream on HEVC output plane"); + if (m_encoder->capture_plane.setStreamStatus(true) < 0) + ORIGINATE_ERROR("Failed to stream on HEVC capture plane"); + + m_encoder->capture_plane.setDQThreadCallback(capturePlaneCallback); + if (m_encoder->capture_plane.startDQThread(this) < 0) + ORIGINATE_ERROR("Failed to start HEVC capture-plane dequeue thread"); + + for (uint32_t i = 0; i < m_encoder->capture_plane.getNumBuffers(); ++i) + { + struct v4l2_buffer v4l2Buf; + struct v4l2_plane planes[MAX_PLANES]; + std::memset(&v4l2Buf, 0, sizeof(v4l2Buf)); + std::memset(planes, 0, sizeof(planes)); + v4l2Buf.index = i; + v4l2Buf.m.planes = planes; + if (m_encoder->capture_plane.qBuffer(v4l2Buf, NULL) < 0) + ORIGINATE_ERROR("Failed to queue HEVC capture-plane buffer"); + } + + return true; + } + + bool queueInput(struct v4l2_buffer& v4l2Buf, DmaBuffer* buffer, uint64_t frameSeq = 0) + { + if (!m_encoder || !buffer) + return false; + const auto queueStart = std::chrono::steady_clock::now(); + { + std::lock_guard lock(m_benchMutex); + m_pendingSeqs.push_back(frameSeq); + if (m_settings.benchEncode) + m_benchQueueTimes.push_back(queueStart); + } + if (m_encoder->output_plane.qBuffer(v4l2Buf, buffer) < 0) + { + forgetQueuedFrameMetadata(); + m_error = true; + return false; + } + return true; + } + + bool dequeueInput(struct v4l2_buffer& v4l2Buf, DmaBuffer** buffer) + { + NvBuffer* sharedBuffer = NULL; + if (!m_encoder) + return false; + if (m_encoder->output_plane.dqBuffer(v4l2Buf, NULL, &sharedBuffer, 10) < 0) + { + if (errno != EAGAIN && errno != EINTR) + m_error = true; + return false; + } + if (buffer) + *buffer = static_cast(sharedBuffer); + return true; + } + + bool sendEos(struct v4l2_buffer& v4l2Buf) + { + if (!m_encoder) + return true; + v4l2Buf.m.planes[0].m.fd = -1; + v4l2Buf.m.planes[0].bytesused = 0; + if (m_encoder->output_plane.qBuffer(v4l2Buf, NULL) < 0) + { + m_error = true; + return false; + } + return true; + } + + bool hasError() const + { + return m_error || (m_sink && m_sink->hasError()); + } + + void shutdown() + { + if (!m_encoder) + return; + m_encoder->capture_plane.waitForDQThread(2000); + printEncodeBenchSummary(); + m_encoder->output_plane.setStreamStatus(false); + m_encoder->capture_plane.setStreamStatus(false); + delete m_encoder; + m_encoder = NULL; + } + +private: + void setOptionalControl(const char* name, int ret) + { + if (ret < 0) + std::fprintf(stderr, "Warning: HEVC encoder did not accept %s control; continuing\n", name); + } + + static bool capturePlaneCallback(struct v4l2_buffer* v4l2Buf, NvBuffer* buffer, NvBuffer*, void* arg) + { + Impl* self = static_cast(arg); + return self->onCapturePlaneBuffer(v4l2Buf, buffer); + } + + bool onCapturePlaneBuffer(struct v4l2_buffer* v4l2Buf, NvBuffer* buffer) + { + if (!v4l2Buf || !buffer) + { + m_error = true; + return false; + } + + const size_t encodedBytes = buffer->planes[0].bytesused; + if (encodedBytes == 0) + return false; + + const uint64_t frameSeq = popPendingFrameSeq(); + recordEncodeLatency(encodedBytes); + + uint32_t senderClockDomain = SENDER_CLOCK_UNKNOWN; + const uint64_t encodeDoneNs = readSenderClockNs(&senderClockDomain); + if (!m_sink || + !m_sink->enqueueEncoded(buffer->planes[0].data, encodedBytes, frameSeq, encodeDoneNs, senderClockDomain)) + { + if (!stopRequested()) + m_error = true; + return false; + } + + ++m_totalFrames; + ++m_intervalFrames; + m_intervalBytes += encodedBytes; + + if (m_settings.kpi && m_settings.logIntervalSeconds > 0) + { + const auto now = std::chrono::steady_clock::now(); + const double elapsed = std::chrono::duration_cast>(now - m_statsStart).count(); + if (elapsed >= static_cast(m_settings.logIntervalSeconds)) + { + const double fpsActual = static_cast(m_intervalFrames) / elapsed; + const double mbps = static_cast(m_intervalBytes) * 8.0 / elapsed / 1.0e6; + if (verboseLoggingEnabled()) + { + std::printf("argus_sender[hevc-direct]: %.2f FPS, %.2f Mbps, total=%llu\n", fpsActual, mbps, + static_cast(m_totalFrames)); + } + m_intervalFrames = 0; + m_intervalBytes = 0; + m_statsStart = now; + } + } + + if (m_encoder->capture_plane.qBuffer(*v4l2Buf, NULL) < 0) + { + m_error = true; + return false; + } + return true; + } + + uint64_t popPendingFrameSeq() + { + std::lock_guard lock(m_benchMutex); + if (m_pendingSeqs.empty()) + return 0; + const uint64_t seq = m_pendingSeqs.front(); + m_pendingSeqs.pop_front(); + return seq; + } + + void forgetQueuedFrameMetadata() + { + std::lock_guard lock(m_benchMutex); + if (!m_pendingSeqs.empty()) + m_pendingSeqs.pop_back(); + if (m_settings.benchEncode && !m_benchQueueTimes.empty()) + m_benchQueueTimes.pop_back(); + } + + void recordEncodeLatency(size_t encodedBytes) + { + if (!m_settings.benchEncode) + return; + + std::chrono::steady_clock::time_point queueTime; + { + std::lock_guard lock(m_benchMutex); + if (m_benchQueueTimes.empty()) + return; + queueTime = m_benchQueueTimes.front(); + m_benchQueueTimes.pop_front(); + } + + const auto now = std::chrono::steady_clock::now(); + const double encodeMs = + std::chrono::duration_cast>(now - queueTime).count(); + + std::lock_guard lock(m_benchMutex); + ++m_benchSamples; + m_benchSumMs += encodeMs; + if (encodeMs < m_benchMinMs) + m_benchMinMs = encodeMs; + if (encodeMs > m_benchMaxMs) + m_benchMaxMs = encodeMs; + + std::printf("argus_sender[encode-bench]: frame=%llu encode_ms=%.3f packet_bytes=%zu\n", + static_cast(m_totalFrames + 1), encodeMs, encodedBytes); + } + + void printEncodeBenchSummary() + { + if (!m_settings.benchEncode) + return; + + std::lock_guard lock(m_benchMutex); + if (m_benchSummaryPrinted) + return; + m_benchSummaryPrinted = true; + + if (m_benchSamples == 0) + { + std::printf("argus_sender[encode-bench-summary]: no encoded frames sampled\n"); + return; + } + + std::printf("argus_sender[encode-bench-summary]: n=%llu mean=%.3f ms min=%.3f ms max=%.3f ms\n", + static_cast(m_benchSamples), m_benchSumMs / static_cast(m_benchSamples), + m_benchMinMs, m_benchMaxMs); + } + + EncodedFrameSink* m_sink; + const HevcEncoderSettings& m_settings; + NvVideoEncoder* m_encoder; + uint64_t m_totalFrames; + uint64_t m_intervalFrames; + uint64_t m_intervalBytes; + std::chrono::steady_clock::time_point m_statsStart; + std::mutex m_benchMutex; + uint64_t m_benchSamples; + double m_benchSumMs; + double m_benchMinMs; + double m_benchMaxMs; + std::deque m_benchQueueTimes; + std::deque m_pendingSeqs; + bool m_benchSummaryPrinted; + std::atomic m_error; +}; + +bool waitForEncoderInput(DirectHevcEncoder* encoder, struct v4l2_buffer* v4l2Buf, DmaBuffer** buffer) +{ + while (!encoder->hasError()) + { + if (encoder->dequeueInput(*v4l2Buf, buffer)) + return true; + if (stopRequested()) + return false; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return false; +} + +DirectHevcEncoder::DirectHevcEncoder(EncodedFrameSink* sink, const HevcEncoderSettings& settings) + : m_impl(new Impl(sink, settings)) +{ +} + +DirectHevcEncoder::~DirectHevcEncoder() +{ +} + +bool DirectHevcEncoder::initialize(uint32_t width, uint32_t height, uint32_t fps, uint32_t gop) +{ + return m_impl->initialize(width, height, fps, gop); +} + +bool DirectHevcEncoder::queueInput(struct v4l2_buffer& v4l2Buf, DmaBuffer* buffer, uint64_t frameSeq) +{ + return m_impl->queueInput(v4l2Buf, buffer, frameSeq); +} + +bool DirectHevcEncoder::dequeueInput(struct v4l2_buffer& v4l2Buf, DmaBuffer** buffer) +{ + return m_impl->dequeueInput(v4l2Buf, buffer); +} + +bool DirectHevcEncoder::sendEos(struct v4l2_buffer& v4l2Buf) +{ + return m_impl->sendEos(v4l2Buf); +} + +bool DirectHevcEncoder::hasError() const +{ + return m_impl->hasError(); +} + +void DirectHevcEncoder::shutdown() +{ + m_impl->shutdown(); +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/HevcEncoder.h b/examples/camera_streamer/argus_sender/src/HevcEncoder.h new file mode 100644 index 0000000000..25bed4e446 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/HevcEncoder.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_HEVC_ENCODER_H +#define ARGUS_SENDER_HEVC_ENCODER_H + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace argus_sender +{ + +class EncodedFrameSink; + +struct HevcEncoderSettings +{ + uint32_t bitrateMbps = 50; + bool kpi = true; + uint32_t logIntervalSeconds = 1; + bool benchEncode = false; +}; + +const uint32_t kEncoderOutputPlaneBuffers = 2; + +class DmaBuffer : public NvBuffer +{ +public: + static DmaBuffer* create(const Argus::Size2D& size, + NvBufSurfaceColorFormat colorFormat, + NvBufSurfaceLayout layout = NVBUF_LAYOUT_BLOCK_LINEAR); + ~DmaBuffer(); + + int getFd() const; + bool mapEglImage(); + EGLImageKHR getEglImage() const; + + // Bind this client-owned DMABUF to the Argus Buffer that writes it. + // A bound buffer is acquired by the application until releaseToArgus(). + bool bindArgusBuffer(Argus::IBufferOutputStream* stream, Argus::Buffer* buffer); + bool markArgusAcquired(Argus::Buffer* buffer); + bool releaseToArgus(); + bool isArgusBound() const; + Argus::Buffer* getArgusBuffer() const; + +private: + DmaBuffer(); + + int m_fd; + NvBufSurface* m_surface; + bool m_eglImageMapped; + Argus::IBufferOutputStream* m_argusStream; + Argus::Buffer* m_argusBuffer; + std::atomic m_argusAcquired; +}; + +class DirectHevcEncoder +{ +public: + DirectHevcEncoder(EncodedFrameSink* sink, const HevcEncoderSettings& settings); + ~DirectHevcEncoder(); + + bool initialize(uint32_t width, uint32_t height, uint32_t fps, uint32_t gop); + bool queueInput(struct v4l2_buffer& v4l2Buf, DmaBuffer* buffer, uint64_t frameSeq = 0); + bool dequeueInput(struct v4l2_buffer& v4l2Buf, DmaBuffer** buffer); + bool sendEos(struct v4l2_buffer& v4l2Buf); + bool hasError() const; + void shutdown(); + +private: + class Impl; + std::unique_ptr m_impl; +}; + +bool waitForEncoderInput(DirectHevcEncoder* encoder, struct v4l2_buffer* v4l2Buf, DmaBuffer** buffer); + +} // namespace argus_sender + +#endif // ARGUS_SENDER_HEVC_ENCODER_H diff --git a/examples/camera_streamer/argus_sender/src/NvBufSurfaceLite.cpp b/examples/camera_streamer/argus_sender/src/NvBufSurfaceLite.cpp new file mode 100644 index 0000000000..90b1a21192 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/NvBufSurfaceLite.cpp @@ -0,0 +1,70 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/* + * Minimal NvBufSurf wrapper for argus_sender. + * + * NVIDIA's sample NvBufSurface.cpp also wraps transform helpers and therefore + * requires libnvbufsurftransform.so. The sender only needs allocation, so this + * file keeps the standalone binary off that optional dependency. + */ + +#include +#include +#include + +int NvBufSurf::NvDestroy(int fd) +{ + if (fd <= 0) + return -1; + + NvBufSurface* nvbufSurf = NULL; + NvBufSurfaceFromFd(fd, reinterpret_cast(&nvbufSurf)); + if (!nvbufSurf) + return -1; + + return NvBufSurfaceDestroy(nvbufSurf); +} + +int NvBufSurf::NvAllocate(NvCommonAllocateParams* allocateParams, uint32_t numBuffers, int* fd) +{ + if (!allocateParams || !fd || numBuffers < 1) + return -1; + + NvBufSurfaceAllocateParams inputParams; + std::memset(&inputParams, 0, sizeof(inputParams)); + inputParams.params.width = allocateParams->width; + inputParams.params.height = allocateParams->height; + inputParams.params.memType = allocateParams->memType; + inputParams.params.layout = allocateParams->layout; + inputParams.params.colorFormat = allocateParams->colorFormat; + inputParams.memtag = allocateParams->memtag; + inputParams.chromaSubsampling.chromaLocHoriz = allocateParams->chromaSubsampling.chromaLocHoriz; + inputParams.chromaSubsampling.chromaLocVert = allocateParams->chromaSubsampling.chromaLocVert; + + for (uint32_t index = 0; index < numBuffers; ++index) + { + NvBufSurface* nvbufSurf = NULL; + const int ret = NvBufSurfaceAllocate(&nvbufSurf, 1, &inputParams); + if (ret < 0 || !nvbufSurf) + return -1; + nvbufSurf->numFilled = 1; + fd[index] = nvbufSurf->surfaceList[0].bufferDesc; + } + + return 0; +} + +int NvBufSurf::NvTransform(NvCommonTransformParams*, int, int) +{ + std::fprintf(stderr, "NvBufSurf::NvTransform is not included in this standalone build\n"); + return -1; +} + +int NvBufSurf::NvTransformAsync(NvCommonTransformParams*, NvBufSurfTransformSyncObj_t*, int, int) +{ + std::fprintf(stderr, "NvBufSurf::NvTransformAsync is not included in this standalone build\n"); + return -1; +} diff --git a/examples/camera_streamer/argus_sender/src/NvRemap.cpp b/examples/camera_streamer/argus_sender/src/NvRemap.cpp new file mode 100644 index 0000000000..9b0d953bbf --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/NvRemap.cpp @@ -0,0 +1,324 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "NvRemap.h" + +#include +#include +#include +#include +#include + +namespace argus_sender +{ +namespace +{ + +const char kRmapMagic[4] = { 'R', 'M', 'A', 'P' }; +const uint32_t kWeightOne = 64; + +bool fail(std::string* error, const std::string& message) +{ + if (error) + *error = message; + return false; +} + +// One fixed-point bilinear tap from a float source coordinate; returns false +// (zero-weight tap) when the 2x2 footprint falls outside the source. +NvRemapTap buildTap(float mx, float my, uint32_t srcWidth, uint32_t srcHeight, bool* outOfSource) +{ + NvRemapTap tap; + if (!(mx >= 0.0F) || !(my >= 0.0F) || mx > static_cast(srcWidth - 1) || my > static_cast(srcHeight - 1)) + { + *outOfSource = true; + return tap; + } + uint32_t x0 = static_cast(mx); + uint32_t y0 = static_cast(my); + // Keep the 2x2 footprint inside the source on the last row/column. + if (x0 >= srcWidth - 1) + x0 = srcWidth - 2; + if (y0 >= srcHeight - 1) + y0 = srcHeight - 2; + const float fx = mx - static_cast(x0); + const float fy = my - static_cast(y0); + uint32_t w01 = static_cast(fx * (1.0F - fy) * kWeightOne + 0.5F); + uint32_t w10 = static_cast((1.0F - fx) * fy * kWeightOne + 0.5F); + uint32_t w11 = static_cast(fx * fy * kWeightOne + 0.5F); + if (w01 + w10 + w11 > kWeightOne) + w11 = kWeightOne - w01 - w10; + const uint32_t w00 = kWeightOne - w01 - w10 - w11; + tap.x0 = static_cast(x0); + tap.y0 = static_cast(y0); + tap.w00 = static_cast(w00); + tap.w01 = static_cast(w01); + tap.w10 = static_cast(w10); + tap.w11 = static_cast(w11); + return tap; +} + +inline uint32_t sampleTap(const NvRemapTap& tap, + const uint8_t* plane, + size_t pitch, + size_t pixelStride, + size_t channelOffset, + uint32_t rowOffset) +{ + const uint8_t* base = plane + static_cast(tap.y0 - rowOffset) * pitch + + static_cast(tap.x0) * pixelStride + channelOffset; + return (static_cast(tap.w00) * base[0] + static_cast(tap.w01) * base[pixelStride] + + static_cast(tap.w10) * base[pitch] + static_cast(tap.w11) * base[pitch + pixelStride] + + kWeightOne / 2) / + kWeightOne; +} + +inline bool tapIsVoid(const NvRemapTap& tap) +{ + return (tap.w00 | tap.w01 | tap.w10 | tap.w11) == 0; +} + +} // namespace + +bool loadNvRemapTable(const std::string& path, + uint32_t srcWidth, + uint32_t srcHeight, + uint32_t outWidth, + uint32_t outHeight, + NvRemapTable* table, + std::string* error) +{ + if (!table) + return fail(error, "remap table output is null"); + *table = NvRemapTable(); + if (srcWidth < 2 || srcHeight < 2 || outWidth == 0 || outHeight == 0 || outWidth % 2 != 0 || outHeight % 2 != 0) + { + return fail(error, "remap source/output dimensions are invalid"); + } + + std::FILE* file = std::fopen(path.c_str(), "rb"); + if (!file) + return fail(error, "cannot open remap file '" + path + "'"); + + char magic[4] = { 0, 0, 0, 0 }; + uint32_t destWidth = 0; + uint32_t destHeight = 0; + float balance = 0.0F; + bool ok = std::fread(magic, 1, 4, file) == 4 && std::fread(&destWidth, 4, 1, file) == 1 && + std::fread(&destHeight, 4, 1, file) == 1 && std::fread(&balance, 4, 1, file) == 1; + if (!ok || std::memcmp(magic, kRmapMagic, 4) != 0) + { + std::fclose(file); + return fail(error, "'" + path + "' is not an RMAP file"); + } + if (destWidth == 0 || destHeight == 0 || destWidth % outWidth != 0 || destHeight % outHeight != 0 || + destWidth / outWidth != destHeight / outHeight || destWidth / outWidth > 2) + { + std::ostringstream message; + message << "'" << path << "' destination " << destWidth << "x" << destHeight << " is not the wire output " + << outWidth << "x" << outHeight << " times a shared supersample factor of 1 or 2"; + std::fclose(file); + return fail(error, message.str()); + } + const uint32_t supersample = destWidth / outWidth; + + const size_t count = static_cast(destWidth) * destHeight; + std::vector mapX(count); + std::vector mapY(count); + ok = std::fread(mapX.data(), sizeof(float), count, file) == count && + std::fread(mapY.data(), sizeof(float), count, file) == count; + std::fclose(file); + if (!ok) + return fail(error, "'" + path + "' is truncated"); + + table->srcWidth = srcWidth; + table->srcHeight = srcHeight; + table->destWidth = destWidth; + table->destHeight = destHeight; + table->outWidth = outWidth; + table->outHeight = outHeight; + table->supersample = supersample; + table->balance = balance; + + size_t outOfSourceCount = 0; + table->lumaTaps.resize(count); + for (size_t i = 0; i < count; ++i) + { + bool outOfSource = false; + table->lumaTaps[i] = buildTap(mapX[i], mapY[i], srcWidth, srcHeight, &outOfSource); + if (outOfSource) + ++outOfSourceCount; + } + if (outOfSourceCount * 100 > count) + { + std::ostringstream message; + message << "'" << path << "' addresses pixels outside the " << srcWidth << "x" << srcHeight << " source for " + << outOfSourceCount << "/" << count << " destinations; the map was built for a different source"; + *table = NvRemapTable(); + return fail(error, message.str()); + } + uint32_t lumaRowFirst = srcHeight - 1; + uint32_t lumaRowLast = 0; + for (size_t i = 0; i < count; ++i) + { + if (tapIsVoid(table->lumaTaps[i])) + continue; + const uint32_t y0 = table->lumaTaps[i].y0; + if (y0 < lumaRowFirst) + lumaRowFirst = y0; + if (y0 + 1 > lumaRowLast) + lumaRowLast = y0 + 1; + } + table->lumaRowFirst = lumaRowFirst; + table->lumaRowLast = lumaRowLast; + + // Chroma taps: one per output UV cell, sampling the half-resolution + // interleaved UV plane at the average of the cell's central luma map + // positions (chroma is already 2x subsampled; a single filtered tap per + // cell is enough). + const uint32_t chromaWidth = outWidth / 2; + const uint32_t chromaHeight = outHeight / 2; + const uint32_t block = 2 * supersample; + table->chromaTaps.resize(static_cast(chromaWidth) * chromaHeight); + for (uint32_t cy = 0; cy < chromaHeight; ++cy) + { + for (uint32_t cx = 0; cx < chromaWidth; ++cx) + { + const uint32_t x = cx * block + block / 2 - 1; + const uint32_t y = cy * block + block / 2 - 1; + float sumX = 0.0F; + float sumY = 0.0F; + for (uint32_t dy = 0; dy < 2; ++dy) + { + const size_t row = static_cast(y + dy) * destWidth + x; + sumX += mapX[row] + mapX[row + 1]; + sumY += mapY[row] + mapY[row + 1]; + } + bool outOfSource = false; + table->chromaTaps[static_cast(cy) * chromaWidth + cx] = + buildTap(sumX * 0.25F * 0.5F, sumY * 0.25F * 0.5F, srcWidth / 2, srcHeight / 2, &outOfSource); + } + } + uint32_t chromaRowFirst = srcHeight / 2 - 1; + uint32_t chromaRowLast = 0; + for (size_t i = 0; i < table->chromaTaps.size(); ++i) + { + if (tapIsVoid(table->chromaTaps[i])) + continue; + const uint32_t y0 = table->chromaTaps[i].y0; + if (y0 < chromaRowFirst) + chromaRowFirst = y0; + if (y0 + 1 > chromaRowLast) + chromaRowLast = y0 + 1; + } + table->chromaRowFirst = chromaRowFirst; + table->chromaRowLast = chromaRowLast; + return true; +} + +namespace +{ + +// Luma resample for output rows [rowBegin, rowEnd) — the unit of the two-way +// split below. +void remapLumaRows(const NvRemapTable& table, + const uint8_t* srcY, + size_t srcYPitch, + uint32_t srcYRowOffset, + uint8_t* dst, + uint32_t rowBegin, + uint32_t rowEnd) +{ + const uint32_t outWidth = table.outWidth; + uint8_t* dstY = dst + static_cast(rowBegin) * outWidth; + + if (table.supersample == 1) + { + const NvRemapTap* tap = table.lumaTaps.data() + static_cast(rowBegin) * outWidth; + for (uint32_t y = rowBegin; y < rowEnd; ++y) + { + for (uint32_t x = 0; x < outWidth; ++x, ++tap) + { + dstY[x] = + tapIsVoid(*tap) ? 0 : static_cast(sampleTap(*tap, srcY, srcYPitch, 1, 0, srcYRowOffset)); + } + dstY += outWidth; + } + return; + } + + // 2x supersample: each output pixel is the box average of a 2x2 block + // of bilinear samples, so minified regions are prefiltered. + const uint32_t destWidth = table.destWidth; + for (uint32_t y = rowBegin; y < rowEnd; ++y) + { + const NvRemapTap* row0 = table.lumaTaps.data() + static_cast(2 * y) * destWidth; + const NvRemapTap* row1 = row0 + destWidth; + for (uint32_t x = 0; x < outWidth; ++x) + { + const NvRemapTap& t00 = row0[2 * x]; + const NvRemapTap& t01 = row0[2 * x + 1]; + const NvRemapTap& t10 = row1[2 * x]; + const NvRemapTap& t11 = row1[2 * x + 1]; + if (tapIsVoid(t00) || tapIsVoid(t01) || tapIsVoid(t10) || tapIsVoid(t11)) + { + dstY[x] = 0; + continue; + } + const uint32_t sum = sampleTap(t00, srcY, srcYPitch, 1, 0, srcYRowOffset) + + sampleTap(t01, srcY, srcYPitch, 1, 0, srcYRowOffset) + + sampleTap(t10, srcY, srcYPitch, 1, 0, srcYRowOffset) + + sampleTap(t11, srcY, srcYPitch, 1, 0, srcYRowOffset); + dstY[x] = static_cast((sum + 2) / 4); + } + dstY += outWidth; + } +} + +} // namespace + +void nvRemapNv12(const NvRemapTable& table, + const uint8_t* srcY, + size_t srcYPitch, + const uint8_t* srcUV, + size_t srcUVPitch, + uint8_t* dst, + uint32_t srcYRowOffset, + uint32_t srcUVRowOffset) +{ + const uint32_t outWidth = table.outWidth; + const uint32_t outHeight = table.outHeight; + uint8_t* dstY = dst; + + // The luma gather dominates the frame cost; split it across a helper + // thread so the per-frame latency stays inside a 60 fps tick. + const uint32_t split = outHeight / 2; + std::thread lower(remapLumaRows, std::cref(table), srcY, srcYPitch, srcYRowOffset, dstY, split, outHeight); + remapLumaRows(table, srcY, srcYPitch, srcYRowOffset, dstY, 0, split); + lower.join(); + + uint8_t* dstUV = dst + static_cast(outWidth) * outHeight; + const uint32_t chromaWidth = outWidth / 2; + const uint32_t chromaHeight = outHeight / 2; + const NvRemapTap* tap = table.chromaTaps.data(); + for (uint32_t y = 0; y < chromaHeight; ++y) + { + for (uint32_t x = 0; x < chromaWidth; ++x, ++tap) + { + if (tapIsVoid(*tap)) + { + dstUV[2 * x] = 128; + dstUV[2 * x + 1] = 128; + continue; + } + dstUV[2 * x] = static_cast(sampleTap(*tap, srcUV, srcUVPitch, 2, 0, srcUVRowOffset)); + dstUV[2 * x + 1] = static_cast(sampleTap(*tap, srcUV, srcUVPitch, 2, 1, srcUVRowOffset)); + } + dstUV += static_cast(chromaWidth) * 2; + } +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/NvRemap.h b/examples/camera_streamer/argus_sender/src/NvRemap.h new file mode 100644 index 0000000000..cb978ef12f --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/NvRemap.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_NV_REMAP_H +#define ARGUS_SENDER_NV_REMAP_H + +#include +#include +#include +#include + +namespace argus_sender +{ + +// Precomputed NV12 lookup-table remap. +// +// Loads the RMAP binaries emitted by +// robotics/hardware_sdk/camera/calib/gen_remap_maps.py (little-endian +// header struct(<4sIIf) = [magic "RMAP", dest width u32, dest height u32, +// balance f32], then map_x[H*W] and map_y[H*W] float32 row-major: the SOURCE +// pixel each destination pixel samples, cv2.remap CV_32FC1 semantics). +// +// The float maps are converted once at load into fixed-point bilinear taps +// (top-left source coordinate + four weights summing to 64), so the per-frame +// work is integer gathers only. The RMAP destination grid may be an integer +// multiple (the supersample factor, 1 or 2 per axis, identical on both) of the +// wire output size; remapNv12() then box-averages each SxS block so the +// minified output is properly prefiltered instead of aliased. +// +// This module is deliberately free of Jetson/Argus dependencies so the LUT +// build and resample math stay unit-testable on x86. + +struct NvRemapTap +{ + uint16_t x0 = 0; + uint16_t y0 = 0; + // Bilinear weights in 1/64 units; all zero marks an out-of-source tap + // (black for luma, neutral 128 for chroma). + uint8_t w00 = 0; + uint8_t w01 = 0; + uint8_t w10 = 0; + uint8_t w11 = 0; +}; + +struct NvRemapTable +{ + uint32_t srcWidth = 0; + uint32_t srcHeight = 0; + // RMAP destination grid (= outWidth/outHeight * supersample). + uint32_t destWidth = 0; + uint32_t destHeight = 0; + uint32_t outWidth = 0; + uint32_t outHeight = 0; + uint32_t supersample = 1; + float balance = 0.0F; + // Source rows the luma taps actually read ([first, last] inclusive of the + // bilinear +1 row). Callers gathering from slow (uncached/write-combined) + // mappings can copy just these rows into cached memory and pass the copy + // with the matching row offset. + uint32_t lumaRowFirst = 0; + uint32_t lumaRowLast = 0; + // Same for the chroma taps, in UV-plane rows. + uint32_t chromaRowFirst = 0; + uint32_t chromaRowLast = 0; + // destWidth x destHeight luma taps, row-major. + std::vector lumaTaps; + // (outWidth/2) x (outHeight/2) chroma taps addressing the half-resolution + // interleaved UV plane, row-major. + std::vector chromaTaps; + + bool loaded() const + { + return !lumaTaps.empty(); + } +}; + +// Reads an RMAP file and builds the fixed-point taps. The RMAP header must +// carry destination dimensions equal to outWidth/outHeight times a shared +// supersample factor of 1 or 2, and every in-range map value must address the +// srcWidth x srcHeight source frame. Map values outside the source become +// zero-weight taps; more than 1% of them is treated as a wrong-source error. +bool loadNvRemapTable(const std::string& path, + uint32_t srcWidth, + uint32_t srcHeight, + uint32_t outWidth, + uint32_t outHeight, + NvRemapTable* table, + std::string* error); + +// Resamples one pitch-linear NV12 frame (srcY / interleaved srcUV) into a +// tightly packed outWidth x outHeight NV12 payload (Y plane then interleaved +// UV plane, no padding). `dst` must hold outWidth * outHeight * 3 / 2 bytes. +// srcYRowOffset/srcUVRowOffset declare how many leading source rows the given +// plane pointers are missing (0 for a full frame): row r of the original +// source lives at plane + (r - offset) * pitch. Use with the table's +// lumaRowFirst/chromaRowFirst when gathering from a partial cached copy. +void nvRemapNv12(const NvRemapTable& table, + const uint8_t* srcY, + size_t srcYPitch, + const uint8_t* srcUV, + size_t srcUVPitch, + uint8_t* dst, + uint32_t srcYRowOffset = 0, + uint32_t srcUVRowOffset = 0); + +} // namespace argus_sender + +#endif // ARGUS_SENDER_NV_REMAP_H diff --git a/examples/camera_streamer/argus_sender/src/PacketSink.h b/examples/camera_streamer/argus_sender/src/PacketSink.h new file mode 100644 index 0000000000..d6ea6c0abc --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/PacketSink.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_PACKET_SINK_H +#define ARGUS_SENDER_PACKET_SINK_H + +#include +#include +#include + +namespace argus_sender +{ + +struct PacketStreamTiming +{ + bool present = false; + bool encoded = false; + uint64_t captureStartSteadyNs = 0; + uint64_t acquireSpreadNs = 0; + uint64_t copyDoneSteadyNs = 0; + uint64_t processStartSteadyNs = 0; + uint64_t processDoneSteadyNs = 0; +}; + +struct PacketSampleTiming +{ + bool present = false; + uint64_t sensorSofTscNs = 0; +}; + +struct PacketTiming +{ + uint64_t captureStartSteadyNs = 0; + uint64_t acquireSpreadNs = 0; + uint64_t captureToEncodeStartNs = 0; + uint64_t captureToProcessDoneNs = 0; + std::vector streams; + // Flattened in stream-major, camera-minor order. + std::vector samples; +}; + +class PacketSink +{ +public: + virtual ~PacketSink() + { + } + virtual bool enqueueFramed(std::vector&& packet) = 0; + virtual bool enqueueFramedWithTiming(std::vector&& packet, const PacketTiming& timing) + { + (void)timing; + return enqueueFramed(std::move(packet)); + } + virtual bool hasError() const = 0; +}; + +} // namespace argus_sender + +#endif // ARGUS_SENDER_PACKET_SINK_H diff --git a/examples/camera_streamer/argus_sender/src/Runtime.cpp b/examples/camera_streamer/argus_sender/src/Runtime.cpp new file mode 100644 index 0000000000..716e87d03b --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/Runtime.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "Runtime.h" + +#include +#include + +namespace argus_sender +{ +namespace +{ + +volatile sig_atomic_t g_stop = 0; +std::atomic g_verboseLogging(false); + +} // namespace + +void handleSignal(int) +{ + g_stop = 1; +} + +bool stopRequested() +{ + return g_stop != 0; +} + +void setVerboseLogging(bool enabled) +{ + g_verboseLogging.store(enabled, std::memory_order_relaxed); +} + +bool verboseLoggingEnabled() +{ + return g_verboseLogging.load(std::memory_order_relaxed); +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/Runtime.h b/examples/camera_streamer/argus_sender/src/Runtime.h new file mode 100644 index 0000000000..36aef73130 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/Runtime.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_RUNTIME_H +#define ARGUS_SENDER_RUNTIME_H + +namespace argus_sender +{ + +void handleSignal(int signalNumber); +bool stopRequested(); +void setVerboseLogging(bool enabled); +bool verboseLoggingEnabled(); + +} // namespace argus_sender + +#endif // ARGUS_SENDER_RUNTIME_H diff --git a/examples/camera_streamer/argus_sender/src/SenderSession.cpp b/examples/camera_streamer/argus_sender/src/SenderSession.cpp new file mode 100644 index 0000000000..57078b245b --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/SenderSession.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "SenderSession.h" + +#include "CaptureCoordinator.h" +#include "Config.h" +#include "Runtime.h" + +#include + +namespace argus_sender +{ + +bool executeConfiguredSender(const AppConfig& config) +{ + uint64_t reconnectSession = 0; + while (!stopRequested()) + { + bool disconnected = false; + if (!runConfiguredCapture(config, &disconnected, reconnectSession > 0)) + { + return false; + } + if (!disconnected || config.runtime.frames != 0) + return true; + ++reconnectSession; + std::printf("AVS1 receiver disconnected; rebuilding the complete capture and processing pipeline\n"); + } + return true; +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/SenderSession.h b/examples/camera_streamer/argus_sender/src/SenderSession.h new file mode 100644 index 0000000000..cab3debbaa --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/SenderSession.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_SENDER_SESSION_H +#define ARGUS_SENDER_SENDER_SESSION_H + +namespace argus_sender +{ + +struct AppConfig; + +bool executeConfiguredSender(const AppConfig& config); + +} // namespace argus_sender + +#endif // ARGUS_SENDER_SENDER_SESSION_H diff --git a/examples/camera_streamer/argus_sender/src/StereoAlignmentPolicy.cpp b/examples/camera_streamer/argus_sender/src/StereoAlignmentPolicy.cpp new file mode 100644 index 0000000000..be8c9b5e27 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/StereoAlignmentPolicy.cpp @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "StereoAlignmentPolicy.h" + +namespace argus_sender +{ +namespace +{ + +uint64_t absoluteDeltaNs(int64_t deltaNs) +{ + if (deltaNs >= 0) + return static_cast(deltaNs); + return static_cast(-(deltaNs + 1)) + 1ULL; +} + +} // namespace + +bool stereoAlignmentDeltaNs(const FrameTiming& left, const FrameTiming& right, int64_t* deltaNs) +{ + if (!deltaNs) + return false; + if (left.hasSensorSofTimestampTsc && right.hasSensorSofTimestampTsc) + { + *deltaNs = signedDeltaNs(right.sensorSofTimestampTscNs, left.sensorSofTimestampTscNs); + return true; + } + if (!left.hasSensorTimestamp || !right.hasSensorTimestamp) + return false; + *deltaNs = signedDeltaNs(right.sensorTimestampNs, left.sensorTimestampNs); + return true; +} + +StereoAlignmentAction chooseStereoAlignmentAction(const FrameTiming& left, + const FrameTiming& right, + uint64_t framePeriodNs, + uint32_t dropsAlreadyApplied, + int64_t* deltaNs) +{ + int64_t measuredDeltaNs = 0; + if (!stereoAlignmentDeltaNs(left, right, &measuredDeltaNs)) + return STEREO_ALIGNMENT_KEEP; + if (deltaNs) + *deltaNs = measuredDeltaNs; + if (framePeriodNs == 0 || dropsAlreadyApplied >= kStereoMaxAlignmentDrops || + absoluteDeltaNs(measuredDeltaNs) <= framePeriodNs / 2) + { + return STEREO_ALIGNMENT_KEEP; + } + return measuredDeltaNs > 0 ? STEREO_ALIGNMENT_DROP_LEFT : STEREO_ALIGNMENT_DROP_RIGHT; +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/StereoAlignmentPolicy.h b/examples/camera_streamer/argus_sender/src/StereoAlignmentPolicy.h new file mode 100644 index 0000000000..d446904273 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/StereoAlignmentPolicy.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_STEREO_ALIGNMENT_POLICY_H +#define ARGUS_SENDER_STEREO_ALIGNMENT_POLICY_H + +#include "StereoTiming.h" + +#include + +namespace argus_sender +{ + +const uint32_t kStereoMaxAlignmentDrops = 4; + +enum StereoAlignmentAction +{ + STEREO_ALIGNMENT_KEEP, + STEREO_ALIGNMENT_DROP_LEFT, + STEREO_ALIGNMENT_DROP_RIGHT +}; + +bool stereoAlignmentDeltaNs(const FrameTiming& left, const FrameTiming& right, int64_t* deltaNs); + +StereoAlignmentAction chooseStereoAlignmentAction(const FrameTiming& left, + const FrameTiming& right, + uint64_t framePeriodNs, + uint32_t dropsAlreadyApplied, + int64_t* deltaNs = NULL); + +} // namespace argus_sender + +#endif // ARGUS_SENDER_STEREO_ALIGNMENT_POLICY_H diff --git a/examples/camera_streamer/argus_sender/src/StereoTiming.cpp b/examples/camera_streamer/argus_sender/src/StereoTiming.cpp new file mode 100644 index 0000000000..4ef97bdf44 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/StereoTiming.cpp @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "StereoTiming.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace argus_sender +{ + +StereoOutputRateLimiter::StereoOutputRateLimiter(uint32_t targetFps) + : m_targetFps(targetFps), + m_intervalNs(targetFps ? (1000000000ULL / targetFps > 0 ? 1000000000ULL / targetFps : 1) : 0), + m_toleranceNs(m_intervalNs > 0 ? std::min(m_intervalNs / 10, 1000000ULL) : 0), + m_source(TIMESTAMP_NONE), + m_lastTimestampNs(0), + m_nextDeadlineNs(0), + m_initialized(false) +{ +} + +bool StereoOutputRateLimiter::shouldOutput(const FrameTiming& left) +{ + if (m_targetFps == 0) + return true; + + TimestampSource source = TIMESTAMP_STEADY; + uint64_t timestampNs = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); + if (left.hasSensorSofTimestampTsc) + { + source = TIMESTAMP_TSC; + timestampNs = left.sensorSofTimestampTscNs; + } + else if (left.hasSensorTimestamp) + { + source = TIMESTAMP_SENSOR; + timestampNs = left.sensorTimestampNs; + } + else if (left.frameTimeNs != 0) + { + source = TIMESTAMP_FRAME; + timestampNs = left.frameTimeNs; + } + + if (!m_initialized || source != m_source || timestampNs < m_lastTimestampNs) + { + m_source = source; + m_lastTimestampNs = timestampNs; + m_nextDeadlineNs = timestampNs; + m_initialized = true; + } + else + { + m_lastTimestampNs = timestampNs; + } + + if (timestampNs < m_nextDeadlineNs) + { + if (m_nextDeadlineNs - timestampNs > m_toleranceNs) + return false; + if (m_intervalNs > std::numeric_limits::max() - m_nextDeadlineNs) + { + m_nextDeadlineNs = timestampNs; + } + else + { + m_nextDeadlineNs += m_intervalNs; + } + return true; + } + + const uint64_t elapsedDeadlineNs = timestampNs - m_nextDeadlineNs; + const uint64_t intervals = elapsedDeadlineNs / m_intervalNs + 1; + if (intervals > (std::numeric_limits::max() - m_nextDeadlineNs) / m_intervalNs) + { + m_nextDeadlineNs = timestampNs; + } + else + { + m_nextDeadlineNs += intervals * m_intervalNs; + } + return true; +} + +int64_t signedDeltaNs(uint64_t rightNs, uint64_t leftNs) +{ + if (rightNs >= leftNs) + return static_cast(rightNs - leftNs); + return -static_cast(leftNs - rightNs); +} + +uint64_t readSenderClockNs(uint32_t* clockDomain) +{ + struct timespec ts; + // Use the chrony-disciplined Unix wall clock for cross-host latency. + // CLOCK_TAI offset is not guaranteed to be initialized consistently across + // the Thor sender and x86 receiver hosts. + if (clock_gettime(CLOCK_REALTIME, &ts) == 0) + { + if (clockDomain) + *clockDomain = SENDER_CLOCK_REALTIME; + return static_cast(ts.tv_sec) * 1000000000ULL + static_cast(ts.tv_nsec); + } + + if (clockDomain) + *clockDomain = SENDER_CLOCK_UNKNOWN; + return 0; +} + +uint64_t readTscNowNs() +{ +#if defined(__aarch64__) + // The ARM generic timer is the Tegra TSC domain that stamps VI + // start-of-frame (Argus getSensorSofTimestampTsc). Scale by the reported + // frequency (31.25 MHz on Orin) rather than a constant. + uint64_t ticks = 0; + uint64_t freq = 0; + __asm__ __volatile__("mrs %0, cntvct_el0" : "=r"(ticks)); + __asm__ __volatile__("mrs %0, cntfrq_el0" : "=r"(freq)); + if (freq == 0) + return 0; + const uint64_t kNsPerSecond = 1000000000ULL; + return (ticks / freq) * kNsPerSecond + ((ticks % freq) * kNsPerSecond) / freq; +#else + return 0; +#endif +} + +namespace +{ +bool applyCaptureMetadata(const Argus::CaptureMetadata* metadata, FrameTiming* timing) +{ + if (!metadata || !timing) + return false; + const Argus::ICaptureMetadata* iMetadata = Argus::interface_cast(metadata); + if (iMetadata) + { + timing->sourceIndex = iMetadata->getSourceIndex(); + timing->sensorExposureTimeNs = iMetadata->getSensorExposureTime(); + timing->hasSensorExposureTime = true; + timing->sensorAnalogGain = iMetadata->getSensorAnalogGain(); + timing->hasSensorAnalogGain = true; + timing->ispDigitalGain = iMetadata->getIspDigitalGain(); + timing->hasIspDigitalGain = true; + const Argus::AwbMode awbMode = iMetadata->getAwbMode(); + std::snprintf(timing->awbModeName, sizeof(timing->awbModeName), "%s", awbMode.getName()); + timing->hasAwbMode = true; + timing->awbCct = iMetadata->getAwbCct(); + timing->hasAwbCct = true; + const Argus::BayerTuple awbGains = iMetadata->getAwbGains(); + timing->awbGainR = awbGains.r(); + timing->awbGainGEven = awbGains.gEven(); + timing->awbGainGOdd = awbGains.gOdd(); + timing->awbGainB = awbGains.b(); + timing->hasAwbGains = true; + timing->hasSourceIndex = true; + timing->sensorTimestampNs = iMetadata->getSensorTimestamp(); + timing->hasSensorTimestamp = true; + if (timing->frameNumber == 0) + timing->frameNumber = iMetadata->getCaptureId(); + if (timing->frameTimeNs == 0) + timing->frameTimeNs = timing->sensorTimestampNs; + } + const Argus::Ext::ISensorTimestampTsc* iSensorTimestampTsc = + Argus::interface_cast(metadata); + if (iSensorTimestampTsc) + { + const uint64_t tsc0 = iSensorTimestampTsc->getSensorSofTimestampTsc(); + const uint64_t tsc1 = iSensorTimestampTsc->getSensorSofTimestampTsc2(); + const uint64_t selectedTsc = timing->hasSourceIndex && timing->sourceIndex == 1 ? tsc1 : tsc0; + if (selectedTsc != 0) + { + timing->sensorSofTimestampTscNs = selectedTsc; + timing->hasSensorSofTimestampTsc = true; + } + } + return true; +} +} // namespace + +bool readFrameTiming(EGLStream::Frame* frame, const char* label, FrameTiming* timing) +{ + if (timing) + *timing = FrameTiming(); + + EGLStream::IFrame* iFrame = Argus::interface_cast(frame); + if (!iFrame) + { + std::fprintf(stderr, "Failed to get %s EGLStream IFrame interface\n", label); + return false; + } + + if (!timing) + return true; + + timing->frameNumber = iFrame->getNumber(); + timing->frameTimeNs = iFrame->getTime(); + EGLStream::IArgusCaptureMetadata* iArgusCaptureMetadata = + Argus::interface_cast(frame); + return !iArgusCaptureMetadata || applyCaptureMetadata(iArgusCaptureMetadata->getMetadata(), timing); +} + +bool readFrameTiming(Argus::Buffer* buffer, const char* label, FrameTiming* timing) +{ + if (timing) + *timing = FrameTiming(); + Argus::IBuffer* iBuffer = Argus::interface_cast(buffer); + if (!iBuffer) + { + std::fprintf(stderr, "Failed to get %s Argus IBuffer interface\n", label); + return false; + } + if (!timing) + return true; + const Argus::CaptureMetadata* metadata = iBuffer->getMetadata(); + if (!metadata) + { + std::fprintf(stderr, "Failed to get %s direct-buffer metadata\n", label); + return false; + } + return applyCaptureMetadata(metadata, timing); +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/StereoTiming.h b/examples/camera_streamer/argus_sender/src/StereoTiming.h new file mode 100644 index 0000000000..894d1b9a47 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/StereoTiming.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_STEREO_TIMING_H +#define ARGUS_SENDER_STEREO_TIMING_H + +#include +#include + +#include + +namespace argus_sender +{ + +enum SenderClockDomain +{ + SENDER_CLOCK_UNKNOWN = 0, + SENDER_CLOCK_TAI = 1, + SENDER_CLOCK_REALTIME = 2 +}; + +struct FrameTiming +{ + uint64_t frameNumber = 0; + uint64_t frameNumberGap = 0; + uint64_t frameTimeNs = 0; + uint64_t acquiredSteadyNs = 0; + // Tegra TSC sampled immediately after the Argus acquire call returned. + // This shares the VI SOF timestamp's clock domain on Orin. + uint64_t acquiredTscNs = 0; + uint64_t sensorExposureTimeNs = 0; + bool hasSensorExposureTime = false; + float sensorAnalogGain = 0.0f; + bool hasSensorAnalogGain = false; + float ispDigitalGain = 0.0f; + bool hasIspDigitalGain = false; + char awbModeName[32] = {}; + bool hasAwbMode = false; + uint32_t awbCct = 0; + bool hasAwbCct = false; + float awbGainR = 0.0f; + float awbGainGEven = 0.0f; + float awbGainGOdd = 0.0f; + float awbGainB = 0.0f; + bool hasAwbGains = false; + uint32_t sourceIndex = 0; + bool hasSourceIndex = false; + uint64_t sensorTimestampNs = 0; + bool hasSensorTimestamp = false; + uint64_t sensorSofTimestampTscNs = 0; + bool hasSensorSofTimestampTsc = false; +}; + +class StereoOutputRateLimiter +{ +public: + explicit StereoOutputRateLimiter(uint32_t targetFps); + bool shouldOutput(const FrameTiming& left); + +private: + enum TimestampSource + { + TIMESTAMP_NONE, + TIMESTAMP_TSC, + TIMESTAMP_SENSOR, + TIMESTAMP_FRAME, + TIMESTAMP_STEADY + }; + + uint32_t m_targetFps; + uint64_t m_intervalNs; + uint64_t m_toleranceNs; + TimestampSource m_source; + uint64_t m_lastTimestampNs; + uint64_t m_nextDeadlineNs; + bool m_initialized; +}; + +int64_t signedDeltaNs(uint64_t rightNs, uint64_t leftNs); +uint64_t readSenderClockNs(uint32_t* clockDomain = 0); +uint64_t readTscNowNs(); +bool readFrameTiming(EGLStream::Frame* frame, const char* label, FrameTiming* timing); +bool readFrameTiming(Argus::Buffer* buffer, const char* label, FrameTiming* timing); + +} // namespace argus_sender + +#endif // ARGUS_SENDER_STEREO_TIMING_H diff --git a/examples/camera_streamer/argus_sender/src/StreamProcessor.cpp b/examples/camera_streamer/argus_sender/src/StreamProcessor.cpp new file mode 100644 index 0000000000..3dc4b925c7 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/StreamProcessor.cpp @@ -0,0 +1,536 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "StreamProcessor.h" + +#include "CaptureManager.h" +#include "FrameBundleAssembler.h" +#include "FrameCopyWorker.h" +#include "HevcEncoder.h" +#include "NvRemap.h" +#include "StereoTiming.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace argus_sender +{ +namespace +{ + +const uint32_t kProcessorBuffers = 2; + +uint64_t steadyNowNs() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +void initializeV4l2Buffer(uint32_t index, struct v4l2_buffer* buffer, struct v4l2_plane planes[MAX_PLANES]) +{ + std::memset(buffer, 0, sizeof(*buffer)); + std::memset(planes, 0, sizeof(struct v4l2_plane) * MAX_PLANES); + buffer->index = index; + buffer->m.planes = planes; +} + +} // namespace + +class StreamProcessor::Impl +{ +public: + struct CameraState + { + std::string label; + std::vector buffers; + std::deque freeIndices; + std::unique_ptr encoder; + std::unique_ptr copyWorker; + // Raw wire payload produced on the copy worker's thread (tight pack + // or remap); collected by finishSubmit after wait(). + std::vector pendingPayload; + DmaBuffer* pendingInput = NULL; + uint32_t inFlight = 0; + uint32_t selectedIndex = 0; + uint64_t directReadySteadyNs = 0; + bool copySubmitted = false; + }; + + Impl(const StreamConfig& config, + const ResolvedOutputGeometry& geometry, + size_t streamIndex, + bool kpi, + uint32_t logIntervalSeconds, + bool benchEncode) + : m_config(config), + m_geometry(geometry), + m_streamIndex(streamIndex), + m_assembler(NULL), + m_bundleSequence(0), + m_copyPending(false) + { + m_encoderSettings.bitrateMbps = config.output.bitrateMbps; + m_encoderSettings.kpi = kpi; + m_encoderSettings.logIntervalSeconds = logIntervalSeconds; + m_encoderSettings.benchEncode = benchEncode; + m_cameras.resize(config.cameras.size()); + for (size_t i = 0; i < m_cameras.size(); ++i) + { + m_cameras[i].label = + config.id + "." + (config.type == STREAM_TYPE_STEREO ? (i == 0 ? "left" : "right") : "mono"); + if (config.output.codec == CODEC_RAW_NV12) + { + m_cameras[i].copyWorker.reset(new FrameCopyWorker(m_cameras[i].label.c_str())); + } + } + } + + ~Impl() + { + shutdown(); + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + for (size_t i = 0; i < m_cameras[camera].buffers.size(); ++i) + delete m_cameras[camera].buffers[i]; + } + } + + bool start(FrameBundleAssembler* assembler, uint32_t encoderFps) + { + if (!assembler || encoderFps == 0) + return false; + m_assembler = assembler; + if (m_config.output.remap.enabled()) + { + std::string remapError; + if (!loadNvRemapTable(m_config.output.remap.file, m_geometry.captureWidth, m_geometry.captureHeight, + m_geometry.width, m_geometry.height, &m_remapTable, &remapError)) + { + std::fprintf(stderr, "Stream %s remap: %s\n", m_config.id.c_str(), remapError.c_str()); + return false; + } + std::printf( + "Stream %s remap: %s (%ux%u source -> %ux%u via %ux " + "supersampled LUT)\n", + m_config.id.c_str(), m_config.output.remap.file.c_str(), m_geometry.captureWidth, + m_geometry.captureHeight, m_geometry.width, m_geometry.height, m_remapTable.supersample); + } + const Argus::Size2D size(m_geometry.captureWidth, m_geometry.captureHeight); + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + CameraState& state = m_cameras[camera]; + if (m_config.output.codec == CODEC_RAW_NV12) + { + for (uint32_t i = 0; i < kProcessorBuffers; ++i) + { + DmaBuffer* buffer = DmaBuffer::create(size, NVBUF_COLOR_FORMAT_NV12, NVBUF_LAYOUT_PITCH); + if (!buffer) + { + std::fprintf(stderr, "Failed to allocate %s processing DMABUF %u\n", state.label.c_str(), i); + return false; + } + state.buffers.push_back(buffer); + state.freeIndices.push_back(i); + } + if (!state.copyWorker || !state.copyWorker->start()) + return false; + } + else + { + for (uint32_t i = 0; i < kEncoderOutputPlaneBuffers; ++i) + state.freeIndices.push_back(i); + state.encoder.reset( + new DirectHevcEncoder(assembler->encodedSink(m_streamIndex, camera), m_encoderSettings)); + const uint32_t gop = m_config.output.gop ? m_config.output.gop : encoderFps; + if (!state.encoder->initialize(m_geometry.width, m_geometry.height, encoderFps, gop)) + { + return false; + } + } + } + return true; + } + + bool waitForInputBuffers() + { + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + CameraState& state = m_cameras[camera]; + while (state.freeIndices.empty() && state.inFlight > 0) + { + if (!releaseOne(&state)) + return false; + } + if (state.freeIndices.empty()) + return false; + } + return true; + } + + bool submitCopies(CapturedFrameSet* frames, uint64_t bundleSequence) + { + if (!frames || frames->frames.size() != m_cameras.size() || frames->dmaBuffers.size() != m_cameras.size() || + m_copyPending || bundleSequence == 0) + { + return false; + } + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + CameraState& state = m_cameras[camera]; + const bool raw = m_config.output.codec == CODEC_RAW_NV12; + if (state.freeIndices.empty() || (raw && (!frames->frames[camera].get() || !state.copyWorker)) || + (!raw && !frames->dmaBuffers[camera].get())) + { + return false; + } + } + m_bundleSequence = bundleSequence; + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + CameraState& state = m_cameras[camera]; + state.selectedIndex = state.freeIndices.front(); + const bool raw = m_config.output.codec == CODEC_RAW_NV12; + if (raw) + { + RawPackJob pack; + pack.width = m_geometry.width; + pack.height = m_geometry.height; + pack.remap = m_remapTable.loaded() ? &m_remapTable : nullptr; + pack.payload = &state.pendingPayload; + state.copySubmitted = + state.copyWorker->submit(frames->frames[camera].get(), state.buffers[state.selectedIndex], &pack); + } + else + { + state.pendingInput = frames->dmaBuffers[camera].release(); + state.directReadySteadyNs = frames->timings[camera].acquiredSteadyNs; + state.copySubmitted = state.pendingInput != NULL; + } + if (!state.copySubmitted) + { + std::fprintf(stderr, "Failed to submit %s frame processing; draining previously submitted work\n", + state.label.c_str()); + discardPendingCopies(); + return false; + } + } + m_copyPending = true; + return true; + } + + bool finishSubmit() + { + if (!m_copyPending) + return false; + bool ok = true; + uint64_t copyDoneSteadyNs = 0; + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + CameraState& state = m_cameras[camera]; + uint64_t cameraCopyDoneSteadyNs = 0; + if (!state.copySubmitted) + ok = false; + else if (m_config.output.codec == CODEC_RAW_NV12) + { + if (!state.copyWorker->wait(&cameraCopyDoneSteadyNs)) + ok = false; + } + else + { + cameraCopyDoneSteadyNs = state.directReadySteadyNs; + } + copyDoneSteadyNs = std::max(copyDoneSteadyNs, cameraCopyDoneSteadyNs); + state.copySubmitted = false; + } + if (!ok) + { + releasePendingDirectInputs(); + m_copyPending = false; + return false; + } + + if (!m_assembler->recordCopyDone(m_streamIndex, m_bundleSequence, copyDoneSteadyNs)) + { + releasePendingDirectInputs(); + m_copyPending = false; + return false; + } + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + CameraState& state = m_cameras[camera]; + const uint32_t index = state.selectedIndex; + const uint64_t encodeStartSteadyNs = steadyNowNs(); + if (!m_assembler->recordProcessStart(m_streamIndex, m_bundleSequence, encodeStartSteadyNs)) + { + ok = false; + break; + } + if (m_config.output.codec == CODEC_RAW_NV12) + { + // The tight pack (and remap, when configured) already ran on + // the copy worker's thread; the payload is ready to submit. + if (state.pendingPayload.empty()) + { + ok = false; + break; + } + uint32_t clockDomain = SENDER_CLOCK_UNKNOWN; + const uint64_t processDoneNs = readSenderClockNs(&clockDomain); + if (!m_assembler->submitRaw(m_streamIndex, camera, m_bundleSequence, std::move(state.pendingPayload), + processDoneNs, clockDomain)) + { + ok = false; + break; + } + state.pendingPayload.clear(); + } + else + { + state.freeIndices.pop_front(); + DmaBuffer* input = state.pendingInput; + state.pendingInput = NULL; + struct v4l2_buffer v4l2Buffer; + struct v4l2_plane planes[MAX_PLANES]; + initializeV4l2Buffer(index, &v4l2Buffer, planes); + if (!state.encoder->queueInput(v4l2Buffer, input, m_bundleSequence)) + { + state.freeIndices.push_front(index); + if (input && !input->releaseToArgus()) + { + std::fprintf(stderr, "Failed to return %s direct input after encoder queue failure\n", + state.label.c_str()); + } + ok = false; + break; + } + ++state.inFlight; + state.directReadySteadyNs = 0; + } + } + if (!ok) + releasePendingDirectInputs(); + m_copyPending = false; + return ok; + } + + bool discardPendingCopies() + { + bool ok = true; + bool hadPending = m_copyPending; + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + CameraState& state = m_cameras[camera]; + if (!state.copySubmitted && !state.pendingInput) + continue; + hadPending = true; + if (m_config.output.codec == CODEC_RAW_NV12) + { + uint64_t ignoredCompletionSteadyNs = 0; + if (!state.copyWorker || !state.copyWorker->wait(&ignoredCompletionSteadyNs)) + { + ok = false; + } + } + else if (state.pendingInput) + { + if (!state.pendingInput->releaseToArgus()) + ok = false; + state.pendingInput = NULL; + } + state.copySubmitted = false; + state.directReadySteadyNs = 0; + } + m_copyPending = false; + m_bundleSequence = 0; + if (hadPending && !ok) + { + std::fprintf(stderr, "Failed while draining abandoned frame-copy work for stream %s\n", m_config.id.c_str()); + } + return ok; + } + + bool drain() + { + bool ok = true; + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + while (m_cameras[camera].inFlight > 0) + { + if (!releaseOne(&m_cameras[camera])) + { + ok = false; + break; + } + } + } + return ok; + } + + bool sendEos() + { + bool ok = true; + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + CameraState& state = m_cameras[camera]; + if (!state.encoder) + continue; + struct v4l2_buffer buffer; + struct v4l2_plane planes[MAX_PLANES]; + const uint32_t index = state.freeIndices.empty() ? 0 : state.freeIndices.front(); + initializeV4l2Buffer(index, &buffer, planes); + if (!state.encoder->sendEos(buffer)) + ok = false; + } + return ok; + } + + void shutdown() + { + releasePendingDirectInputs(); + // A receiver disconnect may skip the coordinator's normal drain. The + // output-plane dequeue owns the final reference to each direct Argus + // buffer, so reclaim those buffers before destroying encoder or Argus + // stream resources. + if (m_config.output.codec == CODEC_HEVC) + drain(); + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + if (m_cameras[camera].encoder) + m_cameras[camera].encoder->shutdown(); + if (m_cameras[camera].copyWorker) + m_cameras[camera].copyWorker->shutdown(); + } + } + + bool hasError() const + { + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + if (m_cameras[camera].encoder && m_cameras[camera].encoder->hasError()) + { + return true; + } + } + return false; + } + +private: + void releasePendingDirectInputs() + { + if (m_config.output.codec != CODEC_HEVC) + return; + for (size_t camera = 0; camera < m_cameras.size(); ++camera) + { + CameraState& state = m_cameras[camera]; + if (state.pendingInput) + { + if (!state.pendingInput->releaseToArgus()) + { + std::fprintf(stderr, "Failed to release pending %s direct input to Argus\n", state.label.c_str()); + } + state.pendingInput = NULL; + } + } + } + + bool releaseOne(CameraState* state) + { + if (!state || !state->encoder || state->inFlight == 0) + return false; + struct v4l2_buffer completed; + struct v4l2_plane planes[MAX_PLANES]; + initializeV4l2Buffer(0, &completed, planes); + DmaBuffer* buffer = NULL; + if (!waitForEncoderInput(state->encoder.get(), &completed, &buffer)) + return false; + if (completed.index >= kEncoderOutputPlaneBuffers || !buffer) + return false; + if (buffer->isArgusBound() && !buffer->releaseToArgus()) + { + std::fprintf(stderr, "Failed to release encoded direct DMABUF to Argus\n"); + // The encoder slot is still dequeued even if returning the backing + // surface to Argus failed. Account it before reporting the error so + // shutdown cannot spin forever on an already-dequeued slot. + state->freeIndices.push_back(completed.index); + --state->inFlight; + return false; + } + state->freeIndices.push_back(completed.index); + --state->inFlight; + return true; + } + + StreamConfig m_config; + ResolvedOutputGeometry m_geometry; + size_t m_streamIndex; + FrameBundleAssembler* m_assembler; + HevcEncoderSettings m_encoderSettings; + std::vector m_cameras; + NvRemapTable m_remapTable; + uint64_t m_bundleSequence; + bool m_copyPending; +}; + +StreamProcessor::StreamProcessor(const StreamConfig& config, + const ResolvedOutputGeometry& geometry, + size_t streamIndex, + bool kpi, + uint32_t logIntervalSeconds, + bool benchEncode) + : m_impl(new Impl(config, geometry, streamIndex, kpi, logIntervalSeconds, benchEncode)) +{ +} +StreamProcessor::~StreamProcessor() +{ +} +bool StreamProcessor::start(FrameBundleAssembler* assembler, uint32_t fps) +{ + return m_impl->start(assembler, fps); +} +bool StreamProcessor::waitForInputBuffers() +{ + return m_impl->waitForInputBuffers(); +} +bool StreamProcessor::submitCopies(CapturedFrameSet* frames, uint64_t sequence) +{ + return m_impl->submitCopies(frames, sequence); +} +bool StreamProcessor::finishSubmit() +{ + return m_impl->finishSubmit(); +} +bool StreamProcessor::discardPendingCopies() +{ + return m_impl->discardPendingCopies(); +} +bool StreamProcessor::drain() +{ + return m_impl->drain(); +} +bool StreamProcessor::sendEos() +{ + return m_impl->sendEos(); +} +void StreamProcessor::shutdown() +{ + m_impl->shutdown(); +} +bool StreamProcessor::hasError() const +{ + return m_impl->hasError(); +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/StreamProcessor.h b/examples/camera_streamer/argus_sender/src/StreamProcessor.h new file mode 100644 index 0000000000..2f9195de57 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/StreamProcessor.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_STREAM_PROCESSOR_H +#define ARGUS_SENDER_STREAM_PROCESSOR_H + +#include "Config.h" +#include "OutputGeometry.h" + +#include +#include + +namespace argus_sender +{ + +class FrameBundleAssembler; +struct CapturedFrameSet; + +class StreamProcessor +{ +public: + StreamProcessor(const StreamConfig& config, + const ResolvedOutputGeometry& geometry, + size_t streamIndex, + bool kpi, + uint32_t logIntervalSeconds, + bool benchEncode); + ~StreamProcessor(); + + bool start(FrameBundleAssembler* assembler, uint32_t encoderFps); + bool waitForInputBuffers(); + bool submitCopies(CapturedFrameSet* frames, uint64_t bundleSequence); + bool finishSubmit(); + bool discardPendingCopies(); + bool drain(); + bool sendEos(); + void shutdown(); + bool hasError() const; + +private: + class Impl; + std::unique_ptr m_impl; +}; + +} // namespace argus_sender + +#endif // ARGUS_SENDER_STREAM_PROCESSOR_H diff --git a/examples/camera_streamer/argus_sender/src/TcpTransport.cpp b/examples/camera_streamer/argus_sender/src/TcpTransport.cpp new file mode 100644 index 0000000000..527eb35820 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/TcpTransport.cpp @@ -0,0 +1,903 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "TcpTransport.h" + +#include "Runtime.h" +#include "StereoTiming.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace argus_sender +{ +uint64_t tcpSteadyNowNs() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} + +struct LatencyAccumulator +{ + uint64_t samples = 0; + uint64_t sumNs = 0; + uint64_t maxNs = 0; + + void record(uint64_t valueNs) + { + ++samples; + sumNs += valueNs; + maxNs = std::max(maxNs, valueNs); + } + + double averageMs() const + { + return samples > 0 ? static_cast(sumNs) / static_cast(samples) / 1000000.0 : 0.0; + } + + double maximumMs() const + { + return static_cast(maxNs) / 1000000.0; + } + + void reset() + { + samples = 0; + sumNs = 0; + maxNs = 0; + } +}; + +struct StreamLatencyStats +{ + bool encoded = false; + LatencyAccumulator acquireOffset; + LatencyAccumulator acquireSpread; + LatencyAccumulator captureToCopyDone; + LatencyAccumulator captureToProcessStart; + LatencyAccumulator captureToProcessDone; + LatencyAccumulator processDoneToSent; + LatencyAccumulator captureToSent; + + void reset() + { + acquireOffset.reset(); + acquireSpread.reset(); + captureToCopyDone.reset(); + captureToProcessStart.reset(); + captureToProcessDone.reset(); + processDoneToSent.reset(); + captureToSent.reset(); + } +}; + +void recordDuration(uint64_t startNs, uint64_t endNs, LatencyAccumulator* metric) +{ + if (metric && startNs != 0 && endNs >= startNs) + metric->record(endNs - startNs); +} + +bool shouldStop(const std::atomic* abortFlag) +{ + return stopRequested() || (abortFlag && abortFlag->load()); +} + +int acceptTcpClient(const std::string& bindHost, uint16_t port, uint32_t retryDelayMs, const std::atomic* abortFlag) +{ + char portString[16]; + std::snprintf(portString, sizeof(portString), "%u", static_cast(port)); + + struct addrinfo hints; + std::memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + + while (!shouldStop(abortFlag)) + { + struct addrinfo* result = NULL; + const char* node = bindHost.empty() ? NULL : bindHost.c_str(); + int rc = getaddrinfo(node, portString, &hints, &result); + if (rc != 0) + { + std::fprintf(stderr, "getaddrinfo(%s:%s) failed: %s\n", bindHost.c_str(), portString, gai_strerror(rc)); + } + else + { + for (struct addrinfo* rp = result; rp != NULL && !shouldStop(abortFlag); rp = rp->ai_next) + { + int listenFd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + if (listenFd < 0) + continue; + + int yes = 1; + setsockopt(listenFd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + struct timeval tv; + tv.tv_sec = 0; + tv.tv_usec = 500000; + setsockopt(listenFd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + + if (bind(listenFd, rp->ai_addr, rp->ai_addrlen) != 0 || listen(listenFd, 1) != 0) + { + close(listenFd); + continue; + } + + std::printf("Listening for TCP receiver on %s:%u\n", bindHost.c_str(), port); + while (!shouldStop(abortFlag)) + { + struct sockaddr_storage peerAddr; + socklen_t peerLen = sizeof(peerAddr); + int fd = accept(listenFd, reinterpret_cast(&peerAddr), &peerLen); + if (fd >= 0) + { + int nodelay = 1; + setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof(nodelay)); + + char peerHost[NI_MAXHOST]; + char peerService[NI_MAXSERV]; + if (getnameinfo(reinterpret_cast(&peerAddr), peerLen, peerHost, sizeof(peerHost), + peerService, sizeof(peerService), NI_NUMERICHOST | NI_NUMERICSERV) == 0) + { + std::printf("Receiver connected from %s:%s\n", peerHost, peerService); + } + else + { + std::printf("Receiver connected\n"); + } + close(listenFd); + freeaddrinfo(result); + return fd; + } + if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) + continue; + std::fprintf(stderr, "accept() failed: %s\n", std::strerror(errno)); + break; + } + close(listenFd); + } + freeaddrinfo(result); + } + + std::fprintf(stderr, "Waiting to listen for TCP receiver on %s:%u...\n", bindHost.c_str(), port); + const uint32_t sleepStepMs = 50; + uint32_t sleptMs = 0; + while (sleptMs < retryDelayMs && !shouldStop(abortFlag)) + { + const uint32_t remainingMs = retryDelayMs - sleptMs; + const uint32_t waitMs = remainingMs < sleepStepMs ? remainingMs : sleepStepMs; + std::this_thread::sleep_for(std::chrono::milliseconds(waitMs)); + sleptMs += waitMs; + } + } + + return -1; +} + +bool sendAll(int fd, + const uint8_t* data, + size_t size, + const std::atomic* abortFlag, + const std::atomic* deadlineSteadyNs) +{ + size_t sent = 0; + while (sent < size && !shouldStop(abortFlag)) + { + const uint64_t deadline = deadlineSteadyNs ? deadlineSteadyNs->load() : 0; + if (deadline != 0 && tcpSteadyNowNs() >= deadline) + return false; +#ifdef MSG_NOSIGNAL + ssize_t n = send(fd, data + sent, size - sent, MSG_NOSIGNAL); +#else + ssize_t n = send(fd, data + sent, size - sent, 0); +#endif + if (n > 0) + { + sent += static_cast(n); + continue; + } + if (n < 0 && errno == EINTR) + continue; + if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) + continue; + if (!shouldStop(abortFlag)) + { + if (n == 0) + std::fprintf(stderr, "TCP send returned zero; closing receiver session\n"); + else + std::fprintf(stderr, "TCP send failed (%s); closing receiver session\n", std::strerror(errno)); + } + return false; + } + return !shouldStop(abortFlag); +} + + +class TcpPacketSender::Impl +{ +public: + struct QueuedPacket + { + std::vector bytes; + PacketTiming timing; + }; + + Impl(int* fd, + const TcpPacketSenderSettings& settings, + uint32_t maxQueuedPackets, + uint16_t port = 0, + const char* label = NULL, + bool appendDump = false) + : m_fd(fd), + m_settings(settings), + m_port(port ? port : settings.port), + m_label(label ? label : ""), + m_maxQueuedPackets(maxQueuedPackets ? maxQueuedPackets : 1), + m_appendDump(appendDump), + m_dumpFd(-1), + m_dumpBytes(0), + m_accepting(false), + m_abort(false), + m_drainDeadlineSteadyNs(0), + m_error(false), + m_disconnected(false), + m_totalPackets(0), + m_intervalPackets(0), + m_intervalBytes(0), + m_enqueueWaitSamples(0), + m_enqueueWaitSumNs(0), + m_enqueueWaitMaxNs(0), + m_intervalSendSamples(0), + m_intervalSendSumNs(0), + m_intervalSendMaxNs(0), + m_intervalLatencySamples(0), + m_intervalEncodeStartSamples(0), + m_acquireSpreadSumNs(0), + m_acquireSpreadMaxNs(0), + m_captureToEncodeStartSumNs(0), + m_captureToEncodeStartMaxNs(0), + m_captureToProcessDoneSumNs(0), + m_captureToProcessDoneMaxNs(0), + m_captureToSentSumNs(0), + m_captureToSentMaxNs(0) + { + } + + ~Impl() + { + shutdown(false); + } + + bool start() + { + if (m_thread.joinable()) + return true; + if (!openWireDump()) + return false; + if (!m_fd || *m_fd < 0 || m_settings.sendPollTimeoutMs == 0) + { + std::fprintf(stderr, "Invalid TCP sender socket or send timeout\n"); + closeWireDump(); + return false; + } + struct timeval sendTimeout; + sendTimeout.tv_sec = m_settings.sendPollTimeoutMs / 1000U; + sendTimeout.tv_usec = static_cast(m_settings.sendPollTimeoutMs % 1000U) * 1000; + if (setsockopt(*m_fd, SOL_SOCKET, SO_SNDTIMEO, &sendTimeout, sizeof(sendTimeout)) != 0) + { + std::fprintf(stderr, "Failed to configure TCP send timeout: %s\n", std::strerror(errno)); + closeWireDump(); + return false; + } + m_statsStart = std::chrono::steady_clock::now(); + m_accepting = true; + m_abort = false; + m_drainDeadlineSteadyNs = 0; + m_error = false; + m_disconnected = false; + try + { + m_thread = std::thread(&Impl::threadMain, this); + } + catch (...) + { + m_accepting = false; + m_error = true; + closeWireDump(); + std::fprintf(stderr, "Failed to start HEVC TCP sender thread\n"); + return false; + } + return true; + } + + bool enqueueFramed(std::vector&& packet, const PacketTiming& timing) + { + if (packet.size() <= sizeof(uint32_t) || packet.size() - sizeof(uint32_t) > std::numeric_limits::max()) + { + std::fprintf(stderr, "Invalid length-prefixed TCP packet size: %zu bytes\n", packet.size()); + m_error = true; + return false; + } + + uint32_t framedSize = 0; + std::memcpy(&framedSize, packet.data(), sizeof(framedSize)); + const size_t payloadSize = packet.size() - sizeof(framedSize); + if (ntohl(framedSize) != payloadSize) + { + std::fprintf( + stderr, "Length-prefixed TCP packet mismatch: prefix=%u payload=%zu\n", ntohl(framedSize), payloadSize); + m_error = true; + return false; + } + const auto enqueueWaitStart = std::chrono::steady_clock::now(); + std::unique_lock lock(m_mutex); + while (m_queue.size() >= m_maxQueuedPackets && m_accepting && !m_abort && !m_error && !stopRequested()) + { + m_spaceAvailable.wait_for(lock, std::chrono::milliseconds(1)); + } + + if (!m_accepting || m_abort || m_error || stopRequested()) + return false; + + QueuedPacket queued; + queued.bytes = std::move(packet); + queued.timing = timing; + m_queue.push_back(std::move(queued)); + lock.unlock(); + + const uint64_t enqueueWaitNs = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - enqueueWaitStart) + .count()); + { + std::lock_guard metricsLock(m_metricsMutex); + ++m_enqueueWaitSamples; + m_enqueueWaitSumNs += enqueueWaitNs; + if (enqueueWaitNs > m_enqueueWaitMaxNs) + m_enqueueWaitMaxNs = enqueueWaitNs; + } + m_packetAvailable.notify_one(); + return true; + } + + bool hasError() const + { + return m_error.load(); + } + + bool disconnected() const + { + return m_disconnected.load(); + } + + void shutdown(bool drain) + { + if (m_thread.joinable()) + { + { + std::lock_guard lock(m_mutex); + m_accepting = false; + if (drain) + { + const uint64_t timeoutNs = static_cast(m_settings.drainTimeoutMs) * 1000000ULL; + const uint64_t nowNs = tcpSteadyNowNs(); + m_drainDeadlineSteadyNs = timeoutNs > std::numeric_limits::max() - nowNs ? + std::numeric_limits::max() : + nowNs + timeoutNs; + } + else + { + m_abort = true; + m_queue.clear(); + } + } + + m_packetAvailable.notify_all(); + m_spaceAvailable.notify_all(); + if (!drain && m_fd && *m_fd >= 0) + ::shutdown(*m_fd, SHUT_RDWR); + + m_thread.join(); + } + closeWireDump(); + } + +private: + bool openWireDump() + { + if (m_settings.dumpWirePath.empty()) + return true; + + const int flags = O_WRONLY | O_CREAT | (m_appendDump ? O_APPEND : O_TRUNC); + m_dumpFd = ::open(m_settings.dumpWirePath.c_str(), flags, 0666); + if (m_dumpFd < 0) + { + std::fprintf( + stderr, "Failed to open wire dump %s: %s\n", m_settings.dumpWirePath.c_str(), std::strerror(errno)); + m_error = true; + return false; + } + + std::printf("Wire dump %s: %s\n", m_appendDump ? "appending" : "started", m_settings.dumpWirePath.c_str()); + return true; + } + + bool writeWireDump(const uint8_t* data, size_t size) + { + if (m_dumpFd < 0) + return true; + + size_t written = 0; + while (written < size) + { + const ssize_t n = ::write(m_dumpFd, data + written, size - written); + if (n > 0) + { + written += static_cast(n); + continue; + } + if (n < 0 && errno == EINTR) + continue; + + if (n == 0) + std::fprintf(stderr, "Wire dump write returned zero for %s\n", m_settings.dumpWirePath.c_str()); + else + std::fprintf(stderr, "Wire dump write failed for %s: %s\n", m_settings.dumpWirePath.c_str(), + std::strerror(errno)); + return false; + } + + m_dumpBytes += size; + return true; + } + + void closeWireDump() + { + if (m_dumpFd < 0) + return; + + if (::close(m_dumpFd) != 0) + std::fprintf( + stderr, "Failed to close wire dump %s: %s\n", m_settings.dumpWirePath.c_str(), std::strerror(errno)); + else + std::printf("Wire dump closed: %s, session_bytes=%llu\n", m_settings.dumpWirePath.c_str(), + static_cast(m_dumpBytes)); + m_dumpFd = -1; + } + + void recordStreamLatency(const PacketTiming& timing, uint64_t sendDoneSteadyNs) + { + if (m_streamLatencyStats.size() < timing.streams.size()) + m_streamLatencyStats.resize(timing.streams.size()); + for (size_t i = 0; i < timing.streams.size(); ++i) + { + const PacketStreamTiming& stream = timing.streams[i]; + if (!stream.present || stream.captureStartSteadyNs == 0) + continue; + StreamLatencyStats& stats = m_streamLatencyStats[i]; + stats.encoded = stream.encoded; + if (timing.captureStartSteadyNs != 0 && stream.captureStartSteadyNs >= timing.captureStartSteadyNs) + { + stats.acquireOffset.record(stream.captureStartSteadyNs - timing.captureStartSteadyNs); + } + stats.acquireSpread.record(stream.acquireSpreadNs); + recordDuration(stream.captureStartSteadyNs, stream.copyDoneSteadyNs, &stats.captureToCopyDone); + recordDuration(stream.captureStartSteadyNs, stream.processStartSteadyNs, &stats.captureToProcessStart); + recordDuration(stream.captureStartSteadyNs, stream.processDoneSteadyNs, &stats.captureToProcessDone); + recordDuration(stream.processDoneSteadyNs, sendDoneSteadyNs, &stats.processDoneToSent); + recordDuration(stream.captureStartSteadyNs, sendDoneSteadyNs, &stats.captureToSent); + } + } + + void recordSofToSentLatency(const PacketTiming& timing, uint64_t sendDoneTscNs) + { + if (sendDoneTscNs == 0) + return; + if (m_sofToSentStats.size() < timing.samples.size()) + m_sofToSentStats.resize(timing.samples.size()); + for (size_t i = 0; i < timing.samples.size(); ++i) + { + const PacketSampleTiming& sample = timing.samples[i]; + if (sample.present && sample.sensorSofTscNs != 0 && sendDoneTscNs >= sample.sensorSofTscNs) + { + m_sofToSentStats[i].record(sendDoneTscNs - sample.sensorSofTscNs); + } + } + } + + void logAndResetSofToSentLatency() + { + bool first = true; + for (size_t i = 0; i < m_sofToSentStats.size(); ++i) + { + LatencyAccumulator& stats = m_sofToSentStats[i]; + if (stats.samples == 0) + continue; + if (first) + { + std::printf("argus_sender[sof-to-sent]: sensor_sof_to_tcp_sent_ms(avg/max) "); + } + const std::string label = i < m_settings.sampleLabels.size() && !m_settings.sampleLabels[i].empty() ? + m_settings.sampleLabels[i] : + std::string("camera") + std::to_string(i); + std::printf("%s%s=%.3f/%.3f", first ? "" : ", ", label.c_str(), stats.averageMs(), stats.maximumMs()); + first = false; + stats.reset(); + } + if (!first) + std::printf("\n"); + } + + void logAndResetStreamLatency() + { + if (!verboseLoggingEnabled()) + { + bool first = true; + for (size_t i = 0; i < m_streamLatencyStats.size(); ++i) + { + StreamLatencyStats& stats = m_streamLatencyStats[i]; + if (stats.captureToSent.samples == 0) + continue; + const std::string label = i < m_settings.streamLabels.size() && !m_settings.streamLabels[i].empty() ? + m_settings.streamLabels[i] : + std::string("stream") + std::to_string(i); + if (first) + { + std::printf("argus_sender[e2e]: capture_to_tcp_sent_ms(avg/max) "); + } + std::printf("%s%s=%.3f/%.3f", first ? "" : ", ", label.c_str(), stats.captureToSent.averageMs(), + stats.captureToSent.maximumMs()); + first = false; + stats.reset(); + } + if (!first) + std::printf("\n"); + return; + } + + for (size_t i = 0; i < m_streamLatencyStats.size(); ++i) + { + StreamLatencyStats& stats = m_streamLatencyStats[i]; + if (stats.captureToSent.samples == 0) + continue; + const std::string label = i < m_settings.streamLabels.size() && !m_settings.streamLabels[i].empty() ? + m_settings.streamLabels[i] : + std::string("stream") + std::to_string(i); + if (stats.encoded) + { + std::printf( + "argus_sender[latency:%s]: n=%llu, " + "acquire_offset_ms(avg/max)=%.3f/%.3f, " + "stream_acquire_spread_ms(avg/max)=%.3f/%.3f, " + "capture_to_copy_done_ms(avg/max)=%.3f/%.3f, " + "capture_to_encode_start_ms(avg/max)=%.3f/%.3f, " + "capture_to_encode_done_ms(avg/max)=%.3f/%.3f, " + "encode_done_to_tcp_sent_ms(avg/max)=%.3f/%.3f, " + "capture_to_tcp_sent_ms(avg/max)=%.3f/%.3f\n", + label.c_str(), static_cast(stats.captureToSent.samples), + stats.acquireOffset.averageMs(), stats.acquireOffset.maximumMs(), stats.acquireSpread.averageMs(), + stats.acquireSpread.maximumMs(), stats.captureToCopyDone.averageMs(), + stats.captureToCopyDone.maximumMs(), stats.captureToProcessStart.averageMs(), + stats.captureToProcessStart.maximumMs(), stats.captureToProcessDone.averageMs(), + stats.captureToProcessDone.maximumMs(), stats.processDoneToSent.averageMs(), + stats.processDoneToSent.maximumMs(), stats.captureToSent.averageMs(), + stats.captureToSent.maximumMs()); + } + else + { + std::printf( + "argus_sender[latency:%s]: n=%llu, " + "acquire_offset_ms(avg/max)=%.3f/%.3f, " + "stream_acquire_spread_ms(avg/max)=%.3f/%.3f, " + "capture_to_copy_done_ms(avg/max)=%.3f/%.3f, " + "capture_to_raw_ready_ms(avg/max)=%.3f/%.3f, " + "raw_ready_to_tcp_sent_ms(avg/max)=%.3f/%.3f, " + "capture_to_tcp_sent_ms(avg/max)=%.3f/%.3f\n", + label.c_str(), static_cast(stats.captureToSent.samples), + stats.acquireOffset.averageMs(), stats.acquireOffset.maximumMs(), stats.acquireSpread.averageMs(), + stats.acquireSpread.maximumMs(), stats.captureToCopyDone.averageMs(), + stats.captureToCopyDone.maximumMs(), stats.captureToProcessDone.averageMs(), + stats.captureToProcessDone.maximumMs(), stats.processDoneToSent.averageMs(), + stats.processDoneToSent.maximumMs(), stats.captureToSent.averageMs(), + stats.captureToSent.maximumMs()); + } + stats.reset(); + } + } + + void threadMain() + { + while (!m_abort && !stopRequested()) + { + QueuedPacket packet; + { + std::unique_lock lock(m_mutex); + m_packetAvailable.wait( + lock, [this]() { return m_abort || !m_accepting || !m_queue.empty() || stopRequested(); }); + + if (m_queue.empty()) + { + if (m_abort || !m_accepting || stopRequested()) + break; + continue; + } + + packet = std::move(m_queue.front()); + m_queue.pop_front(); + m_spaceAvailable.notify_one(); + } + + const auto sendStart = std::chrono::steady_clock::now(); + if (!sendAll(*m_fd, packet.bytes.data(), packet.bytes.size(), &m_abort, &m_drainDeadlineSteadyNs)) + { + if (!m_abort && !stopRequested()) + { + const uint64_t deadline = m_drainDeadlineSteadyNs.load(); + const bool drainTimedOut = deadline != 0 && tcpSteadyNowNs() >= deadline; + if (drainTimedOut) + { + std::fprintf(stderr, "TCP drain timed out after %u ms; aborting the sender socket\n", + m_settings.drainTimeoutMs); + if (m_fd && *m_fd >= 0) + ::shutdown(*m_fd, SHUT_RDWR); + } + else + { + m_disconnected = true; + } + m_error = true; + } + break; + } + const uint64_t sendNs = static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - sendStart).count()); + ++m_intervalSendSamples; + m_intervalSendSumNs += sendNs; + if (sendNs > m_intervalSendMaxNs) + m_intervalSendMaxNs = sendNs; + + const uint64_t sendDoneSteadyNs = tcpSteadyNowNs(); + // Argus sensor SOF metadata and CNTVCT_EL0 use the same Tegra TSC + // domain. This endpoint is sampled after the complete framed + // packet has been written to the TCP socket. + const uint64_t sendDoneTscNs = readTscNowNs(); + if (packet.timing.captureStartSteadyNs != 0 && sendDoneSteadyNs >= packet.timing.captureStartSteadyNs) + { + const uint64_t captureToSentNs = sendDoneSteadyNs - packet.timing.captureStartSteadyNs; + ++m_intervalLatencySamples; + m_acquireSpreadSumNs += packet.timing.acquireSpreadNs; + if (packet.timing.acquireSpreadNs > m_acquireSpreadMaxNs) + m_acquireSpreadMaxNs = packet.timing.acquireSpreadNs; + if (packet.timing.captureToEncodeStartNs != 0) + { + ++m_intervalEncodeStartSamples; + m_captureToEncodeStartSumNs += packet.timing.captureToEncodeStartNs; + if (packet.timing.captureToEncodeStartNs > m_captureToEncodeStartMaxNs) + { + m_captureToEncodeStartMaxNs = packet.timing.captureToEncodeStartNs; + } + } + m_captureToProcessDoneSumNs += packet.timing.captureToProcessDoneNs; + if (packet.timing.captureToProcessDoneNs > m_captureToProcessDoneMaxNs) + { + m_captureToProcessDoneMaxNs = packet.timing.captureToProcessDoneNs; + } + m_captureToSentSumNs += captureToSentNs; + if (captureToSentNs > m_captureToSentMaxNs) + m_captureToSentMaxNs = captureToSentNs; + } + + if (m_settings.kpi && m_settings.logIntervalSeconds > 0) + { + recordStreamLatency(packet.timing, sendDoneSteadyNs); + recordSofToSentLatency(packet.timing, sendDoneTscNs); + } + if (!writeWireDump(packet.bytes.data(), packet.bytes.size())) + { + m_error = true; + break; + } + + ++m_totalPackets; + ++m_intervalPackets; + if (packet.bytes.size() >= 4) + m_intervalBytes += packet.bytes.size() - 4; + + if (m_settings.kpi && m_settings.logIntervalSeconds > 0) + { + const auto now = std::chrono::steady_clock::now(); + const double elapsed = + std::chrono::duration_cast>(now - m_statsStart).count(); + if (elapsed >= static_cast(m_settings.logIntervalSeconds)) + { + const double fpsActual = static_cast(m_intervalPackets) / elapsed; + const double mbps = static_cast(m_intervalBytes) * 8.0 / elapsed / 1.0e6; + uint64_t enqueueWaitSamples = 0; + uint64_t enqueueWaitSumNs = 0; + uint64_t enqueueWaitMaxNs = 0; + { + std::lock_guard metricsLock(m_metricsMutex); + enqueueWaitSamples = m_enqueueWaitSamples; + enqueueWaitSumNs = m_enqueueWaitSumNs; + enqueueWaitMaxNs = m_enqueueWaitMaxNs; + m_enqueueWaitSamples = 0; + m_enqueueWaitSumNs = 0; + m_enqueueWaitMaxNs = 0; + } + const double enqueueWaitAvgUs = + enqueueWaitSamples > 0 ? static_cast(enqueueWaitSumNs) / enqueueWaitSamples / 1000.0 : + 0.0; + const double sendAvgUs = m_intervalSendSamples > 0 ? static_cast(m_intervalSendSumNs) / + m_intervalSendSamples / 1000.0 : + 0.0; + const double captureToEncodeStartAvgMs = + m_intervalEncodeStartSamples > 0 ? + static_cast(m_captureToEncodeStartSumNs) / m_intervalEncodeStartSamples / 1000000.0 : + 0.0; + const double acquireSpreadAvgMs = + m_intervalLatencySamples > 0 ? + static_cast(m_acquireSpreadSumNs) / m_intervalLatencySamples / 1000000.0 : + 0.0; + const double captureToProcessDoneAvgMs = + m_intervalLatencySamples > 0 ? + static_cast(m_captureToProcessDoneSumNs) / m_intervalLatencySamples / 1000000.0 : + 0.0; + const double captureToSentAvgMs = + m_intervalLatencySamples > 0 ? + static_cast(m_captureToSentSumNs) / m_intervalLatencySamples / 1000000.0 : + 0.0; + if (verboseLoggingEnabled()) + { + if (m_label.empty()) + std::printf("argus_sender[tcp-hevc]: "); + else + std::printf("argus_sender[tcp-hevc:%s]: ", m_label.c_str()); + std::printf( + "%.2f FPS, %.2f Mbps, " + "tcp_enqueue_wait_us(avg/max)=%.3f/%.3f, " + "tcp_send_us(avg/max)=%.3f/%.3f, " + "acquire_spread_ms(avg/max)=%.3f/%.3f, " + "capture_to_encode_start_ms(avg/max)=%.3f/%.3f, " + "capture_to_encode_done_ms(avg/max)=%.3f/%.3f, " + "capture_to_tcp_sent_ms(avg/max)=%.3f/%.3f, " + "total=%llu\n", + fpsActual, mbps, enqueueWaitAvgUs, static_cast(enqueueWaitMaxNs) / 1000.0, + sendAvgUs, static_cast(m_intervalSendMaxNs) / 1000.0, acquireSpreadAvgMs, + static_cast(m_acquireSpreadMaxNs) / 1000000.0, captureToEncodeStartAvgMs, + static_cast(m_captureToEncodeStartMaxNs) / 1000000.0, captureToProcessDoneAvgMs, + static_cast(m_captureToProcessDoneMaxNs) / 1000000.0, captureToSentAvgMs, + static_cast(m_captureToSentMaxNs) / 1000000.0, + static_cast(m_totalPackets)); + } + logAndResetStreamLatency(); + logAndResetSofToSentLatency(); + m_intervalPackets = 0; + m_intervalBytes = 0; + m_intervalSendSamples = 0; + m_intervalSendSumNs = 0; + m_intervalSendMaxNs = 0; + m_intervalLatencySamples = 0; + m_intervalEncodeStartSamples = 0; + m_acquireSpreadSumNs = 0; + m_acquireSpreadMaxNs = 0; + m_captureToEncodeStartSumNs = 0; + m_captureToEncodeStartMaxNs = 0; + m_captureToProcessDoneSumNs = 0; + m_captureToProcessDoneMaxNs = 0; + m_captureToSentSumNs = 0; + m_captureToSentMaxNs = 0; + m_statsStart = now; + } + } + } + } + + int* m_fd; + const TcpPacketSenderSettings& m_settings; + const uint16_t m_port; + const std::string m_label; + const size_t m_maxQueuedPackets; + const bool m_appendDump; + int m_dumpFd; + uint64_t m_dumpBytes; + std::thread m_thread; + std::mutex m_mutex; + std::mutex m_metricsMutex; + std::condition_variable m_packetAvailable; + std::condition_variable m_spaceAvailable; + std::deque m_queue; + std::atomic m_accepting; + std::atomic m_abort; + std::atomic m_drainDeadlineSteadyNs; + std::atomic m_error; + std::atomic m_disconnected; + uint64_t m_totalPackets; + uint64_t m_intervalPackets; + uint64_t m_intervalBytes; + uint64_t m_enqueueWaitSamples; + uint64_t m_enqueueWaitSumNs; + uint64_t m_enqueueWaitMaxNs; + uint64_t m_intervalSendSamples; + uint64_t m_intervalSendSumNs; + uint64_t m_intervalSendMaxNs; + uint64_t m_intervalLatencySamples; + std::vector m_streamLatencyStats; + std::vector m_sofToSentStats; + uint64_t m_intervalEncodeStartSamples; + uint64_t m_acquireSpreadSumNs; + uint64_t m_acquireSpreadMaxNs; + uint64_t m_captureToEncodeStartSumNs; + uint64_t m_captureToEncodeStartMaxNs; + uint64_t m_captureToProcessDoneSumNs; + uint64_t m_captureToProcessDoneMaxNs; + uint64_t m_captureToSentSumNs; + uint64_t m_captureToSentMaxNs; + std::chrono::steady_clock::time_point m_statsStart; +}; + + +TcpPacketSender::TcpPacketSender(int* fd, + const TcpPacketSenderSettings& settings, + uint32_t maxQueuedPackets, + uint16_t port, + const char* label, + bool appendDump) + : m_impl(new Impl(fd, settings, maxQueuedPackets, port, label, appendDump)) +{ +} + +TcpPacketSender::~TcpPacketSender() +{ +} + +bool TcpPacketSender::start() +{ + return m_impl->start(); +} + +bool TcpPacketSender::enqueueFramed(std::vector&& packet) +{ + return m_impl->enqueueFramed(std::move(packet), PacketTiming()); +} + +bool TcpPacketSender::enqueueFramedWithTiming(std::vector&& packet, const PacketTiming& timing) +{ + return m_impl->enqueueFramed(std::move(packet), timing); +} + +bool TcpPacketSender::hasError() const +{ + return m_impl->hasError(); +} + +bool TcpPacketSender::disconnected() const +{ + return m_impl->disconnected(); +} + +void TcpPacketSender::shutdown(bool drain) +{ + m_impl->shutdown(drain); +} + +} // namespace argus_sender diff --git a/examples/camera_streamer/argus_sender/src/TcpTransport.h b/examples/camera_streamer/argus_sender/src/TcpTransport.h new file mode 100644 index 0000000000..d70d7a5b10 --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/TcpTransport.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ARGUS_SENDER_TCP_TRANSPORT_H +#define ARGUS_SENDER_TCP_TRANSPORT_H + +#include "PacketSink.h" + +#include +#include +#include +#include +#include + +namespace argus_sender +{ + +struct TcpPacketSenderSettings +{ + uint16_t port = 5000; + std::string dumpWirePath; + std::vector streamLabels; + std::vector sampleLabels; + bool kpi = true; + uint32_t logIntervalSeconds = 1; + uint32_t sendPollTimeoutMs = 100; + uint32_t drainTimeoutMs = 2000; +}; + +int acceptTcpClient(const std::string& bindHost, + uint16_t port, + uint32_t retryDelayMs, + const std::atomic* abortFlag = NULL); + +class TcpPacketSender : public PacketSink +{ +public: + TcpPacketSender(int* fd, + const TcpPacketSenderSettings& settings, + uint32_t maxQueuedPackets, + uint16_t port = 0, + const char* label = NULL, + bool appendDump = false); + ~TcpPacketSender(); + + bool start(); + bool enqueueFramed(std::vector&& packet) override; + bool enqueueFramedWithTiming(std::vector&& packet, const PacketTiming& timing) override; + bool hasError() const override; + bool disconnected() const; + void shutdown(bool drain); + +private: + class Impl; + std::unique_ptr m_impl; +}; + +} // namespace argus_sender + +#endif // ARGUS_SENDER_TCP_TRANSPORT_H diff --git a/examples/camera_streamer/argus_sender/src/main.cpp b/examples/camera_streamer/argus_sender/src/main.cpp new file mode 100644 index 0000000000..06a18b231d --- /dev/null +++ b/examples/camera_streamer/argus_sender/src/main.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "CaptureCoordinator.h" +#include "Config.h" +#include "Runtime.h" +#include "SenderSession.h" + +#include +#include +#include +#include + +int main(int argc, char** argv) +{ + // stdout is block-buffered when it is connected to journald instead of a + // terminal. Flush each newline-terminated diagnostic promptly. + std::setvbuf(stdout, nullptr, _IOLBF, 0); + + signal(SIGINT, argus_sender::handleSignal); + signal(SIGTERM, argus_sender::handleSignal); + signal(SIGPIPE, SIG_IGN); + + argus_sender::CommandLine commandLine; + if (!argus_sender::parseCommandLine(argc, argv, &commandLine)) + { + argus_sender::printUsage(argv[0]); + return EXIT_FAILURE; + } + argus_sender::setVerboseLogging(commandLine.verbose); + if (commandLine.listModes) + return argus_sender::printCameraModes() ? EXIT_SUCCESS : EXIT_FAILURE; + + argus_sender::AppConfig config; + std::string error; + if (!argus_sender::loadConfigFile(commandLine.configPath, &config, &error)) + { + std::fprintf(stderr, "Configuration error: %s\n", error.c_str()); + return EXIT_FAILURE; + } + if (commandLine.checkConfig) + { + std::printf("argus_sender: config ok; streams=%zu, bind=%s:%u\n", config.streams.size(), + config.network.bindHost.c_str(), config.network.port); + return EXIT_SUCCESS; + } + return argus_sender::executeConfiguredSender(config) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/examples/camera_streamer/tests/cpp/alignment_policy/CMakeLists.txt b/examples/camera_streamer/tests/cpp/alignment_policy/CMakeLists.txt new file mode 100644 index 0000000000..4948a438b2 --- /dev/null +++ b/examples/camera_streamer/tests/cpp/alignment_policy/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +add_executable(argus_sender_test_alignment_policy + ${CAMERA_STREAMER_TEST_ROOT}/cpp/test_alignment_policy.cpp + ${ARGUS_SENDER_ROOT}/src/StereoAlignmentPolicy.cpp + ${ARGUS_SENDER_ROOT}/src/StereoTiming.cpp +) +argus_sender_add_cpp_test(argus_sender_test_alignment_policy alignment_policy) diff --git a/examples/camera_streamer/tests/cpp/bundle_assembler/CMakeLists.txt b/examples/camera_streamer/tests/cpp/bundle_assembler/CMakeLists.txt new file mode 100644 index 0000000000..460075ec9d --- /dev/null +++ b/examples/camera_streamer/tests/cpp/bundle_assembler/CMakeLists.txt @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +add_executable(argus_sender_test_bundle_assembler + ${CAMERA_STREAMER_TEST_ROOT}/cpp/test_bundle_assembler.cpp + ${ARGUS_SENDER_ROOT}/src/FrameBundleAssembler.cpp + ${ARGUS_SENDER_ROOT}/src/Avs1Wire.cpp + ${ARGUS_SENDER_ROOT}/src/StereoTiming.cpp + ${ARGUS_SENDER_ROOT}/src/Runtime.cpp +) +argus_sender_add_cpp_test(argus_sender_test_bundle_assembler bundle_assembler) +target_compile_definitions(argus_sender_test_bundle_assembler PRIVATE ARGUS_SENDER_TEST_SUBMISSION_HOOK) diff --git a/examples/camera_streamer/tests/cpp/nv_remap/CMakeLists.txt b/examples/camera_streamer/tests/cpp/nv_remap/CMakeLists.txt new file mode 100644 index 0000000000..7eadff47fe --- /dev/null +++ b/examples/camera_streamer/tests/cpp/nv_remap/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +add_executable(argus_sender_test_nv_remap + ${CAMERA_STREAMER_TEST_ROOT}/cpp/test_nv_remap.cpp + ${ARGUS_SENDER_ROOT}/src/NvRemap.cpp +) +argus_sender_add_cpp_test(argus_sender_test_nv_remap nv_remap) diff --git a/examples/camera_streamer/tests/cpp/rate_limiter/CMakeLists.txt b/examples/camera_streamer/tests/cpp/rate_limiter/CMakeLists.txt new file mode 100644 index 0000000000..8cbbe0c519 --- /dev/null +++ b/examples/camera_streamer/tests/cpp/rate_limiter/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +add_executable(argus_sender_test_rate_limiter + ${CAMERA_STREAMER_TEST_ROOT}/cpp/test_rate_limiter.cpp + ${ARGUS_SENDER_ROOT}/src/StereoTiming.cpp + ${ARGUS_SENDER_ROOT}/src/MixedRateScheduler.cpp +) +argus_sender_add_cpp_test(argus_sender_test_rate_limiter rate_limiter) diff --git a/examples/camera_streamer/tests/cpp/tcp_transport/CMakeLists.txt b/examples/camera_streamer/tests/cpp/tcp_transport/CMakeLists.txt new file mode 100644 index 0000000000..c01f25f4a4 --- /dev/null +++ b/examples/camera_streamer/tests/cpp/tcp_transport/CMakeLists.txt @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +add_executable(argus_sender_test_tcp_transport + ${CAMERA_STREAMER_TEST_ROOT}/cpp/test_tcp_transport.cpp + ${ARGUS_SENDER_ROOT}/src/TcpTransport.cpp + ${ARGUS_SENDER_ROOT}/src/StereoTiming.cpp + ${ARGUS_SENDER_ROOT}/src/Runtime.cpp +) +argus_sender_add_cpp_test(argus_sender_test_tcp_transport tcp_transport) +set_tests_properties(tcp_transport PROPERTIES LABELS "loopback") diff --git a/examples/camera_streamer/tests/cpp/test_alignment_policy.cpp b/examples/camera_streamer/tests/cpp/test_alignment_policy.cpp new file mode 100644 index 0000000000..8dd7c361cc --- /dev/null +++ b/examples/camera_streamer/tests/cpp/test_alignment_policy.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "StereoAlignmentPolicy.h" +#include "TestHarness.h" + +int main() +{ + TestContext test("alignment_policy"); + const uint64_t framePeriodNs = 16666667ULL; + const uint64_t halfFrameNs = framePeriodNs / 2; + + argus_sender::FrameTiming left; + argus_sender::FrameTiming right; + left.hasSensorSofTimestampTsc = true; + right.hasSensorSofTimestampTsc = true; + left.sensorSofTimestampTscNs = 1000000000ULL; + + int64_t deltaNs = 0; + right.sensorSofTimestampTscNs = left.sensorSofTimestampTscNs + halfFrameNs; + TEST_CHECK_EQ(test, argus_sender::chooseStereoAlignmentAction(left, right, framePeriodNs, 0, &deltaNs), + argus_sender::STEREO_ALIGNMENT_KEEP); + TEST_CHECK_EQ(test, deltaNs, static_cast(halfFrameNs)); + + right.sensorSofTimestampTscNs = left.sensorSofTimestampTscNs + halfFrameNs + 1; + TEST_CHECK_EQ(test, argus_sender::chooseStereoAlignmentAction(left, right, framePeriodNs, 0), + argus_sender::STEREO_ALIGNMENT_DROP_LEFT); + + right.sensorSofTimestampTscNs = left.sensorSofTimestampTscNs - halfFrameNs - 1; + TEST_CHECK_EQ(test, argus_sender::chooseStereoAlignmentAction(left, right, framePeriodNs, 0), + argus_sender::STEREO_ALIGNMENT_DROP_RIGHT); + + TEST_CHECK_EQ( + test, + argus_sender::chooseStereoAlignmentAction(left, right, framePeriodNs, argus_sender::kStereoMaxAlignmentDrops), + argus_sender::STEREO_ALIGNMENT_KEEP); + + TEST_CHECK_EQ( + test, argus_sender::chooseStereoAlignmentAction(left, right, 0, 0), argus_sender::STEREO_ALIGNMENT_KEEP); + + left.hasSensorSofTimestampTsc = false; + right.hasSensorSofTimestampTsc = false; + left.hasSensorTimestamp = true; + right.hasSensorTimestamp = true; + left.sensorTimestampNs = 2000000000ULL; + right.sensorTimestampNs = left.sensorTimestampNs + halfFrameNs + 1; + TEST_CHECK_EQ(test, argus_sender::chooseStereoAlignmentAction(left, right, framePeriodNs, 0), + argus_sender::STEREO_ALIGNMENT_DROP_LEFT); + + left.hasSensorSofTimestampTsc = true; + right.hasSensorSofTimestampTsc = true; + left.sensorSofTimestampTscNs = 3000000000ULL; + right.sensorSofTimestampTscNs = left.sensorSofTimestampTscNs + 1; + TEST_CHECK_EQ(test, argus_sender::chooseStereoAlignmentAction(left, right, framePeriodNs, 0), + argus_sender::STEREO_ALIGNMENT_KEEP); + + left.hasSensorSofTimestampTsc = false; + right.hasSensorSofTimestampTsc = false; + left.hasSensorTimestamp = false; + right.hasSensorTimestamp = false; + TEST_CHECK(test, !argus_sender::stereoAlignmentDeltaNs(left, right, &deltaNs)); + TEST_CHECK_EQ(test, argus_sender::chooseStereoAlignmentAction(left, right, framePeriodNs, 0), + argus_sender::STEREO_ALIGNMENT_KEEP); + + return test.finish(); +} diff --git a/examples/camera_streamer/tests/cpp/test_bundle_assembler.cpp b/examples/camera_streamer/tests/cpp/test_bundle_assembler.cpp new file mode 100644 index 0000000000..db66886a3c --- /dev/null +++ b/examples/camera_streamer/tests/cpp/test_bundle_assembler.cpp @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "EncodedFrameSink.h" +#include "FrameBundleAssembler.h" +#include "PacketSink.h" +#include "TestHarness.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +std::mutex submissionMutex; +std::condition_variable submissionChanged; +bool observeSecondSubmission = false; +bool secondEntered = false; +bool secondContended = false; + +class FakePacketSink : public argus_sender::PacketSink +{ +public: + bool enqueueFramed(std::vector&& packet) override + { + packets.push_back(std::move(packet)); + return true; + } + + bool enqueueFramedWithTiming(std::vector&& packet, const argus_sender::PacketTiming& timing) override + { + timings.push_back(timing); + packets.push_back(std::move(packet)); + return true; + } + + bool hasError() const override + { + return false; + } + + std::vector> packets; + std::vector timings; +}; + +class BlockingPacketSink : public FakePacketSink +{ +public: + bool enqueueFramedWithTiming(std::vector&& packet, const argus_sender::PacketTiming& timing) override + { + std::unique_lock lock(mutex); + if (!entered) + { + entered = true; + changed.notify_all(); + changed.wait(lock, [this] { return released; }); + } + return FakePacketSink::enqueueFramedWithTiming(std::move(packet), timing); + } + std::mutex mutex; + std::condition_variable changed; + bool entered = false; + bool released = false; +}; + +std::vector templates() +{ + std::vector result(2); + result[0].flags = argus_sender::AVS1_STREAM_STEREO; + result[0].codec = argus_sender::AVS1_CODEC_HEVC; + result[0].width = 1280; + result[0].height = 720; + result[0].fpsNumerator = 60; + result[0].samples.resize(2); + result[0].samples[0].cameraIndex = 0; + result[0].samples[0].role = argus_sender::AVS1_ROLE_LEFT; + result[0].samples[1].cameraIndex = 1; + result[0].samples[1].role = argus_sender::AVS1_ROLE_RIGHT; + + result[1].codec = argus_sender::AVS1_CODEC_RAW; + result[1].width = 4; + result[1].height = 2; + result[1].fpsNumerator = 30; + result[1].samples.resize(1); + result[1].samples[0].cameraIndex = 0; + result[1].samples[0].role = argus_sender::AVS1_ROLE_MONO; + return result; +} + +argus_sender::FrameTiming timing(uint64_t frame, uint64_t tsc) +{ + argus_sender::FrameTiming result; + result.frameNumber = frame; + result.frameTimeNs = tsc + 100; + result.sensorSofTimestampTscNs = tsc; + result.hasSensorSofTimestampTsc = true; + return result; +} + +} // namespace + +namespace argus_sender +{ + +void observeSubmissionLock(std::mutex& mutex, uint64_t bundleSequence) +{ + std::lock_guard lock(submissionMutex); + if (!observeSecondSubmission || bundleSequence != 2) + return; + // Observe contention inside submitOwned, while the first enqueue is still blocked. + secondContended = !mutex.try_lock(); + if (!secondContended) + mutex.unlock(); + secondEntered = true; + submissionChanged.notify_all(); +} + +} // namespace argus_sender + +int main() +{ + TestContext test("bundle_assembler"); + FakePacketSink sink; + argus_sender::FrameBundleAssembler assembler(&sink, 77, templates()); + + std::vector bundle1(2); + bundle1[0].present = true; + bundle1[0].sequence = 1; + bundle1[0].timings.push_back(timing(10, 1000)); + bundle1[0].timings.push_back(timing(11, 1004)); + const uint64_t captureStartSteadyNs = static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()) - + 2000000ULL; + bundle1[0].timings[0].acquiredSteadyNs = captureStartSteadyNs; + bundle1[0].timings[1].acquiredSteadyNs = captureStartSteadyNs + 1000000ULL; + TEST_CHECK(test, assembler.beginBundle(1, bundle1, 2000, 2100, argus_sender::SENDER_CLOCK_REALTIME, 1300)); + TEST_CHECK(test, assembler.recordCopyDone(0, 1, captureStartSteadyNs + 1400000ULL)); + TEST_CHECK(test, assembler.recordProcessStart(0, 1, captureStartSteadyNs + 1500000ULL)); + TEST_CHECK(test, assembler.recordProcessStart(0, 1, captureStartSteadyNs + 1700000ULL)); + + std::vector bundle2(2); + bundle2[1].present = true; + bundle2[1].sequence = 1; + bundle2[1].timings.push_back(timing(20, 3000)); + const uint64_t rawCaptureStartSteadyNs = captureStartSteadyNs + 500000ULL; + bundle2[1].timings[0].acquiredSteadyNs = rawCaptureStartSteadyNs; + TEST_CHECK(test, assembler.beginBundle(2, bundle2, 4000, 4100, argus_sender::SENDER_CLOCK_REALTIME, 0)); + TEST_CHECK(test, assembler.recordCopyDone(1, 2, rawCaptureStartSteadyNs + 200000ULL)); + TEST_CHECK(test, assembler.recordProcessStart(1, 2, rawCaptureStartSteadyNs + 300000ULL)); + TEST_CHECK( + test, assembler.submitRaw(1, 0, 2, std::vector(12, 0x80), 4200, argus_sender::SENDER_CLOCK_REALTIME)); + TEST_CHECK(test, sink.packets.empty()); + + const uint8_t right[] = { 0, 0, 1, 0x26 }; + TEST_CHECK(test, assembler.encodedSink(0, 1)->enqueueEncoded( + right, sizeof(right), 1, 2200, argus_sender::SENDER_CLOCK_REALTIME)); + TEST_CHECK(test, sink.packets.empty()); + const uint8_t left[] = { 0, 0, 1, 0x20 }; + TEST_CHECK(test, assembler.encodedSink(0, 0)->enqueueEncoded( + left, sizeof(left), 1, 2201, argus_sender::SENDER_CLOCK_REALTIME)); + TEST_CHECK_EQ(test, sink.packets.size(), static_cast(2)); + TEST_CHECK_EQ(test, sink.timings.size(), static_cast(2)); + TEST_CHECK_EQ(test, sink.timings[0].captureStartSteadyNs, captureStartSteadyNs); + TEST_CHECK_EQ(test, sink.timings[0].acquireSpreadNs, 1000000ULL); + TEST_CHECK_EQ(test, sink.timings[0].captureToEncodeStartNs, 1500000ULL); + TEST_CHECK(test, sink.timings[0].captureToProcessDoneNs != 0); + TEST_CHECK_EQ(test, sink.timings[0].streams.size(), static_cast(2)); + TEST_CHECK(test, sink.timings[0].streams[0].present); + TEST_CHECK(test, sink.timings[0].streams[0].encoded); + TEST_CHECK_EQ(test, sink.timings[0].samples.size(), static_cast(3)); + TEST_CHECK(test, sink.timings[0].samples[0].present); + TEST_CHECK_EQ(test, sink.timings[0].samples[0].sensorSofTscNs, 1000ULL); + TEST_CHECK(test, sink.timings[0].samples[1].present); + TEST_CHECK_EQ(test, sink.timings[0].samples[1].sensorSofTscNs, 1004ULL); + TEST_CHECK(test, !sink.timings[0].samples[2].present); + TEST_CHECK_EQ(test, sink.timings[0].streams[0].captureStartSteadyNs, captureStartSteadyNs); + TEST_CHECK_EQ(test, sink.timings[0].streams[0].acquireSpreadNs, 1000000ULL); + TEST_CHECK_EQ(test, sink.timings[0].streams[0].copyDoneSteadyNs, captureStartSteadyNs + 1400000ULL); + TEST_CHECK_EQ(test, sink.timings[0].streams[0].processStartSteadyNs, captureStartSteadyNs + 1500000ULL); + TEST_CHECK(test, sink.timings[0].streams[0].processDoneSteadyNs >= sink.timings[0].streams[0].processStartSteadyNs); + TEST_CHECK(test, sink.timings[1].streams[1].present); + TEST_CHECK(test, !sink.timings[1].streams[1].encoded); + TEST_CHECK_EQ(test, sink.timings[1].samples.size(), static_cast(3)); + TEST_CHECK(test, !sink.timings[1].samples[0].present); + TEST_CHECK(test, !sink.timings[1].samples[1].present); + TEST_CHECK(test, sink.timings[1].samples[2].present); + TEST_CHECK_EQ(test, sink.timings[1].samples[2].sensorSofTscNs, 3000ULL); + TEST_CHECK_EQ(test, sink.timings[1].streams[1].captureStartSteadyNs, rawCaptureStartSteadyNs); + TEST_CHECK_EQ(test, sink.timings[1].streams[1].copyDoneSteadyNs, rawCaptureStartSteadyNs + 200000ULL); + TEST_CHECK_EQ(test, sink.timings[1].streams[1].processStartSteadyNs, rawCaptureStartSteadyNs + 300000ULL); + + argus_sender::Avs1Bundle parsed; + const char* error = 0; + TEST_CHECK(test, argus_sender::parseAvs1WireRecord(sink.packets[0].data(), sink.packets[0].size(), &parsed, &error)); + TEST_CHECK_EQ(test, parsed.sequence, 1ULL); + TEST_CHECK(test, parsed.flags & argus_sender::AVS1_BUNDLE_SENDER_TSC_VALID); + TEST_CHECK_EQ(test, parsed.senderCaptureReadyTscNs, 1300ULL); + TEST_CHECK(test, parsed.streams[0].flags & argus_sender::AVS1_STREAM_PRESENT); + TEST_CHECK(test, !(parsed.streams[1].flags & argus_sender::AVS1_STREAM_PRESENT)); + + TEST_CHECK(test, argus_sender::parseAvs1WireRecord(sink.packets[1].data(), sink.packets[1].size(), &parsed, &error)); + TEST_CHECK_EQ(test, parsed.sequence, 2ULL); + TEST_CHECK(test, !(parsed.flags & argus_sender::AVS1_BUNDLE_SENDER_TSC_VALID)); + TEST_CHECK_EQ(test, parsed.senderCaptureReadyTscNs, 0ULL); + TEST_CHECK(test, parsed.streams[1].flags & argus_sender::AVS1_STREAM_PRESENT); + + // A TSC without a valid sender clock must not set SENDER_TSC_VALID. + std::vector bundle3(2); + bundle3[1].present = true; + bundle3[1].sequence = 2; + bundle3[1].timings.push_back(timing(30, 5000)); + bundle3[1].timings[0].acquiredSteadyNs = rawCaptureStartSteadyNs; + TEST_CHECK(test, assembler.beginBundle(3, bundle3, 6000, 6100, argus_sender::SENDER_CLOCK_UNKNOWN, 6300)); + TEST_CHECK( + test, assembler.submitRaw(1, 0, 3, std::vector(12, 0x80), 6200, argus_sender::SENDER_CLOCK_REALTIME)); + TEST_CHECK_EQ(test, sink.packets.size(), static_cast(3)); + TEST_CHECK(test, argus_sender::parseAvs1WireRecord(sink.packets[2].data(), sink.packets[2].size(), &parsed, &error)); + TEST_CHECK_EQ(test, parsed.sequence, 3ULL); + TEST_CHECK(test, !(parsed.flags & argus_sender::AVS1_BUNDLE_SENDER_CLOCK_VALID)); + TEST_CHECK(test, !(parsed.flags & argus_sender::AVS1_BUNDLE_SENDER_TSC_VALID)); + TEST_CHECK_EQ(test, parsed.senderCaptureReadyTscNs, 0ULL); + // Hold the first enqueue while another encoder callback completes the next bundle. + BlockingPacketSink blockedSink; + argus_sender::FrameBundleAssembler concurrent(&blockedSink, 88, templates()); + TEST_CHECK(test, concurrent.beginBundle(1, bundle3, 6000, 6100, argus_sender::SENDER_CLOCK_UNKNOWN, 0)); + TEST_CHECK(test, concurrent.beginBundle(2, bundle3, 6000, 6100, argus_sender::SENDER_CLOCK_UNKNOWN, 0)); + std::atomic firstOk(false), secondOk(false); + observeSecondSubmission = true; + std::thread first( + [&] + { + firstOk = + concurrent.submitRaw(1, 0, 1, std::vector(12, 0x80), 6200, argus_sender::SENDER_CLOCK_REALTIME); + }); + { + std::unique_lock lock(blockedSink.mutex); + TEST_CHECK( + test, blockedSink.changed.wait_for(lock, std::chrono::seconds(2), [&] { return blockedSink.entered; })); + } + std::thread second( + [&] + { + secondOk = + concurrent.submitRaw(1, 0, 2, std::vector(12, 0x80), 6200, argus_sender::SENDER_CLOCK_REALTIME); + }); + { + std::unique_lock lock(submissionMutex); + TEST_CHECK(test, submissionChanged.wait_for(lock, std::chrono::seconds(2), [] { return secondEntered; })); + TEST_CHECK(test, secondContended); + } + TEST_CHECK(test, !secondOk); + { + std::lock_guard lock(blockedSink.mutex); + blockedSink.released = true; + } + blockedSink.changed.notify_all(); + first.join(); + second.join(); + TEST_CHECK(test, firstOk && secondOk); + TEST_CHECK_EQ(test, blockedSink.packets.size(), static_cast(2)); + for (size_t i = 0; i < blockedSink.packets.size(); ++i) + { + TEST_CHECK(test, argus_sender::parseAvs1WireRecord( + blockedSink.packets[i].data(), blockedSink.packets[i].size(), &parsed, &error)); + TEST_CHECK_EQ(test, parsed.sequence, i + 1); + } + return test.finish(); +} diff --git a/examples/camera_streamer/tests/cpp/test_nv_remap.cpp b/examples/camera_streamer/tests/cpp/test_nv_remap.cpp new file mode 100644 index 0000000000..3a3dde9085 --- /dev/null +++ b/examples/camera_streamer/tests/cpp/test_nv_remap.cpp @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "NvRemap.h" +#include "TestHarness.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +std::string writeRmap(uint32_t destWidth, + uint32_t destHeight, + const std::vector& mapX, + const std::vector& mapY, + const char* magic = "RMAP") +{ + char path[] = "/tmp/argus_sender_rmap_XXXXXX"; + const int fd = mkstemp(path); + if (fd < 0) + return ""; + close(fd); + std::ofstream file(path, std::ios::binary); + const float balance = 0.0F; + file.write(magic, 4); + file.write(reinterpret_cast(&destWidth), 4); + file.write(reinterpret_cast(&destHeight), 4); + file.write(reinterpret_cast(&balance), 4); + file.write(reinterpret_cast(mapX.data()), static_cast(mapX.size() * sizeof(float))); + file.write(reinterpret_cast(mapY.data()), static_cast(mapY.size() * sizeof(float))); + return path; +} + +// Reference float bilinear sample of a tightly packed plane. +double bilinear(const std::vector& plane, uint32_t width, double x, double y) +{ + const uint32_t x0 = static_cast(x); + const uint32_t y0 = static_cast(y); + const double fx = x - x0; + const double fy = y - y0; + const size_t base = static_cast(y0) * width + x0; + return plane[base] * (1 - fx) * (1 - fy) + plane[base + 1] * fx * (1 - fy) + plane[base + width] * (1 - fx) * fy + + plane[base + width + 1] * fx * fy; +} + +} // namespace + +int main() +{ + TestContext test("nv_remap"); + + // Synthetic pitch-linear NV12 source: 16x8 luma gradient, flat chroma. + const uint32_t srcWidth = 16; + const uint32_t srcHeight = 8; + const size_t srcYPitch = 24; // wider than the image: pitch must be honored + const size_t srcUVPitch = 24; + std::vector srcY(srcYPitch * srcHeight, 0); + std::vector tightY(srcWidth * srcHeight, 0); + for (uint32_t y = 0; y < srcHeight; ++y) + { + for (uint32_t x = 0; x < srcWidth; ++x) + { + const uint8_t value = static_cast(10 * x + 3 * y); + srcY[y * srcYPitch + x] = value; + tightY[y * srcWidth + x] = value; + } + } + std::vector srcUV(srcUVPitch * srcHeight / 2, 0); + for (uint32_t y = 0; y < srcHeight / 2; ++y) + { + for (uint32_t x = 0; x < srcWidth / 2; ++x) + { + srcUV[y * srcUVPitch + 2 * x] = static_cast(60 + 4 * x); + srcUV[y * srcUVPitch + 2 * x + 1] = static_cast(200 - 4 * y); + } + } + + // --- supersample 1, identity map: output equals the source region. --- + { + const uint32_t outWidth = 8; + const uint32_t outHeight = 4; + std::vector mapX(outWidth * outHeight); + std::vector mapY(outWidth * outHeight); + for (uint32_t y = 0; y < outHeight; ++y) + { + for (uint32_t x = 0; x < outWidth; ++x) + { + mapX[y * outWidth + x] = static_cast(x); + mapY[y * outWidth + x] = static_cast(y); + } + } + const std::string path = writeRmap(outWidth, outHeight, mapX, mapY); + argus_sender::NvRemapTable table; + std::string error; + TEST_CHECK(test, argus_sender::loadNvRemapTable(path, srcWidth, srcHeight, outWidth, outHeight, &table, &error)); + TEST_CHECK_EQ(test, table.supersample, 1U); + std::vector out(outWidth * outHeight * 3 / 2, 0xAA); + argus_sender::nvRemapNv12(table, srcY.data(), srcYPitch, srcUV.data(), srcUVPitch, out.data()); + bool lumaExact = true; + for (uint32_t y = 0; y < outHeight; ++y) + for (uint32_t x = 0; x < outWidth; ++x) + lumaExact = lumaExact && out[y * outWidth + x] == tightY[y * srcWidth + x]; + TEST_CHECK(test, lumaExact); + // Chroma taps are sited at the luma cell center (a quarter UV texel + // off the top-left texel on an identity map), so allow interpolation. + const uint8_t* outUV = out.data() + outWidth * outHeight; + TEST_CHECK(test, std::fabs(static_cast(outUV[0]) - srcUV[0]) <= 2.0); + TEST_CHECK(test, std::fabs(static_cast(outUV[1]) - srcUV[1]) <= 2.0); + std::remove(path.c_str()); + } + + // --- supersample 2: 2x2 box of bilinear samples matches a float + // reference within rounding. --- + { + const uint32_t outWidth = 4; + const uint32_t outHeight = 2; + const uint32_t destWidth = 8; + const uint32_t destHeight = 4; + std::vector mapX(destWidth * destHeight); + std::vector mapY(destWidth * destHeight); + for (uint32_t y = 0; y < destHeight; ++y) + { + for (uint32_t x = 0; x < destWidth; ++x) + { + mapX[y * destWidth + x] = 0.75F + 1.4F * x; + mapY[y * destWidth + x] = 0.5F + 1.2F * y; + } + } + const std::string path = writeRmap(destWidth, destHeight, mapX, mapY); + argus_sender::NvRemapTable table; + std::string error; + TEST_CHECK(test, argus_sender::loadNvRemapTable(path, srcWidth, srcHeight, outWidth, outHeight, &table, &error)); + TEST_CHECK_EQ(test, table.supersample, 2U); + std::vector out(outWidth * outHeight * 3 / 2, 0); + argus_sender::nvRemapNv12(table, srcY.data(), srcYPitch, srcUV.data(), srcUVPitch, out.data()); + double maxError = 0.0; + for (uint32_t y = 0; y < outHeight; ++y) + { + for (uint32_t x = 0; x < outWidth; ++x) + { + double sum = 0.0; + for (uint32_t dy = 0; dy < 2; ++dy) + for (uint32_t dx = 0; dx < 2; ++dx) + { + const size_t i = static_cast(2 * y + dy) * destWidth + 2 * x + dx; + sum += bilinear(tightY, srcWidth, mapX[i], mapY[i]); + } + maxError = std::max(maxError, std::fabs(sum / 4.0 - out[y * outWidth + x])); + } + } + // Fixed-point taps (1/64 weights) plus two rounding stages. + TEST_CHECK(test, maxError <= 1.5); + std::remove(path.c_str()); + } + + // --- out-of-source map values become black luma / neutral chroma. --- + { + const uint32_t outWidth = 4; + const uint32_t outHeight = 2; + std::vector mapX(outWidth * outHeight, -5.0F); + std::vector mapY(outWidth * outHeight, -5.0F); + const std::string path = writeRmap(outWidth, outHeight, mapX, mapY); + argus_sender::NvRemapTable table; + std::string error; + // 100% out-of-source is rejected as a wrong-source map... + TEST_CHECK(test, !argus_sender::loadNvRemapTable(path, srcWidth, srcHeight, outWidth, outHeight, &table, &error)); + TEST_CHECK(test, error.find("different source") != std::string::npos); + std::remove(path.c_str()); + + // ...but a single stray pixel passes and renders black/neutral. + for (size_t i = 1; i < mapX.size(); ++i) + { + mapX[i] = 1.0F; + mapY[i] = 1.0F; + } + const std::string path2 = writeRmap(outWidth, outHeight, mapX, mapY); + TEST_CHECK( + test, !argus_sender::loadNvRemapTable(path2, srcWidth, srcHeight, outWidth, outHeight, &table, &error)); + std::remove(path2.c_str()); + } + + // --- header validation. --- + { + std::vector mapX(6, 1.0F); + std::vector mapY(6, 1.0F); + argus_sender::NvRemapTable table; + std::string error; + + const std::string badMagic = writeRmap(3, 2, mapX, mapY, "RMPX"); + TEST_CHECK(test, !argus_sender::loadNvRemapTable(badMagic, srcWidth, srcHeight, 4, 2, &table, &error)); + TEST_CHECK(test, error.find("not an RMAP") != std::string::npos); + std::remove(badMagic.c_str()); + + // dest 6x2 is not out 4x2 times a shared factor of 1 or 2. + std::vector mapX62(12, 1.0F); + std::vector mapY62(12, 1.0F); + const std::string badFactor = writeRmap(6, 2, mapX62, mapY62); + TEST_CHECK(test, !argus_sender::loadNvRemapTable(badFactor, srcWidth, srcHeight, 4, 2, &table, &error)); + TEST_CHECK(test, error.find("supersample") != std::string::npos); + std::remove(badFactor.c_str()); + } + + return test.finish(); +} diff --git a/examples/camera_streamer/tests/cpp/test_rate_limiter.cpp b/examples/camera_streamer/tests/cpp/test_rate_limiter.cpp new file mode 100644 index 0000000000..efb3d76eab --- /dev/null +++ b/examples/camera_streamer/tests/cpp/test_rate_limiter.cpp @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "MixedRateScheduler.h" +#include "StereoTiming.h" +#include "TestHarness.h" + +#include +#include + +namespace +{ + +uint64_t countOutputFrames(uint32_t targetFps, uint64_t inputFrames, uint64_t inputPeriodNs) +{ + argus_sender::StereoOutputRateLimiter limiter(targetFps); + uint64_t outputFrames = 0; + for (uint64_t i = 0; i < inputFrames; ++i) + { + argus_sender::FrameTiming timing; + timing.hasSensorSofTimestampTsc = true; + timing.sensorSofTimestampTscNs = 1000000000ULL + i * inputPeriodNs; + if (limiter.shouldOutput(timing)) + ++outputFrames; + } + return outputFrames; +} + +} // namespace + +int main() +{ + TestContext test("rate_limiter"); + const uint64_t period60FpsNs = 16666667ULL; + + TEST_CHECK_EQ(test, countOutputFrames(0, 600, period60FpsNs), 600ULL); + TEST_CHECK_EQ(test, countOutputFrames(60, 600, period60FpsNs), 600ULL); + TEST_CHECK_EQ(test, countOutputFrames(30, 600, period60FpsNs), 300ULL); + TEST_CHECK_EQ(test, countOutputFrames(45, 600, period60FpsNs), 450ULL); + TEST_CHECK_EQ(test, countOutputFrames(45, 36000, period60FpsNs), 27000ULL); + + argus_sender::StereoOutputRateLimiter limiter(30); + { + argus_sender::StereoOutputRateLimiter jittered60Fps(60); + uint64_t outputFrames = 0; + for (uint64_t i = 0; i < 600; ++i) + { + argus_sender::FrameTiming jittered; + jittered.hasSensorSofTimestampTsc = true; + const int64_t jitterNs = i % 2 == 0 ? 500000 : -500000; + jittered.sensorSofTimestampTscNs = + static_cast(1000000000LL + static_cast(i * period60FpsNs) + jitterNs); + if (jittered60Fps.shouldOutput(jittered)) + ++outputFrames; + } + TEST_CHECK_EQ(test, outputFrames, 600ULL); + } + + { + argus_sender::StereoOutputRateLimiter baseLimiter(45); + argus_sender::MixedRateScheduler scheduler({ 45, 30 }); + uint64_t bundles = 0; + uint64_t lowerRateFrames = 0; + for (uint64_t i = 0; i < 600; ++i) + { + argus_sender::FrameTiming baseTiming; + baseTiming.hasSensorSofTimestampTsc = true; + baseTiming.sensorSofTimestampTscNs = 1000000000ULL + i * period60FpsNs; + if (!baseLimiter.shouldOutput(baseTiming)) + continue; + const std::vector due = scheduler.next(); + ++bundles; + lowerRateFrames += due[1] ? 1 : 0; + } + TEST_CHECK_EQ(test, bundles, 450ULL); + TEST_CHECK_EQ(test, lowerRateFrames, 300ULL); + } + + argus_sender::FrameTiming timing; + timing.hasSensorSofTimestampTsc = true; + timing.sensorSofTimestampTscNs = 1000000000ULL; + TEST_CHECK(test, limiter.shouldOutput(timing)); + timing.sensorSofTimestampTscNs += period60FpsNs; + TEST_CHECK(test, !limiter.shouldOutput(timing)); + + timing.sensorSofTimestampTscNs = 500000000ULL; + TEST_CHECK(test, limiter.shouldOutput(timing)); + + timing.hasSensorSofTimestampTsc = false; + timing.hasSensorTimestamp = true; + timing.sensorTimestampNs = 2000000000ULL; + TEST_CHECK(test, limiter.shouldOutput(timing)); + + TEST_CHECK_EQ(test, argus_sender::signedDeltaNs(20, 10), 10LL); + TEST_CHECK_EQ(test, argus_sender::signedDeltaNs(10, 20), -10LL); + + return test.finish(); +} diff --git a/examples/camera_streamer/tests/cpp/test_tcp_transport.cpp b/examples/camera_streamer/tests/cpp/test_tcp_transport.cpp new file mode 100644 index 0000000000..7cf310f8a5 --- /dev/null +++ b/examples/camera_streamer/tests/cpp/test_tcp_transport.cpp @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "TcpTransport.h" +#include "TestHarness.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +std::vector readBytes(int fd, size_t size) +{ + std::vector data(size); + size_t received = 0; + while (received < size) + { + const ssize_t count = recv(fd, data.data() + received, size - received, 0); + if (count <= 0) + { + data.resize(received); + break; + } + received += static_cast(count); + } + return data; +} + +std::vector readFile(const std::string& path) +{ + std::ifstream stream(path.c_str(), std::ios::binary); + return std::vector(std::istreambuf_iterator(stream), std::istreambuf_iterator()); +} + +} // namespace + +int main() +{ + TestContext test("tcp_transport"); + const std::vector packet = { 0x00, 0x00, 0x00, 0x05, 0x48, 0x56, 0x53, 0x32, 0x03 }; + + { + int sockets[2] = { -1, -1 }; + TEST_CHECK_EQ(test, socketpair(AF_UNIX, SOCK_STREAM, 0, sockets), 0); + argus_sender::TcpPacketSenderSettings settings; + settings.kpi = false; + int senderFd = sockets[0]; + argus_sender::TcpPacketSender sender(&senderFd, settings, 1, settings.port, "test"); + TEST_CHECK(test, sender.start()); + std::vector queued = packet; + TEST_CHECK(test, sender.enqueueFramed(std::move(queued))); + sender.shutdown(true); + TEST_CHECK(test, readBytes(sockets[1], packet.size()) == packet); + close(sockets[0]); + close(sockets[1]); + } + + { + char path[] = "/tmp/argus_sender_tcp_test_XXXXXX"; + const int temporaryFd = mkstemp(path); + TEST_CHECK(test, temporaryFd >= 0); + if (temporaryFd >= 0) + close(temporaryFd); + + int sockets[2] = { -1, -1 }; + TEST_CHECK_EQ(test, socketpair(AF_UNIX, SOCK_STREAM, 0, sockets), 0); + argus_sender::TcpPacketSenderSettings settings; + settings.kpi = false; + settings.dumpWirePath = path; + int senderFd = sockets[0]; + argus_sender::TcpPacketSender sender(&senderFd, settings, 1, settings.port, "dump-test"); + TEST_CHECK(test, sender.start()); + std::vector queued = packet; + TEST_CHECK(test, sender.enqueueFramed(std::move(queued))); + sender.shutdown(true); + TEST_CHECK(test, readBytes(sockets[1], packet.size()) == packet); + TEST_CHECK(test, readFile(path) == packet); + unlink(path); + close(sockets[0]); + close(sockets[1]); + } + + { + int sockets[2] = { -1, -1 }; + TEST_CHECK_EQ(test, socketpair(AF_UNIX, SOCK_STREAM, 0, sockets), 0); + argus_sender::TcpPacketSenderSettings settings; + settings.kpi = false; + int senderFd = sockets[0]; + argus_sender::TcpPacketSender sender(&senderFd, settings, 1, settings.port, "invalid-test"); + TEST_CHECK(test, sender.start()); + std::vector malformed = { 0x00, 0x00, 0x00, 0x02, 0x01, 0x02, 0x03 }; + TEST_CHECK(test, !sender.enqueueFramed(std::move(malformed))); + TEST_CHECK(test, sender.hasError()); + sender.shutdown(false); + close(sockets[0]); + close(sockets[1]); + } + + { + int sockets[2] = { -1, -1 }; + TEST_CHECK_EQ(test, socketpair(AF_UNIX, SOCK_STREAM, 0, sockets), 0); + argus_sender::TcpPacketSenderSettings settings; + settings.kpi = false; + int senderFd = sockets[0]; + argus_sender::TcpPacketSender sender(&senderFd, settings, 1, settings.port, "disconnect-test"); + TEST_CHECK(test, sender.start()); + close(sockets[1]); + sockets[1] = -1; + std::vector queued = packet; + TEST_CHECK(test, sender.enqueueFramed(std::move(queued))); + for (uint32_t attempt = 0; attempt < 100 && !sender.disconnected(); ++attempt) + { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + TEST_CHECK(test, sender.disconnected()); + TEST_CHECK(test, sender.hasError()); + sender.shutdown(false); + close(sockets[0]); + } + + { + int sockets[2] = { -1, -1 }; + TEST_CHECK_EQ(test, socketpair(AF_UNIX, SOCK_STREAM, 0, sockets), 0); + int sendBufferBytes = 4096; + TEST_CHECK_EQ(test, setsockopt(sockets[0], SOL_SOCKET, SO_SNDBUF, &sendBufferBytes, sizeof(sendBufferBytes)), 0); + argus_sender::TcpPacketSenderSettings settings; + settings.kpi = false; + settings.sendPollTimeoutMs = 20; + settings.drainTimeoutMs = 100; + int senderFd = sockets[0]; + argus_sender::TcpPacketSender sender(&senderFd, settings, 1, settings.port, "stalled-drain-test"); + TEST_CHECK(test, sender.start()); + const size_t payloadBytes = 8U * 1024U * 1024U; + std::vector queued(payloadBytes + sizeof(uint32_t), 0x5a); + const uint32_t framedSize = htonl(static_cast(payloadBytes)); + std::memcpy(queued.data(), &framedSize, sizeof(framedSize)); + TEST_CHECK(test, sender.enqueueFramed(std::move(queued))); + const std::chrono::steady_clock::time_point shutdownStart = std::chrono::steady_clock::now(); + sender.shutdown(true); + const double shutdownSeconds = + std::chrono::duration(std::chrono::steady_clock::now() - shutdownStart).count(); + TEST_CHECK(test, shutdownSeconds < 1.0); + TEST_CHECK(test, sender.hasError()); + close(sockets[0]); + close(sockets[1]); + } + + return test.finish(); +} diff --git a/examples/camera_streamer/tests/hardware/avs1_smoke.py b/examples/camera_streamer/tests/hardware/avs1_smoke.py new file mode 100755 index 0000000000..9af9c8acc0 --- /dev/null +++ b/examples/camera_streamer/tests/hardware/avs1_smoke.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded AVS1 stereo hardware smoke test.""" + +from __future__ import annotations + +import argparse +import socket +import struct +import subprocess +import sys +import time +from pathlib import Path + + +HEADER = struct.Struct("!4sHHIHHQQQQIIQ") +STREAM = struct.Struct("!HHBBHIIIIQHHI") +SAMPLE = struct.Struct("!HBBHHIIQQQQQQII") + + +def recv_exact(sock: socket.socket, count: int) -> bytes: + data = bytearray(count) + view = memoryview(data) + received = 0 + while received < count: + size = sock.recv_into(view[received:]) + if size == 0: + raise EOFError("sender closed before the requested AVS1 bundles arrived") + received += size + return bytes(data) + + +def nal_types(data: bytes) -> list[int]: + result: list[int] = [] + index = 0 + while index + 4 < len(data): + if data[index : index + 3] == b"\x00\x00\x01": + header = index + 3 + index = header + 2 + elif data[index : index + 4] == b"\x00\x00\x00\x01": + header = index + 4 + index = header + 2 + else: + index += 1 + continue + if header < len(data): + result.append((data[header] >> 1) & 0x3F) + return result + + +def parse_first_stream(packet: bytes): + if len(packet) < HEADER.size: + raise ValueError("short AVS1 packet") + fields = HEADER.unpack_from(packet) + if fields[0] != b"AVS1" or fields[1] != 1 or fields[3] != HEADER.size: + raise ValueError("bad AVS1 header") + stream_count, sample_count = fields[4], fields[5] + descriptor_bytes, payload_bytes = fields[10], fields[11] + if stream_count < 1 or sample_count < 2: + raise ValueError("smoke packet has no stereo stream") + stream = STREAM.unpack_from(packet, HEADER.size) + if stream[0] != 0 or not stream[1] & 1 or not stream[1] & 2 or stream[2] != 1: + raise ValueError("stream zero is not present stereo HEVC") + first_sample, count = stream[10], stream[11] + if first_sample != 0 or count != 2: + raise ValueError("stream zero does not contain two ordered samples") + sample_base = HEADER.size + stream_count * STREAM.size + samples = [ + SAMPLE.unpack_from(packet, sample_base + index * SAMPLE.size) + for index in range(2) + ] + payload_start = HEADER.size + descriptor_bytes + payloads = [] + tsc = [] + for camera, sample in enumerate(samples): + if sample[0] != 0 or sample[1] != camera or not sample[3] & 1: + raise ValueError("invalid stereo sample descriptor") + offset, size = sample[13], sample[14] + if offset + size > payload_bytes or size == 0: + raise ValueError("invalid stereo sample payload range") + payloads.append(packet[payload_start + offset : payload_start + offset + size]) + tsc.append(sample[8] if sample[3] & 2 else None) + return fields[6], payloads, tsc + + +def connect_with_retry( + port: int, process: subprocess.Popen, timeout: float +) -> socket.socket: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError( + "argus_sender exited before accepting the smoke receiver" + ) + try: + sock = socket.create_connection(("127.0.0.1", port), timeout=0.5) + sock.settimeout(10.0) + return sock + except OSError: + time.sleep(0.05) + raise TimeoutError("timed out connecting to argus_sender") + + +def configured_port(config_path: str) -> int: + import yaml + + config = yaml.safe_load(Path(config_path).read_text()) + port = config.get("network", {}).get("port", 5000) + if isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535: + raise ValueError("network.port must be an integer in 1..65535") + return port + + +def validate_frame_limit(config_path: str, frames: int) -> None: + import yaml + + config = yaml.safe_load(Path(config_path).read_text()) + limit = config.get("runtime", {}).get("frames", 0) + # The sender must exit after the last bundle rather than wait for a reconnect. + if ( + isinstance(limit, bool) + or not isinstance(limit, int) + or limit <= 0 + or frames <= 0 + or limit != frames + ): + raise ValueError("runtime.frames must be positive and equal to --frames") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--sender", required=True) + parser.add_argument("--config", required=True) + parser.add_argument("--frames", type=int, default=120) + args = parser.parse_args() + try: + port = configured_port(args.config) + validate_frame_limit(args.config, args.frames) + except ValueError as exc: + parser.error(str(exc)) + + process = subprocess.Popen( + [args.sender, "--config", args.config], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + output = "" + try: + sock = connect_with_retry(port, process, 15.0) + first_nals: list[list[int]] | None = None + last_tsc: list[int | None] = [None, None] + max_delta_us = 0.0 + with sock: + for expected_sequence in range(1, args.frames + 1): + (length,) = struct.unpack("!I", recv_exact(sock, 4)) + if length <= 0 or length > 256 * 1024 * 1024: + raise ValueError(f"invalid AVS1 record length {length}") + sequence, payloads, tsc = parse_first_stream(recv_exact(sock, length)) + if sequence != expected_sequence: + raise ValueError( + f"AVS1 sequence {sequence}, expected {expected_sequence}" + ) + if first_nals is None: + first_nals = [nal_types(payload) for payload in payloads] + if all(value is not None for value in tsc): + max_delta_us = max( + max_delta_us, abs(int(tsc[1]) - int(tsc[0])) / 1000.0 + ) + for eye in range(2): + if ( + tsc[eye] is not None + and last_tsc[eye] is not None + and int(tsc[eye]) <= int(last_tsc[eye]) + ): + raise ValueError(f"eye {eye} TSC did not increase") + last_tsc[eye] = tsc[eye] + output, _ = process.communicate(timeout=20.0) + if process.returncode != 0: + raise RuntimeError(f"argus_sender exited {process.returncode}\n{output}") + assert first_nals is not None + for eye, types in enumerate(first_nals): + if not {32, 33, 34}.issubset(types) or not any( + 16 <= value <= 23 for value in types + ): + raise ValueError(f"eye {eye} first AU lacks VPS/SPS/PPS/IRAP: {types}") + print( + f"PASS: AVS1 stereo hardware smoke received {args.frames} bundles; " + f"max_abs_tsc_delta_us={max_delta_us:.3f}; first_nals={first_nals}" + ) + return 0 + except Exception as exc: + if process.poll() is None: + process.terminate() + try: + remaining, _ = process.communicate(timeout=5.0) + output += remaining + except subprocess.TimeoutExpired: + process.kill() + remaining, _ = process.communicate() + output += remaining + print(f"FAIL: {exc}", file=sys.stderr) + if output: + print("--- argus_sender output ---", file=sys.stderr) + print(output, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/camera_streamer/tests/python/test_smoke_config.py b/examples/camera_streamer/tests/python/test_smoke_config.py new file mode 100644 index 0000000000..743cd03784 --- /dev/null +++ b/examples/camera_streamer/tests/python/test_smoke_config.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "hardware")) +from avs1_smoke import configured_port, validate_frame_limit # noqa: E402 + + +class SmokeConfigTests(unittest.TestCase): + def test_configured_port_and_sender_default(self): + with tempfile.TemporaryDirectory() as root: + path = Path(root) / "camera.yaml" + for text, expected in [("network: {port: 54321}", 54321), ("{}", 5000)]: + path.write_text(text) + self.assertEqual(configured_port(str(path)), expected) + for value in [0, 65536, True, "not-a-port"]: + path.write_text(f"network: {{port: {value}}}") + with self.assertRaises(ValueError): + configured_port(str(path)) + + def test_frame_limit_matches_requested_bundles(self): + with tempfile.TemporaryDirectory() as root: + path = Path(root) / "camera.yaml" + path.write_text("runtime: {frames: 120}") + validate_frame_limit(str(path), 120) + for text, requested in [ + ("{}", 120), + ("runtime: {frames: 0}", 120), + ("runtime: {frames: -1}", 120), + ("runtime: {frames: true}", 1), + ("runtime: {frames: 120}", 60), + ("runtime: {frames: 60}", 120), + ("runtime: {frames: 120}", 0), + ]: + with self.subTest(text=text, requested=requested): + path.write_text(text) + with self.assertRaisesRegex(ValueError, "runtime.frames"): + validate_frame_limit(str(path), requested)