From 2dab90b149cf05b8e67c8c41ae8f978f6c3b59f8 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 14 Oct 2026 20:16:35 -0600 Subject: [PATCH 01/13] Fix "local track published for unknown sid" warning --- src/ffi_client.cpp | 19 ++++++++--- src/ffi_client.h | 6 ++-- src/local_participant.cpp | 27 +++++++-------- .../test_local_track_publish_sid.cpp | 33 +++++++++++++++++++ 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/ffi_client.cpp b/src/ffi_client.cpp index 867eba94..2a3eba73 100644 --- a/src/ffi_client.cpp +++ b/src/ffi_client.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -694,9 +695,9 @@ std::future FfiClient::simulateScenarioAsync(uintptr_t room_handle, int sc } // Participant APIs Implementation -std::future FfiClient::publishTrackAsync(std::uint64_t local_participant_handle, - std::uint64_t track_handle, - const TrackPublishOptions& options) { +std::future FfiClient::publishTrackAsync( + std::uint64_t local_participant_handle, std::uint64_t track_handle, const TrackPublishOptions& options, + std::function on_success) { // Generate client-side async_id first const AsyncId async_id = generateAsyncId(); @@ -708,7 +709,8 @@ std::future FfiClient::publishTrackAsync(std::uint return event.has_publish_track() && event.publish_track().async_id() == async_id; }, // Handler: resolve with publication or throw error - [](const proto::FfiEvent& event, std::promise& pr) { + [on_success = std::move(on_success)](const proto::FfiEvent& event, + std::promise& pr) { const auto& cb = event.publish_track(); // Oneof message { string error = 2; OwnedTrackPublication publication = @@ -723,7 +725,14 @@ std::future FfiClient::publishTrackAsync(std::uint } const proto::OwnedTrackPublication& pub = cb.publication(); - pr.set_value(pub); + try { + if (on_success) { + on_success(pub); + } + pr.set_value(pub); + } catch (...) { + pr.set_exception(std::current_exception()); + } }); // Build and send the request diff --git a/src/ffi_client.h b/src/ffi_client.h index 0104de76..4f10b1fa 100644 --- a/src/ffi_client.h +++ b/src/ffi_client.h @@ -109,9 +109,9 @@ class LIVEKIT_INTERNAL_API FfiClient { std::future simulateScenarioAsync(uintptr_t room_handle, int scenario); // Participant APIs - std::future publishTrackAsync(std::uint64_t local_participant_handle, - std::uint64_t track_handle, - const TrackPublishOptions& options); + std::future publishTrackAsync( + std::uint64_t local_participant_handle, std::uint64_t track_handle, const TrackPublishOptions& options, + std::function on_success = {}); std::future unpublishTrackAsync(std::uint64_t local_participant_handle, const std::string& track_sid, bool stop_on_unpublish); std::future publishDataAsync(std::uint64_t local_participant_handle, const std::uint8_t* data_ptr, diff --git a/src/local_participant.cpp b/src/local_participant.cpp index 8dbc1fdc..2607963d 100644 --- a/src/local_participant.cpp +++ b/src/local_participant.cpp @@ -191,21 +191,18 @@ void LocalParticipant::publishTrack(const std::shared_ptr& track, const T if (track_handle == 0) { throw std::runtime_error("LocalParticipant::publishTrack: invalid track FFI handle"); } - auto fut = FfiClient::instance().publishTrackAsync(static_cast(participant_handle), - static_cast(track_handle), options); - - // Will throw if the async op fails (error in callback). - const proto::OwnedTrackPublication owned_pub = fut.get(); - - // Construct a LocalTrackPublication from the proto publication. - auto publication = std::make_shared(owned_pub); - - const std::string sid = publication->sid(); - { - const std::scoped_lock guard(published_tracks_mutex_); - published_tracks_by_sid_[sid] = std::weak_ptr(track); - track->setPublication(publication); - } + auto fut = FfiClient::instance().publishTrackAsync( + static_cast(participant_handle), static_cast(track_handle), options, + [this, track](const proto::OwnedTrackPublication& owned_pub) { + auto publication = std::make_shared(owned_pub); + const std::scoped_lock guard(published_tracks_mutex_); + published_tracks_by_sid_[publication->sid()] = track; + track->setPublication(publication); + }); + + // Publication state is installed on the FFI thread before this future is + // completed and before Rust can emit LocalTrackPublished. + (void)fut.get(); } std::shared_ptr LocalParticipant::publishVideoTrack(const std::string& name, diff --git a/src/tests/integration/test_local_track_publish_sid.cpp b/src/tests/integration/test_local_track_publish_sid.cpp index ee767450..21707f4b 100644 --- a/src/tests/integration/test_local_track_publish_sid.cpp +++ b/src/tests/integration/test_local_track_publish_sid.cpp @@ -17,9 +17,12 @@ #include #include +#include #include +#include #include #include +#include #include "../common/audio_utils.h" #include "../common/room_test_access.h" @@ -31,6 +34,30 @@ namespace { using namespace std::chrono_literals; +class LocalPublicationDelegate final : public RoomDelegate { +public: + void onLocalTrackPublished(Room&, const LocalTrackPublishedEvent& event) override { + if (!event.publication) { + return; + } + { + const std::scoped_lock guard(mutex_); + published_sids_.insert(event.publication->sid()); + } + cv_.notify_all(); + } + + bool waitForPublished(const std::string& sid, std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, timeout, [&] { return published_sids_.count(sid) != 0; }); + } + +private: + std::mutex mutex_; + std::condition_variable cv_; + std::unordered_set published_sids_; +}; + void expectTrackSidAssigned(const Track& track, const LocalTrackPublication& publication) { const std::string& track_sid = track.sid(); const std::string& publication_sid = publication.sid(); @@ -58,6 +85,8 @@ TEST_F(LocalTrackPublishSidTest, PublishVideoTrackAssignsSid) { failIfNotConfigured(); Room room; + LocalPublicationDelegate delegate; + room.setDelegate(&delegate); const RoomOptions room_options; ASSERT_TRUE(room.connect(config_.url, config_.token_a, room_options)); @@ -69,6 +98,7 @@ TEST_F(LocalTrackPublishSidTest, PublishVideoTrackAssignsSid) { ASSERT_NE(track->publication(), nullptr); expectTrackSidAssigned(*track, *track->publication()); + EXPECT_TRUE(delegate.waitForPublished(track->sid(), 5s)); lockLocalParticipant(room)->unpublishTrack(track->publication()->sid()); } @@ -76,6 +106,8 @@ TEST_F(LocalTrackPublishSidTest, PublishAudioTrackAssignsSid) { failIfNotConfigured(); Room room; + LocalPublicationDelegate delegate; + room.setDelegate(&delegate); const RoomOptions room_options; ASSERT_TRUE(room.connect(config_.url, config_.token_a, room_options)); @@ -87,6 +119,7 @@ TEST_F(LocalTrackPublishSidTest, PublishAudioTrackAssignsSid) { ASSERT_NE(track->publication(), nullptr); expectTrackSidAssigned(*track, *track->publication()); + EXPECT_TRUE(delegate.waitForPublished(track->sid(), 5s)); lockLocalParticipant(room)->unpublishTrack(track->publication()->sid()); } From 5adaeef0c0cbe5b0a29dcdc495751ac04b0dec17 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 17:55:31 -0700 Subject: [PATCH 02/13] Memory regression test --- .github/workflows/ci.yml | 1 + .github/workflows/tests.yml | 15 ++ docs/testing.md | 26 +++ scripts/memory-regression.sh | 95 +++++++++++ src/tests/CMakeLists.txt | 19 +++ src/tests/memory/source_lifecycle_tester.cpp | 160 +++++++++++++++++++ 6 files changed, 316 insertions(+) create mode 100755 scripts/memory-regression.sh create mode 100644 src/tests/memory/source_lifecycle_tester.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c20ff88..87152589 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,7 @@ jobs: - client-sdk-rust - client-sdk-rust/** - cmake/** + - scripts/memory-regression.sh - scripts/set-test-tokens.sh - scripts/run-with-backtrace.sh - CMakeLists.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index efeaf448..c38a0a69 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -425,6 +425,21 @@ jobs: --gtest_recreate_environments_when_repeating=1 \ --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml + - name: Run Linux memory regression gate + if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' + timeout-minutes: 10 + shell: bash + env: + MALLOC_ARENA_MAX: "1" + run: | + set -euo pipefail + scripts/memory-regression.sh \ + --build-dir "${{ env.BUILD_DIR }}" \ + --iterations 100 \ + --warmup 20 \ + --max-rss-growth-kib 8192 \ + --max-thread-growth 0 + - name: Run stress tests if: matrix.e2e-testing && inputs.run_stress_tests timeout-minutes: ${{ inputs.stress_timeout_minutes }} diff --git a/docs/testing.md b/docs/testing.md index 33919c0e..383dc48a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -106,6 +106,32 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls. - **Stress**: high throughput, bidirectional RPC, memory pressure. +## Linux memory regression gate + +The Linux PR gate repeatedly initializes the SDK, creates and destroys unused +audio/video sources and tracks, then shuts the SDK down. It compares process +RSS and thread count after cycle 20 with cycle 100, failing on thread growth or +more than 8 MiB of RSS growth. + +The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits +glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a +controlled CI signal, not evidence that the default allocator returns all +freed pages to the OS. + +To reproduce locally (no LiveKit server or tokens are required): + +```bash +./build.sh release-tests +scripts/memory-regression.sh \ + --iterations 100 --warmup 20 \ + --max-rss-growth-kib 8192 --max-thread-growth 0 +``` + +The limits are command-line options so a longer local run can use different +budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends a compact +results table when `GITHUB_STEP_SUMMARY` is set. Hardware encoding and +connected-room lifecycle coverage are separate follow-up work. + ## Memory checks (valgrind) Run `valgrind` against the test binaries to check for memory leaks and other diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh new file mode 100755 index 00000000..c3f217d4 --- /dev/null +++ b/scripts/memory-regression.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# Copyright 2026 LiveKit +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: memory-regression.sh [OPTIONS] + +Run the Linux memory lifecycle regression tester and append a table to the +GitHub step summary when GITHUB_STEP_SUMMARY is set. + +Options: + --build-dir PATH Build directory (default: build-release) + --iterations N Lifecycle cycles (default: 100) + --warmup N Baseline cycle (default: 20) + --max-rss-growth-kib N Allowed RSS growth (default: 8192) + --max-thread-growth N Allowed thread growth (default: 0) + -h, --help Show this help + +Environment: + MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_ITERATIONS, + MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, + MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX +EOF +} + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" +iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" +warmup="${MEMORY_REGRESSION_WARMUP:-20}" +max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" +max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" + +while (($#)); do + case "$1" in + --build-dir) build_dir="$2"; shift 2 ;; + --iterations) iterations="$2"; shift 2 ;; + --warmup) warmup="$2"; shift 2 ;; + --max-rss-growth-kib) max_rss_growth_kib="$2"; shift 2 ;; + --max-thread-growth) max_thread_growth="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: unknown option: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if [[ "${build_dir}" != /* ]]; then + build_dir="${repo_root}/${build_dir}" +fi +tester="${build_dir}/bin/livekit_memory_source_lifecycle_tester" +if [[ ! -x "${tester}" ]]; then + echo "ERROR: ${tester} not found. Run ./build.sh release-tests first." >&2 + exit 1 +fi + +set +e +output="$( + MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \ + --iterations "${iterations}" \ + --warmup "${warmup}" \ + --max-rss-growth-kib "${max_rss_growth_kib}" \ + --max-thread-growth "${max_thread_growth}" 2>&1 +)" +status=$? +set -e +printf '%s\n' "${output}" + +report_regex='memory lifecycle: RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +if [[ -n "${GITHUB_STEP_SUMMARY:-}" && "${output}" =~ ${report_regex} ]]; then + { + echo "## Linux memory regression" + echo + echo "| Metric | Baseline | Final | Growth | Limit |" + echo "|---|---:|---:|---:|---:|" + echo "| RSS | ${BASH_REMATCH[1]} KiB | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${max_rss_growth_kib} KiB |" + echo "| Threads | ${BASH_REMATCH[4]} | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${max_thread_growth} |" + echo + echo "**Verdict: ${BASH_REMATCH[7]}**" + } >> "${GITHUB_STEP_SUMMARY}" +fi + +exit "${status}" diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 9583af73..c9080662 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -348,3 +348,22 @@ endif() if(TARGET livekit_stress_tests) add_dependencies(run_all_tests livekit_stress_tests) endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_executable(livekit_memory_source_lifecycle_tester + "${CMAKE_CURRENT_SOURCE_DIR}/memory/source_lifecycle_tester.cpp" + ) + target_link_libraries(livekit_memory_source_lifecycle_tester PRIVATE livekit) + target_include_directories(livekit_memory_source_lifecycle_tester PRIVATE + ${LIVEKIT_ROOT_DIR}/include + ) + add_custom_command(TARGET livekit_memory_source_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/liblivekit_ffi.so" + $ + COMMENT "Copying shared libraries next to livekit_memory_source_lifecycle_tester" + ) +endif() diff --git a/src/tests/memory/source_lifecycle_tester.cpp b/src/tests/memory/source_lifecycle_tester.cpp new file mode 100644 index 00000000..197da1ee --- /dev/null +++ b/src/tests/memory/source_lifecycle_tester.cpp @@ -0,0 +1,160 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct Options { + std::uint64_t iterations{100}; + std::uint64_t warmup{20}; + std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; + std::uint64_t max_thread_growth{0}; +}; + +struct ProcessSample { + std::uint64_t rss_kib{}; + std::uint64_t threads{}; +}; + +std::uint64_t parseUnsigned(std::string_view value, std::string_view option) { + std::uint64_t parsed = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { + throw std::runtime_error("invalid value for " + std::string(option)); + } + return parsed; +} + +Options parseOptions(int argc, char* argv[]) { + Options options; + for (int index = 1; index < argc; ++index) { + const std::string_view option(argv[index]); + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + std::string(option)); + } + const std::string_view value(argv[++index]); + if (option == "--iterations") { + options.iterations = parseUnsigned(value, option); + } else if (option == "--warmup") { + options.warmup = parseUnsigned(value, option); + } else if (option == "--max-rss-growth-kib") { + options.max_rss_growth_kib = parseUnsigned(value, option); + } else if (option == "--max-thread-growth") { + options.max_thread_growth = parseUnsigned(value, option); + } else { + throw std::runtime_error("unknown option: " + std::string(option)); + } + } + if (options.warmup == 0 || options.warmup >= options.iterations) { + throw std::runtime_error("--warmup must be between 1 and --iterations - 1"); + } + return options; +} + +std::uint64_t statusValue(std::string_view line) { + const auto separator = line.find(':'); + if (separator == std::string_view::npos) { + throw std::runtime_error("malformed /proc/self/status value"); + } + line.remove_prefix(separator + 1); + while (!line.empty() && std::isspace(static_cast(line.front())) != 0) { + line.remove_prefix(1); + } + const auto end = line.find_first_not_of("0123456789"); + return parseUnsigned(line.substr(0, end), "/proc/self/status"); +} + +ProcessSample sampleProcess() { + ProcessSample sample; + std::ifstream status("/proc/self/status"); + std::string line; + while (std::getline(status, line)) { + if (line.rfind("VmRSS:", 0) == 0) { + sample.rss_kib = statusValue(line); + } else if (line.rfind("Threads:", 0) == 0) { + sample.threads = statusValue(line); + } + } + if (sample.rss_kib == 0 || sample.threads == 0) { + throw std::runtime_error("failed to read RSS or thread count from /proc/self/status"); + } + return sample; +} + +void runCycle() { + if (!livekit::initialize(livekit::LogLevel::Warn)) { + throw std::runtime_error("LiveKit initialization failed"); + } + try { + { + auto audio_source = std::make_shared(48'000, 1, 100); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); + auto video_source = std::make_shared(640, 360); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); + if (!audio_track || !video_track) { + throw std::runtime_error("failed to create local tracks"); + } + } + } catch (...) { + livekit::shutdown(); + throw; + } + livekit::shutdown(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); +} + +} // namespace + +int main(int argc, char* argv[]) { + try { + const Options options = parseOptions(argc, argv); + ProcessSample baseline; + ProcessSample final; + for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { + runCycle(); + if (iteration == options.warmup) { + baseline = sampleProcess(); + } else if (iteration == options.iterations) { + final = sampleProcess(); + } + } + + const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); + const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); + const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && + thread_growth <= static_cast(options.max_thread_growth); + std::cout << "memory lifecycle: RSS " << baseline.rss_kib << " -> " << final.rss_kib << " KiB (" << rss_growth + << " KiB), threads " << baseline.threads << " -> " << final.threads << " (" << thread_growth + << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + return 1; + } +} From a89e5ebaa328f8a5971c8a38b468797e263e84ef Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 20:01:44 -0700 Subject: [PATCH 03/13] More scenarios --- .github/workflows/tests.yml | 14 +- docs/testing.md | 35 +- scripts/memory-regression.sh | 92 +++-- src/tests/CMakeLists.txt | 16 +- src/tests/memory/lifecycle_tester.cpp | 334 +++++++++++++++++++ src/tests/memory/source_lifecycle_tester.cpp | 160 --------- 6 files changed, 451 insertions(+), 200 deletions(-) create mode 100644 src/tests/memory/lifecycle_tester.cpp delete mode 100644 src/tests/memory/source_lifecycle_tester.cpp diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c38a0a69..9accd92d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -357,7 +357,11 @@ jobs: --gtest_output="xml:${{ env.BUILD_DIR }}\unit-test-results.xml" - name: Start livekit-server - if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests) + if: >- + matrix.e2e-testing && + (inputs.integration_repeat > 0 || + inputs.run_stress_tests || + (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release')) id: livekit_server uses: livekit/dev-server-action@5d4d5337a875e2d1afd37bed03c601d159dab002 # v1.1.1 with: @@ -368,7 +372,11 @@ jobs: # Needed by token helper script - name: Install livekit-cli - if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests) + if: >- + matrix.e2e-testing && + (inputs.integration_repeat > 0 || + inputs.run_stress_tests || + (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release')) shell: bash env: # Windows installs lk via `gh api` / `gh release download`, which need this env var @@ -433,8 +441,10 @@ jobs: MALLOC_ARENA_MAX: "1" run: | set -euo pipefail + source scripts/set-test-tokens.sh scripts/memory-regression.sh \ --build-dir "${{ env.BUILD_DIR }}" \ + --scenario all \ --iterations 100 \ --warmup 20 \ --max-rss-growth-kib 8192 \ diff --git a/docs/testing.md b/docs/testing.md index 383dc48a..90ebb4d5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -44,6 +44,7 @@ __Note:__ The tests require tokens and a running LiveKit server. See the section | `livekit_unit_tests` | Pure unit tests (no server required) | | `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality | | `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) | +| `livekit_memory_lifecycle_tester` | Linux lifecycle RSS and thread regression scenarios | ## Running a local LiveKit server for tests @@ -108,29 +109,47 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - ## Linux memory regression gate -The Linux PR gate repeatedly initializes the SDK, creates and destroys unused -audio/video sources and tracks, then shuts the SDK down. It compares process -RSS and thread count after cycle 20 with cycle 100, failing on thread growth or -more than 8 MiB of RSS growth. +The Linux PR gate runs three lifecycle scenarios in separate processes: + +- `sdk-sources` repeatedly initializes the SDK, creates and destroys unused + audio/video sources and tracks, then shuts the SDK down. +- `room-client-leave` repeatedly connects, disconnects, and destroys a room. +- `room-server-delete` repeatedly connects and uses `lk` to delete the local + test room, then verifies the disconnect and end-of-stream callbacks. + +Each scenario compares process RSS and thread count after cycle 20 with cycle +100, failing on thread growth or more than 8 MiB of RSS growth. The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a controlled CI signal, not evidence that the default allocator returns all freed pages to the OS. -To reproduce locally (no LiveKit server or tokens are required): +The offline source scenario requires no LiveKit server or tokens: ```bash ./build.sh release-tests scripts/memory-regression.sh \ + --scenario sdk-sources \ --iterations 100 --warmup 20 \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` +To run either room scenario, start `livekit-server --dev`, install the `lk` +CLI, and source the local test tokens. Use `--scenario all` to run all three +scenarios, each in a fresh tester process: + +```bash +source scripts/set-test-tokens.sh +scripts/memory-regression.sh --scenario room-client-leave +scripts/memory-regression.sh --scenario room-server-delete +scripts/memory-regression.sh --scenario all +``` + The limits are command-line options so a longer local run can use different -budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends a compact -results table when `GITHUB_STEP_SUMMARY` is set. Hardware encoding and -connected-room lifecycle coverage are separate follow-up work. +budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends one results +row per scenario when `GITHUB_STEP_SUMMARY` is set. Hardware encoding coverage +is separate follow-up work. ## Memory checks (valgrind) diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index c3f217d4..1032d769 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -25,6 +25,8 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) + --scenario NAME sdk-sources, room-client-leave, + room-server-delete, or all (default: sdk-sources) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -32,14 +34,17 @@ Options: -h, --help Show this help Environment: - MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_ITERATIONS, + MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_SCENARIO, + MEMORY_REGRESSION_ITERATIONS, MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, - MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX + MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. + Room scenarios also require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF } repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" +scenario="${MEMORY_REGRESSION_SCENARIO:-sdk-sources}" iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" warmup="${MEMORY_REGRESSION_WARMUP:-20}" max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" @@ -48,6 +53,7 @@ max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" while (($#)); do case "$1" in --build-dir) build_dir="$2"; shift 2 ;; + --scenario) scenario="$2"; shift 2 ;; --iterations) iterations="$2"; shift 2 ;; --warmup) warmup="$2"; shift 2 ;; --max-rss-growth-kib) max_rss_growth_kib="$2"; shift 2 ;; @@ -60,36 +66,78 @@ done if [[ "${build_dir}" != /* ]]; then build_dir="${repo_root}/${build_dir}" fi -tester="${build_dir}/bin/livekit_memory_source_lifecycle_tester" +tester="${build_dir}/bin/livekit_memory_lifecycle_tester" if [[ ! -x "${tester}" ]]; then echo "ERROR: ${tester} not found. Run ./build.sh release-tests first." >&2 exit 1 fi -set +e -output="$( - MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \ - --iterations "${iterations}" \ - --warmup "${warmup}" \ - --max-rss-growth-kib "${max_rss_growth_kib}" \ - --max-thread-growth "${max_thread_growth}" 2>&1 -)" -status=$? -set -e -printf '%s\n' "${output}" +case "${scenario}" in + sdk-sources|room-client-leave|room-server-delete) + scenarios=("${scenario}") + ;; + all) + scenarios=(sdk-sources room-client-leave room-server-delete) + ;; + *) + echo "ERROR: invalid scenario: ${scenario}" >&2 + usage >&2 + exit 2 + ;; +esac -report_regex='memory lifecycle: RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' -if [[ -n "${GITHUB_STEP_SUMMARY:-}" && "${output}" =~ ${report_regex} ]]; then +for current_scenario in "${scenarios[@]}"; do + if [[ "${current_scenario}" != "sdk-sources" ]] && + [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then + echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 + echo "Run: source scripts/set-test-tokens.sh" >&2 + exit 1 + fi +done + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { echo "## Linux memory regression" echo - echo "| Metric | Baseline | Final | Growth | Limit |" - echo "|---|---:|---:|---:|---:|" - echo "| RSS | ${BASH_REMATCH[1]} KiB | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${max_rss_growth_kib} KiB |" - echo "| Threads | ${BASH_REMATCH[4]} | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${max_thread_growth} |" + echo "Limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." echo - echo "**Verdict: ${BASH_REMATCH[7]}**" + echo "| Scenario | RSS baseline | RSS final | RSS growth | Threads baseline | Threads final | Thread growth | Verdict |" + echo "|---|---:|---:|---:|---:|---:|---:|---|" } >> "${GITHUB_STEP_SUMMARY}" fi -exit "${status}" +report_regex='memory lifecycle: scenario=([a-z-]+), RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +overall_status=0 +for current_scenario in "${scenarios[@]}"; do + set +e + output="$( + MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \ + --scenario "${current_scenario}" \ + --iterations "${iterations}" \ + --warmup "${warmup}" \ + --max-rss-growth-kib "${max_rss_growth_kib}" \ + --max-thread-growth "${max_thread_growth}" 2>&1 + )" + status=$? + set -e + printf '%s\n' "${output}" + + if [[ "${output}" =~ ${report_regex} ]]; then + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + echo "| ${BASH_REMATCH[1]} | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} |" \ + >> "${GITHUB_STEP_SUMMARY}" + fi + else + echo "ERROR: ${current_scenario} did not produce a memory report." >&2 + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + echo "| ${current_scenario} | — | — | — | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" + fi + status=1 + fi + + if [[ "${status}" -ne 0 ]]; then + overall_status=1 + fi +done + +exit "${overall_status}" diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index c9080662..1204193c 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -350,20 +350,20 @@ if(TARGET livekit_stress_tests) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - add_executable(livekit_memory_source_lifecycle_tester - "${CMAKE_CURRENT_SOURCE_DIR}/memory/source_lifecycle_tester.cpp" + add_executable(livekit_memory_lifecycle_tester + "${CMAKE_CURRENT_SOURCE_DIR}/memory/lifecycle_tester.cpp" ) - target_link_libraries(livekit_memory_source_lifecycle_tester PRIVATE livekit) - target_include_directories(livekit_memory_source_lifecycle_tester PRIVATE + target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit) + target_include_directories(livekit_memory_lifecycle_tester PRIVATE ${LIVEKIT_ROOT_DIR}/include ) - add_custom_command(TARGET livekit_memory_source_lifecycle_tester POST_BUILD + add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different "$/liblivekit_ffi.so" - $ - COMMENT "Copying shared libraries next to livekit_memory_source_lifecycle_tester" + $ + COMMENT "Copying shared libraries next to livekit_memory_lifecycle_tester" ) endif() diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp new file mode 100644 index 00000000..8fbb7535 --- /dev/null +++ b/src/tests/memory/lifecycle_tester.cpp @@ -0,0 +1,334 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace std::chrono_literals; + +constexpr std::string_view kLocalTestLiveKitUrl = "ws://localhost:7880"; + +enum class Scenario { + SdkSources, + RoomClientLeave, + RoomServerDelete, +}; + +struct Options { + Scenario scenario{Scenario::SdkSources}; + std::uint64_t iterations{100}; + std::uint64_t warmup{20}; + std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; + std::uint64_t max_thread_growth{0}; +}; + +struct ProcessSample { + std::uint64_t rss_kib{}; + std::uint64_t threads{}; +}; + +struct RoomCredentials { + std::string url; + std::string token; +}; + +class SdkLifetime { +public: + SdkLifetime() { + if (!livekit::initialize(livekit::LogLevel::Warn)) { + throw std::runtime_error("LiveKit initialization failed"); + } + } + + ~SdkLifetime() { livekit::shutdown(); } + + SdkLifetime(const SdkLifetime&) = delete; + SdkLifetime& operator=(const SdkLifetime&) = delete; +}; + +class DisconnectTrackingDelegate final : public livekit::RoomDelegate { +public: + void onDisconnected(livekit::Room&, const livekit::DisconnectedEvent& event) override { + { + const std::scoped_lock lock(mutex_); + reason_ = event.reason; + ++disconnect_count_; + } + condition_.notify_all(); + } + + void onRoomEos(livekit::Room&, const livekit::RoomEosEvent&) override { + { + const std::scoped_lock lock(mutex_); + ++eos_count_; + } + condition_.notify_all(); + } + + bool waitForTeardown(std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return condition_.wait_for(lock, timeout, [this]() { return disconnect_count_ > 0 && eos_count_ > 0; }); + } + + void requireDisconnect(livekit::DisconnectReason expected_reason) const { + const std::scoped_lock lock(mutex_); + if (disconnect_count_ != 1 || reason_ != expected_reason) { + throw std::runtime_error("unexpected room disconnect event"); + } + } + + void requireServerTeardown() const { + const std::scoped_lock lock(mutex_); + if (disconnect_count_ != 1 || eos_count_ != 1 || reason_ != livekit::DisconnectReason::RoomDeleted) { + throw std::runtime_error("unexpected server room teardown events"); + } + } + +private: + mutable std::mutex mutex_; + std::condition_variable condition_; + livekit::DisconnectReason reason_{livekit::DisconnectReason::Unknown}; + int disconnect_count_{0}; + int eos_count_{0}; +}; + +std::string_view scenarioName(Scenario scenario) { + switch (scenario) { + case Scenario::SdkSources: + return "sdk-sources"; + case Scenario::RoomClientLeave: + return "room-client-leave"; + case Scenario::RoomServerDelete: + return "room-server-delete"; + } + throw std::runtime_error("unknown scenario"); +} + +Scenario parseScenario(std::string_view value) { + if (value == "sdk-sources") { + return Scenario::SdkSources; + } + if (value == "room-client-leave") { + return Scenario::RoomClientLeave; + } + if (value == "room-server-delete") { + return Scenario::RoomServerDelete; + } + throw std::runtime_error("invalid value for --scenario"); +} + +std::uint64_t parseUnsigned(std::string_view value, std::string_view option) { + std::uint64_t parsed = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { + throw std::runtime_error("invalid value for " + std::string(option)); + } + return parsed; +} + +Options parseOptions(int argc, char* argv[]) { + Options options; + for (int index = 1; index < argc; ++index) { + const std::string_view option(argv[index]); + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + std::string(option)); + } + const std::string_view value(argv[++index]); + if (option == "--scenario") { + options.scenario = parseScenario(value); + } else if (option == "--iterations") { + options.iterations = parseUnsigned(value, option); + } else if (option == "--warmup") { + options.warmup = parseUnsigned(value, option); + } else if (option == "--max-rss-growth-kib") { + options.max_rss_growth_kib = parseUnsigned(value, option); + } else if (option == "--max-thread-growth") { + options.max_thread_growth = parseUnsigned(value, option); + } else { + throw std::runtime_error("unknown option: " + std::string(option)); + } + } + if (options.warmup == 0 || options.warmup >= options.iterations) { + throw std::runtime_error("--warmup must be between 1 and --iterations - 1"); + } + return options; +} + +std::uint64_t statusValue(std::string_view line) { + const auto separator = line.find(':'); + if (separator == std::string_view::npos) { + throw std::runtime_error("malformed /proc/self/status value"); + } + line.remove_prefix(separator + 1); + while (!line.empty() && std::isspace(static_cast(line.front())) != 0) { + line.remove_prefix(1); + } + const auto end = line.find_first_not_of("0123456789"); + return parseUnsigned(line.substr(0, end), "/proc/self/status"); +} + +ProcessSample sampleProcess() { + ProcessSample sample; + std::ifstream status("/proc/self/status"); + std::string line; + while (std::getline(status, line)) { + if (line.rfind("VmRSS:", 0) == 0) { + sample.rss_kib = statusValue(line); + } else if (line.rfind("Threads:", 0) == 0) { + sample.threads = statusValue(line); + } + } + if (sample.rss_kib == 0 || sample.threads == 0) { + throw std::runtime_error("failed to read RSS or thread count from /proc/self/status"); + } + return sample; +} + +RoomCredentials roomCredentials() { + const char* url = std::getenv("LIVEKIT_URL"); + const char* token = std::getenv("LIVEKIT_TOKEN_A"); + if (url == nullptr || token == nullptr || *url == '\0' || *token == '\0') { + throw std::runtime_error("room scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A"); + } + return {url, token}; +} + +void runSdkSourcesCycle() { + SdkLifetime sdk; + auto audio_source = std::make_shared(48'000, 1, 100); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); + auto video_source = std::make_shared(640, 360); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); + if (!audio_track || !video_track) { + throw std::runtime_error("failed to create local tracks"); + } +} + +void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { + if (!room.connect(credentials.url, credentials.token, livekit::RoomOptions{})) { + throw std::runtime_error("room connection failed"); + } + if (room.connectionState() != livekit::ConnectionState::Connected || room.localParticipant().expired()) { + throw std::runtime_error("room did not reach a connected state"); + } +} + +void runRoomClientLeaveCycle(const RoomCredentials& credentials) { + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + if (room.connectionState() != livekit::ConnectionState::Disconnected || !room.localParticipant().expired()) { + throw std::runtime_error("client room disconnect did not tear down local state"); + } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); +} + +void runRoomServerDeleteCycle(const RoomCredentials& credentials) { + if (std::string_view(credentials.url) != kLocalTestLiveKitUrl) { + throw std::runtime_error("room-server-delete requires LIVEKIT_URL=ws://localhost:7880"); + } + + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + + // The room name is fixed rather than interpolating untrusted environment data into the command. + if (std::system("lk --dev room delete --yes cpp_data_track_test") != 0) { + throw std::runtime_error("failed to delete local test room"); + } + if (!delegate.waitForTeardown(10s)) { + throw std::runtime_error("timed out waiting for server room teardown"); + } + if (room.connectionState() != livekit::ConnectionState::Disconnected || !room.localParticipant().expired()) { + throw std::runtime_error("server room deletion did not tear down local state"); + } + delegate.requireServerTeardown(); +} + +void runCycle(Scenario scenario, const RoomCredentials& credentials) { + switch (scenario) { + case Scenario::SdkSources: + runSdkSourcesCycle(); + return; + case Scenario::RoomClientLeave: + runRoomClientLeaveCycle(credentials); + return; + case Scenario::RoomServerDelete: + runRoomServerDeleteCycle(credentials); + return; + } + throw std::runtime_error("unknown scenario"); +} + +} // namespace + +int main(int argc, char* argv[]) { + try { + const Options options = parseOptions(argc, argv); + const bool room_scenario = options.scenario != Scenario::SdkSources; + const RoomCredentials credentials = room_scenario ? roomCredentials() : RoomCredentials{}; + std::unique_ptr room_sdk; + if (room_scenario) { + room_sdk = std::make_unique(); + } + + ProcessSample baseline; + ProcessSample final; + for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { + runCycle(options.scenario, credentials); + std::this_thread::sleep_for(100ms); + if (iteration == options.warmup) { + baseline = sampleProcess(); + } else if (iteration == options.iterations) { + final = sampleProcess(); + } + } + + const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); + const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); + const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && + thread_growth <= static_cast(options.max_thread_growth); + std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS " << baseline.rss_kib + << " -> " << final.rss_kib << " KiB (" << rss_growth << " KiB), threads " << baseline.threads << " -> " + << final.threads << " (" << thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/src/tests/memory/source_lifecycle_tester.cpp b/src/tests/memory/source_lifecycle_tester.cpp deleted file mode 100644 index 197da1ee..00000000 --- a/src/tests/memory/source_lifecycle_tester.cpp +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2026 LiveKit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - -struct Options { - std::uint64_t iterations{100}; - std::uint64_t warmup{20}; - std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; - std::uint64_t max_thread_growth{0}; -}; - -struct ProcessSample { - std::uint64_t rss_kib{}; - std::uint64_t threads{}; -}; - -std::uint64_t parseUnsigned(std::string_view value, std::string_view option) { - std::uint64_t parsed = 0; - const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); - if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { - throw std::runtime_error("invalid value for " + std::string(option)); - } - return parsed; -} - -Options parseOptions(int argc, char* argv[]) { - Options options; - for (int index = 1; index < argc; ++index) { - const std::string_view option(argv[index]); - if (index + 1 >= argc) { - throw std::runtime_error("missing value for " + std::string(option)); - } - const std::string_view value(argv[++index]); - if (option == "--iterations") { - options.iterations = parseUnsigned(value, option); - } else if (option == "--warmup") { - options.warmup = parseUnsigned(value, option); - } else if (option == "--max-rss-growth-kib") { - options.max_rss_growth_kib = parseUnsigned(value, option); - } else if (option == "--max-thread-growth") { - options.max_thread_growth = parseUnsigned(value, option); - } else { - throw std::runtime_error("unknown option: " + std::string(option)); - } - } - if (options.warmup == 0 || options.warmup >= options.iterations) { - throw std::runtime_error("--warmup must be between 1 and --iterations - 1"); - } - return options; -} - -std::uint64_t statusValue(std::string_view line) { - const auto separator = line.find(':'); - if (separator == std::string_view::npos) { - throw std::runtime_error("malformed /proc/self/status value"); - } - line.remove_prefix(separator + 1); - while (!line.empty() && std::isspace(static_cast(line.front())) != 0) { - line.remove_prefix(1); - } - const auto end = line.find_first_not_of("0123456789"); - return parseUnsigned(line.substr(0, end), "/proc/self/status"); -} - -ProcessSample sampleProcess() { - ProcessSample sample; - std::ifstream status("/proc/self/status"); - std::string line; - while (std::getline(status, line)) { - if (line.rfind("VmRSS:", 0) == 0) { - sample.rss_kib = statusValue(line); - } else if (line.rfind("Threads:", 0) == 0) { - sample.threads = statusValue(line); - } - } - if (sample.rss_kib == 0 || sample.threads == 0) { - throw std::runtime_error("failed to read RSS or thread count from /proc/self/status"); - } - return sample; -} - -void runCycle() { - if (!livekit::initialize(livekit::LogLevel::Warn)) { - throw std::runtime_error("LiveKit initialization failed"); - } - try { - { - auto audio_source = std::make_shared(48'000, 1, 100); - auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); - auto video_source = std::make_shared(640, 360); - auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); - if (!audio_track || !video_track) { - throw std::runtime_error("failed to create local tracks"); - } - } - } catch (...) { - livekit::shutdown(); - throw; - } - livekit::shutdown(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); -} - -} // namespace - -int main(int argc, char* argv[]) { - try { - const Options options = parseOptions(argc, argv); - ProcessSample baseline; - ProcessSample final; - for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { - runCycle(); - if (iteration == options.warmup) { - baseline = sampleProcess(); - } else if (iteration == options.iterations) { - final = sampleProcess(); - } - } - - const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); - const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); - const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && - thread_growth <= static_cast(options.max_thread_growth); - std::cout << "memory lifecycle: RSS " << baseline.rss_kib << " -> " << final.rss_kib << " KiB (" << rss_growth - << " KiB), threads " << baseline.threads << " -> " << final.threads << " (" << thread_growth - << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; - return passed ? 0 : 1; - } catch (const std::exception& error) { - std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; - return 1; - } -} From a92fea49f7a9b71f663d56b4bcaa72ce720ae751 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 20:31:38 -0700 Subject: [PATCH 04/13] Split audio/video track --- docs/testing.md | 18 +++++---- scripts/memory-regression.sh | 12 +++--- src/tests/memory/lifecycle_tester.cpp | 57 +++++++++++++++++++++------ 3 files changed, 61 insertions(+), 26 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 90ebb4d5..3e7c264d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -109,10 +109,12 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - ## Linux memory regression gate -The Linux PR gate runs three lifecycle scenarios in separate processes: +The Linux PR gate runs four lifecycle scenarios in separate processes: -- `sdk-sources` repeatedly initializes the SDK, creates and destroys unused - audio/video sources and tracks, then shuts the SDK down. +- `audio-track` repeatedly initializes the SDK, creates an audio source and + track, captures synthetic PCM frames, destroys them, and shuts the SDK down. +- `video-track` repeatedly initializes the SDK, creates a video source and + track, captures synthetic I420 frames, destroys them, and shuts the SDK down. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. - `room-server-delete` repeatedly connects and uses `lk` to delete the local test room, then verifies the disconnect and end-of-stream callbacks. @@ -125,18 +127,20 @@ glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a controlled CI signal, not evidence that the default allocator returns all freed pages to the OS. -The offline source scenario requires no LiveKit server or tokens: +The offline track scenarios require no LiveKit server or tokens: ```bash ./build.sh release-tests -scripts/memory-regression.sh \ - --scenario sdk-sources \ +scripts/memory-regression.sh --scenario audio-track \ + --iterations 100 --warmup 20 \ + --max-rss-growth-kib 8192 --max-thread-growth 0 +scripts/memory-regression.sh --scenario video-track \ --iterations 100 --warmup 20 \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` To run either room scenario, start `livekit-server --dev`, install the `lk` -CLI, and source the local test tokens. Use `--scenario all` to run all three +CLI, and source the local test tokens. Use `--scenario all` to run all four scenarios, each in a fresh tester process: ```bash diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index 1032d769..74ba29da 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -25,8 +25,8 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) - --scenario NAME sdk-sources, room-client-leave, - room-server-delete, or all (default: sdk-sources) + --scenario NAME audio-track, video-track, room-client-leave, + room-server-delete, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -44,7 +44,7 @@ EOF repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" -scenario="${MEMORY_REGRESSION_SCENARIO:-sdk-sources}" +scenario="${MEMORY_REGRESSION_SCENARIO:-audio-track}" iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" warmup="${MEMORY_REGRESSION_WARMUP:-20}" max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" @@ -73,11 +73,11 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - sdk-sources|room-client-leave|room-server-delete) + audio-track|video-track|room-client-leave|room-server-delete) scenarios=("${scenario}") ;; all) - scenarios=(sdk-sources room-client-leave room-server-delete) + scenarios=(audio-track video-track room-client-leave room-server-delete) ;; *) echo "ERROR: invalid scenario: ${scenario}" >&2 @@ -87,7 +87,7 @@ case "${scenario}" in esac for current_scenario in "${scenarios[@]}"; do - if [[ "${current_scenario}" != "sdk-sources" ]] && + if [[ "${current_scenario}" == room-* ]] && [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 echo "Run: source scripts/set-test-tokens.sh" >&2 diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index 8fbb7535..46576c9d 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -16,6 +16,7 @@ #include +#include #include #include #include @@ -36,15 +37,17 @@ namespace { using namespace std::chrono_literals; constexpr std::string_view kLocalTestLiveKitUrl = "ws://localhost:7880"; +constexpr int kFramesPerCycle = 3; enum class Scenario { - SdkSources, + AudioTrack, + VideoTrack, RoomClientLeave, RoomServerDelete, }; struct Options { - Scenario scenario{Scenario::SdkSources}; + Scenario scenario{Scenario::AudioTrack}; std::uint64_t iterations{100}; std::uint64_t warmup{20}; std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; @@ -123,8 +126,10 @@ class DisconnectTrackingDelegate final : public livekit::RoomDelegate { std::string_view scenarioName(Scenario scenario) { switch (scenario) { - case Scenario::SdkSources: - return "sdk-sources"; + case Scenario::AudioTrack: + return "audio-track"; + case Scenario::VideoTrack: + return "video-track"; case Scenario::RoomClientLeave: return "room-client-leave"; case Scenario::RoomServerDelete: @@ -134,8 +139,11 @@ std::string_view scenarioName(Scenario scenario) { } Scenario parseScenario(std::string_view value) { - if (value == "sdk-sources") { - return Scenario::SdkSources; + if (value == "audio-track") { + return Scenario::AudioTrack; + } + if (value == "video-track") { + return Scenario::VideoTrack; } if (value == "room-client-leave") { return Scenario::RoomClientLeave; @@ -222,14 +230,33 @@ RoomCredentials roomCredentials() { return {url, token}; } -void runSdkSourcesCycle() { +void runAudioTrackCycle() { SdkLifetime sdk; - auto audio_source = std::make_shared(48'000, 1, 100); + auto audio_source = std::make_shared(48'000, 1); auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); + if (!audio_track) { + throw std::runtime_error("failed to create local audio track"); + } + + auto frame = livekit::AudioFrame::create(48'000, 1, 480); + std::fill(frame.data().begin(), frame.data().end(), 100); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + audio_source->captureFrame(frame, 1'000); + } +} + +void runVideoTrackCycle() { + SdkLifetime sdk; auto video_source = std::make_shared(640, 360); auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); - if (!audio_track || !video_track) { - throw std::runtime_error("failed to create local tracks"); + if (!video_track) { + throw std::runtime_error("failed to create local video track"); + } + + auto frame = livekit::VideoFrame::create(640, 360, livekit::VideoBufferType::I420); + std::fill(frame.data(), frame.data() + frame.dataSize(), 0x7f); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + video_source->captureFrame(frame); } } @@ -282,8 +309,11 @@ void runRoomServerDeleteCycle(const RoomCredentials& credentials) { void runCycle(Scenario scenario, const RoomCredentials& credentials) { switch (scenario) { - case Scenario::SdkSources: - runSdkSourcesCycle(); + case Scenario::AudioTrack: + runAudioTrackCycle(); + return; + case Scenario::VideoTrack: + runVideoTrackCycle(); return; case Scenario::RoomClientLeave: runRoomClientLeaveCycle(credentials); @@ -300,7 +330,8 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); - const bool room_scenario = options.scenario != Scenario::SdkSources; + const bool room_scenario = + options.scenario == Scenario::RoomClientLeave || options.scenario == Scenario::RoomServerDelete; const RoomCredentials credentials = room_scenario ? roomCredentials() : RoomCredentials{}; std::unique_ptr room_sdk; if (room_scenario) { From 8ba2935cb17c70413583225ac49f356dc491150a Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 20:46:37 -0700 Subject: [PATCH 05/13] Warm up stats vs. final --- docs/testing.md | 15 +++-- scripts/memory-regression.sh | 25 +++++---- src/tests/memory/lifecycle_tester.cpp | 80 ++++++++++++++++++++++----- 3 files changed, 91 insertions(+), 29 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 3e7c264d..2378b9bf 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -109,18 +109,22 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - ## Linux memory regression gate -The Linux PR gate runs four lifecycle scenarios in separate processes: +The Linux PR gate runs five lifecycle scenarios in separate processes: - `audio-track` repeatedly initializes the SDK, creates an audio source and track, captures synthetic PCM frames, destroys them, and shuts the SDK down. - `video-track` repeatedly initializes the SDK, creates a video source and track, captures synthetic I420 frames, destroys them, and shuts the SDK down. +- `data-track` repeatedly connects, publishes a data track, pushes synthetic + payloads, unpublishes it, and disconnects. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. - `room-server-delete` repeatedly connects and uses `lk` to delete the local test room, then verifies the disconnect and end-of-stream callbacks. -Each scenario compares process RSS and thread count after cycle 20 with cycle -100, failing on thread growth or more than 8 MiB of RSS growth. +Each scenario reports process RSS and thread count at cycle 0, after warmup +cycle 20, and after final cycle 100. The cycle 0 to warmup growth shows expected +one-time initialization; the gate applies to warmup-to-final growth and fails +on thread growth or more than 8 MiB of RSS growth. The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a @@ -139,12 +143,13 @@ scripts/memory-regression.sh --scenario video-track \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` -To run either room scenario, start `livekit-server --dev`, install the `lk` -CLI, and source the local test tokens. Use `--scenario all` to run all four +To run the data-track or room scenarios, start `livekit-server --dev`, install +the `lk` CLI, and source the local test tokens. Use `--scenario all` to run all five scenarios, each in a fresh tester process: ```bash source scripts/set-test-tokens.sh +scripts/memory-regression.sh --scenario data-track scripts/memory-regression.sh --scenario room-client-leave scripts/memory-regression.sh --scenario room-server-delete scripts/memory-regression.sh --scenario all diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index 74ba29da..d805ffc2 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -25,8 +25,9 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) - --scenario NAME audio-track, video-track, room-client-leave, - room-server-delete, or all (default: audio-track) + --scenario NAME audio-track, video-track, data-track, + room-client-leave, room-server-delete, or all + (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -73,11 +74,11 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - audio-track|video-track|room-client-leave|room-server-delete) + audio-track|video-track|data-track|room-client-leave|room-server-delete) scenarios=("${scenario}") ;; all) - scenarios=(audio-track video-track room-client-leave room-server-delete) + scenarios=(audio-track video-track data-track room-client-leave room-server-delete) ;; *) echo "ERROR: invalid scenario: ${scenario}" >&2 @@ -87,7 +88,7 @@ case "${scenario}" in esac for current_scenario in "${scenarios[@]}"; do - if [[ "${current_scenario}" == room-* ]] && + if [[ "${current_scenario}" == "data-track" || "${current_scenario}" == room-* ]] && [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 echo "Run: source scripts/set-test-tokens.sh" >&2 @@ -99,14 +100,14 @@ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { echo "## Linux memory regression" echo - echo "Limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." + echo "Warmup → final limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." echo - echo "| Scenario | RSS baseline | RSS final | RSS growth | Threads baseline | Threads final | Thread growth | Verdict |" - echo "|---|---:|---:|---:|---:|---:|---:|---|" + echo "| Scenario | Metric | Cycle 0 | Warmup (${warmup}) | Final (${iterations}) | 0 → warmup | Warmup → final | Verdict |" + echo "|---|---|---:|---:|---:|---:|---:|---|" } >> "${GITHUB_STEP_SUMMARY}" fi -report_regex='memory lifecycle: scenario=([a-z-]+), RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+) -> warmup ([0-9]+) KiB \((-?[0-9]+) KiB\) -> final ([0-9]+) KiB \((-?[0-9]+) KiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' overall_status=0 for current_scenario in "${scenarios[@]}"; do set +e @@ -124,8 +125,10 @@ for current_scenario in "${scenarios[@]}"; do if [[ "${output}" =~ ${report_regex} ]]; then if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - echo "| ${BASH_REMATCH[1]} | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} |" \ - >> "${GITHUB_STEP_SUMMARY}" + { + echo "| ${BASH_REMATCH[1]} | RSS | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[5]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[6]} KiB | ${BASH_REMATCH[12]} |" + echo "| ${BASH_REMATCH[1]} | Threads | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} | ${BASH_REMATCH[10]} | ${BASH_REMATCH[9]} | ${BASH_REMATCH[11]} | ${BASH_REMATCH[12]} |" + } >> "${GITHUB_STEP_SUMMARY}" fi else echo "ERROR: ${current_scenario} did not produce a memory report." >&2 diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index 46576c9d..7c8cfc9d 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -15,6 +15,7 @@ */ #include +#include #include #include @@ -42,6 +43,7 @@ constexpr int kFramesPerCycle = 3; enum class Scenario { AudioTrack, VideoTrack, + DataTrack, RoomClientLeave, RoomServerDelete, }; @@ -130,6 +132,8 @@ std::string_view scenarioName(Scenario scenario) { return "audio-track"; case Scenario::VideoTrack: return "video-track"; + case Scenario::DataTrack: + return "data-track"; case Scenario::RoomClientLeave: return "room-client-leave"; case Scenario::RoomServerDelete: @@ -145,6 +149,9 @@ Scenario parseScenario(std::string_view value) { if (value == "video-track") { return Scenario::VideoTrack; } + if (value == "data-track") { + return Scenario::DataTrack; + } if (value == "room-client-leave") { return Scenario::RoomClientLeave; } @@ -269,6 +276,41 @@ void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { } } +void runDataTrackCycle(const RoomCredentials& credentials) { + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing data track"); + } + auto publish_result = participant->publishDataTrack("memory-data"); + if (!publish_result) { + throw std::runtime_error("failed to publish local data track"); + } + + const auto& track = publish_result.value(); + livekit::DataTrackFrame frame; + frame.payload.assign(256, 0xFA); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + frame.user_timestamp = static_cast(frame_index); + if (!track->tryPush(frame)) { + throw std::runtime_error("failed to push local data track frame"); + } + } + + track->unpublishDataTrack(); + if (track->isPublished()) { + throw std::runtime_error("local data track remained published"); + } + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); +} + void runRoomClientLeaveCycle(const RoomCredentials& credentials) { DisconnectTrackingDelegate delegate; livekit::Room room; @@ -315,6 +357,9 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { case Scenario::VideoTrack: runVideoTrackCycle(); return; + case Scenario::DataTrack: + runDataTrackCycle(credentials); + return; case Scenario::RoomClientLeave: runRoomClientLeaveCycle(credentials); return; @@ -330,33 +375,42 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); - const bool room_scenario = - options.scenario == Scenario::RoomClientLeave || options.scenario == Scenario::RoomServerDelete; - const RoomCredentials credentials = room_scenario ? roomCredentials() : RoomCredentials{}; + const bool connected_scenario = options.scenario == Scenario::DataTrack || + options.scenario == Scenario::RoomClientLeave || + options.scenario == Scenario::RoomServerDelete; + const RoomCredentials credentials = connected_scenario ? roomCredentials() : RoomCredentials{}; std::unique_ptr room_sdk; - if (room_scenario) { + if (connected_scenario) { room_sdk = std::make_unique(); } - ProcessSample baseline; + const ProcessSample initial = sampleProcess(); + ProcessSample warmup; ProcessSample final; for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { runCycle(options.scenario, credentials); std::this_thread::sleep_for(100ms); if (iteration == options.warmup) { - baseline = sampleProcess(); + warmup = sampleProcess(); } else if (iteration == options.iterations) { final = sampleProcess(); } } - const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); - const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); - const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && - thread_growth <= static_cast(options.max_thread_growth); - std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS " << baseline.rss_kib - << " -> " << final.rss_kib << " KiB (" << rss_growth << " KiB), threads " << baseline.threads << " -> " - << final.threads << " (" << thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + const auto warmup_rss_growth = + static_cast(warmup.rss_kib) - static_cast(initial.rss_kib); + const auto final_rss_growth = static_cast(final.rss_kib) - static_cast(warmup.rss_kib); + const auto warmup_thread_growth = + static_cast(warmup.threads) - static_cast(initial.threads); + const auto final_thread_growth = + static_cast(final.threads) - static_cast(warmup.threads); + const bool passed = final_rss_growth <= static_cast(options.max_rss_growth_kib) && + final_thread_growth <= static_cast(options.max_thread_growth); + std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS 0 " << initial.rss_kib + << " -> warmup " << warmup.rss_kib << " KiB (" << warmup_rss_growth << " KiB) -> final " << final.rss_kib + << " KiB (" << final_rss_growth << " KiB), threads 0 " << initial.threads << " -> warmup " + << warmup.threads << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" + << final_thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; return passed ? 0 : 1; } catch (const std::exception& error) { std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; From 682f6d6e680fdf4f4fc2eed86470f65f4703882b Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 8 Oct 2026 09:39:58 -0700 Subject: [PATCH 06/13] Cleanup docs, comments, add room reuse --- .github/workflows/tests.yml | 2 - docs/testing.md | 44 +++++----- scripts/memory-regression.sh | 38 +++++--- src/tests/memory/lifecycle_tester.cpp | 121 ++++++++++++++++++++++---- 4 files changed, 150 insertions(+), 55 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9accd92d..187dacbd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -437,8 +437,6 @@ jobs: if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' timeout-minutes: 10 shell: bash - env: - MALLOC_ARENA_MAX: "1" run: | set -euo pipefail source scripts/set-test-tokens.sh diff --git a/docs/testing.md b/docs/testing.md index 2378b9bf..2e5f64b4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -107,34 +107,42 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls. - **Stress**: high throughput, bidirectional RPC, memory pressure. -## Linux memory regression gate +## Memory lifecycle tester -The Linux PR gate runs five lifecycle scenarios in separate processes: +> Note: Linux only -- `audio-track` repeatedly initializes the SDK, creates an audio source and - track, captures synthetic PCM frames, destroys them, and shuts the SDK down. -- `video-track` repeatedly initializes the SDK, creates a video source and - track, captures synthetic I420 frames, destroys them, and shuts the SDK down. +The memory lifecycle tester binary catches resource leak regressions in an +automated fashion, and is invoked in CI. This single binary runs individual +or groups of scenarios: + +- `audio-track` repeatedly initializes the SDK, connects, publishes an audio + track, captures synthetic PCM frames, unpublishes, disconnects, and shuts down. +- `video-track` repeatedly initializes the SDK, connects, publishes a video + track, captures synthetic I420 frames, unpublishes, disconnects, and shuts down. - `data-track` repeatedly connects, publishes a data track, pushes synthetic payloads, unpublishes it, and disconnects. +- `room-reuse` repeatedly connects and disconnects the same room object. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. - `room-server-delete` repeatedly connects and uses `lk` to delete the local test room, then verifies the disconnect and end-of-stream callbacks. Each scenario reports process RSS and thread count at cycle 0, after warmup -cycle 20, and after final cycle 100. The cycle 0 to warmup growth shows expected -one-time initialization; the gate applies to warmup-to-final growth and fails -on thread growth or more than 8 MiB of RSS growth. +cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth +and fails on thread growth or more than the configured amount of RSS growth. -The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits -glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a +The binary can be run directly, or via a helper script at `scripts/memory-regression.sh`, +which invokes the binary but also has GitHub actions support for a results summary table. +The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process. +This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a controlled CI signal, not evidence that the default allocator returns all freed pages to the OS. -The offline track scenarios require no LiveKit server or tokens: +Start `livekit-server --dev`, install the `lk` CLI, and source the local test +tokens before running any scenario: ```bash ./build.sh release-tests +source scripts/set-test-tokens.sh scripts/memory-regression.sh --scenario audio-track \ --iterations 100 --warmup 20 \ --max-rss-growth-kib 8192 --max-thread-growth 0 @@ -143,24 +151,20 @@ scripts/memory-regression.sh --scenario video-track \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` -To run the data-track or room scenarios, start `livekit-server --dev`, install -the `lk` CLI, and source the local test tokens. Use `--scenario all` to run all five -scenarios, each in a fresh tester process: +Use `--scenario all` to run all six scenarios, each in a fresh tester process: ```bash -source scripts/set-test-tokens.sh scripts/memory-regression.sh --scenario data-track +scripts/memory-regression.sh --scenario room-reuse scripts/memory-regression.sh --scenario room-client-leave scripts/memory-regression.sh --scenario room-server-delete scripts/memory-regression.sh --scenario all ``` The limits are command-line options so a longer local run can use different -budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends one results -row per scenario when `GITHUB_STEP_SUMMARY` is set. Hardware encoding coverage -is separate follow-up work. +budgets. -## Memory checks (valgrind) +## Additional memory checks (valgrind) Run `valgrind` against the test binaries to check for memory leaks and other issues. See [tools.md](tools.md) for the recipe. diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index d805ffc2..056baa79 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -26,8 +26,8 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) --scenario NAME audio-track, video-track, data-track, - room-client-leave, room-server-delete, or all - (default: audio-track) + room-reuse, room-client-leave, + room-server-delete, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -39,7 +39,7 @@ Environment: MEMORY_REGRESSION_ITERATIONS, MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. - Room scenarios also require LIVEKIT_URL and LIVEKIT_TOKEN_A. + All scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF } @@ -74,11 +74,11 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - audio-track|video-track|data-track|room-client-leave|room-server-delete) + audio-track|video-track|data-track|room-reuse|room-client-leave|room-server-delete) scenarios=("${scenario}") ;; all) - scenarios=(audio-track video-track data-track room-client-leave room-server-delete) + scenarios=(audio-track video-track data-track room-reuse room-client-leave room-server-delete) ;; *) echo "ERROR: invalid scenario: ${scenario}" >&2 @@ -88,8 +88,7 @@ case "${scenario}" in esac for current_scenario in "${scenarios[@]}"; do - if [[ "${current_scenario}" == "data-track" || "${current_scenario}" == room-* ]] && - [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then + if [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 echo "Run: source scripts/set-test-tokens.sh" >&2 exit 1 @@ -102,12 +101,21 @@ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then echo echo "Warmup → final limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." echo - echo "| Scenario | Metric | Cycle 0 | Warmup (${warmup}) | Final (${iterations}) | 0 → warmup | Warmup → final | Verdict |" - echo "|---|---|---:|---:|---:|---:|---:|---|" + echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} |" + echo "|---|---:|---:|---:|---:|" } >> "${GITHUB_STEP_SUMMARY}" fi report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+) -> warmup ([0-9]+) KiB \((-?[0-9]+) KiB\) -> final ([0-9]+) KiB \((-?[0-9]+) KiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' + +format_growth() { + if [[ "$1" == "0" || "$1" == -* ]]; then + printf '%s' "$1" + else + printf '+%s' "$1" + fi +} + overall_status=0 for current_scenario in "${scenarios[@]}"; do set +e @@ -125,15 +133,17 @@ for current_scenario in "${scenarios[@]}"; do if [[ "${output}" =~ ${report_regex} ]]; then if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - { - echo "| ${BASH_REMATCH[1]} | RSS | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[5]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[6]} KiB | ${BASH_REMATCH[12]} |" - echo "| ${BASH_REMATCH[1]} | Threads | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} | ${BASH_REMATCH[10]} | ${BASH_REMATCH[9]} | ${BASH_REMATCH[11]} | ${BASH_REMATCH[12]} |" - } >> "${GITHUB_STEP_SUMMARY}" + rss_warmup_growth="$(format_growth "${BASH_REMATCH[4]}")" + rss_final_growth="$(format_growth "${BASH_REMATCH[6]}")" + thread_warmup_growth="$(format_growth "${BASH_REMATCH[9]}")" + thread_final_growth="$(format_growth "${BASH_REMATCH[11]}")" + echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} KiB | ${rss_final_growth} KiB | ${thread_warmup_growth} | ${thread_final_growth} |" \ + >> "${GITHUB_STEP_SUMMARY}" fi else echo "ERROR: ${current_scenario} did not produce a memory report." >&2 if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - echo "| ${current_scenario} | — | — | — | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" + echo "| ${current_scenario} | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" fi status=1 fi diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index 7c8cfc9d..fb9f0462 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -44,6 +44,7 @@ enum class Scenario { AudioTrack, VideoTrack, DataTrack, + RoomReuse, RoomClientLeave, RoomServerDelete, }; @@ -134,6 +135,8 @@ std::string_view scenarioName(Scenario scenario) { return "video-track"; case Scenario::DataTrack: return "data-track"; + case Scenario::RoomReuse: + return "room-reuse"; case Scenario::RoomClientLeave: return "room-client-leave"; case Scenario::RoomServerDelete: @@ -152,6 +155,9 @@ Scenario parseScenario(std::string_view value) { if (value == "data-track") { return Scenario::DataTrack; } + if (value == "room-reuse") { + return Scenario::RoomReuse; + } if (value == "room-client-leave") { return Scenario::RoomClientLeave; } @@ -232,50 +238,103 @@ RoomCredentials roomCredentials() { const char* url = std::getenv("LIVEKIT_URL"); const char* token = std::getenv("LIVEKIT_TOKEN_A"); if (url == nullptr || token == nullptr || *url == '\0' || *token == '\0') { - throw std::runtime_error("room scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A"); + throw std::runtime_error("memory scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A"); } return {url, token}; } -void runAudioTrackCycle() { +void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { + if (!room.connect(credentials.url, credentials.token, livekit::RoomOptions{})) { + throw std::runtime_error("room connection failed"); + } + if (room.connectionState() != livekit::ConnectionState::Connected || room.localParticipant().expired()) { + throw std::runtime_error("room did not reach a connected state"); + } +} + +// Exercises SDK startup, room connection, audio publication and capture, +// explicit unpublication, client disconnect, and complete process-side teardown. +void runAudioTrackCycle(const RoomCredentials& credentials) { SdkLifetime sdk; + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + auto audio_source = std::make_shared(48'000, 1); auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); if (!audio_track) { throw std::runtime_error("failed to create local audio track"); } + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing audio track"); + } + livekit::TrackPublishOptions publish_options; + publish_options.source = livekit::TrackSource::SOURCE_MICROPHONE; + participant->publishTrack(audio_track, publish_options); + const auto publication = audio_track->publication(); + if (!publication) { + throw std::runtime_error("failed to publish local audio track"); + } + auto frame = livekit::AudioFrame::create(48'000, 1, 480); std::fill(frame.data().begin(), frame.data().end(), 100); for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { audio_source->captureFrame(frame, 1'000); } + + participant->unpublishTrack(publication->sid()); + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } -void runVideoTrackCycle() { +// Exercises SDK startup, room connection, video publication and capture, +// encoder/resource teardown, explicit unpublication, and client disconnect. +void runVideoTrackCycle(const RoomCredentials& credentials) { SdkLifetime sdk; + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + auto video_source = std::make_shared(640, 360); auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); if (!video_track) { throw std::runtime_error("failed to create local video track"); } + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing video track"); + } + livekit::TrackPublishOptions publish_options; + publish_options.source = livekit::TrackSource::SOURCE_CAMERA; + publish_options.simulcast = false; + participant->publishTrack(video_track, publish_options); + const auto publication = video_track->publication(); + if (!publication) { + throw std::runtime_error("failed to publish local video track"); + } + auto frame = livekit::VideoFrame::create(640, 360, livekit::VideoBufferType::I420); std::fill(frame.data(), frame.data() + frame.dataSize(), 0x7f); for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { video_source->captureFrame(frame); } -} -void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { - if (!room.connect(credentials.url, credentials.token, livekit::RoomOptions{})) { - throw std::runtime_error("room connection failed"); - } - if (room.connectionState() != livekit::ConnectionState::Connected || room.localParticipant().expired()) { - throw std::runtime_error("room did not reach a connected state"); + participant->unpublishTrack(publication->sid()); + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } +// Exercises data-track publication, payload enqueueing, explicit unpublication, +// and room teardown while the SDK remains initialized across cycles. void runDataTrackCycle(const RoomCredentials& credentials) { DisconnectTrackingDelegate delegate; livekit::Room room; @@ -311,6 +370,20 @@ void runDataTrackCycle(const RoomCredentials& credentials) { delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } +// Exercises repeated connect/disconnect cycles on one Room instance to detect +// connection state, participant, listener, or FFI resources retained by reuse. +void runRoomReuseCycle(livekit::Room& room, const RoomCredentials& credentials) { + connectRoom(room, credentials); + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + if (room.connectionState() != livekit::ConnectionState::Disconnected || !room.localParticipant().expired()) { + throw std::runtime_error("reused room disconnect did not tear down local state"); + } +} + +// Exercises application-initiated disconnect and destruction of a fresh Room, +// including local-participant cleanup and the expected disconnect callback. void runRoomClientLeaveCycle(const RoomCredentials& credentials) { DisconnectTrackingDelegate delegate; livekit::Room room; @@ -326,6 +399,8 @@ void runRoomClientLeaveCycle(const RoomCredentials& credentials) { delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } +// Exercises server-initiated room deletion and validates disconnect, event-stream +// completion, local-participant cleanup, and destruction of the torn-down Room. void runRoomServerDeleteCycle(const RoomCredentials& credentials) { if (std::string_view(credentials.url) != kLocalTestLiveKitUrl) { throw std::runtime_error("room-server-delete requires LIVEKIT_URL=ws://localhost:7880"); @@ -349,17 +424,23 @@ void runRoomServerDeleteCycle(const RoomCredentials& credentials) { delegate.requireServerTeardown(); } -void runCycle(Scenario scenario, const RoomCredentials& credentials) { +void runCycle(Scenario scenario, const RoomCredentials& credentials, livekit::Room* reusable_room) { switch (scenario) { case Scenario::AudioTrack: - runAudioTrackCycle(); + runAudioTrackCycle(credentials); return; case Scenario::VideoTrack: - runVideoTrackCycle(); + runVideoTrackCycle(credentials); return; case Scenario::DataTrack: runDataTrackCycle(credentials); return; + case Scenario::RoomReuse: + if (reusable_room == nullptr) { + throw std::runtime_error("room-reuse scenario is missing its persistent room"); + } + runRoomReuseCycle(*reusable_room, credentials); + return; case Scenario::RoomClientLeave: runRoomClientLeaveCycle(credentials); return; @@ -375,20 +456,22 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); - const bool connected_scenario = options.scenario == Scenario::DataTrack || - options.scenario == Scenario::RoomClientLeave || - options.scenario == Scenario::RoomServerDelete; - const RoomCredentials credentials = connected_scenario ? roomCredentials() : RoomCredentials{}; + const RoomCredentials credentials = roomCredentials(); + const bool per_cycle_sdk = options.scenario == Scenario::AudioTrack || options.scenario == Scenario::VideoTrack; std::unique_ptr room_sdk; - if (connected_scenario) { + if (!per_cycle_sdk) { room_sdk = std::make_unique(); } + std::unique_ptr reusable_room; + if (options.scenario == Scenario::RoomReuse) { + reusable_room = std::make_unique(); + } const ProcessSample initial = sampleProcess(); ProcessSample warmup; ProcessSample final; for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { - runCycle(options.scenario, credentials); + runCycle(options.scenario, credentials, reusable_room.get()); std::this_thread::sleep_for(100ms); if (iteration == options.warmup) { warmup = sampleProcess(); From d638cca74f5fc1b4b53ba1c323416d0e4b0ee8c0 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 8 Oct 2026 11:33:09 -0700 Subject: [PATCH 07/13] Megabytes, emojis --- .github/workflows/tests.yml | 4 ++-- docs/testing.md | 4 ++-- scripts/memory-regression.sh | 33 ++++++++++++++++----------- src/tests/memory/lifecycle_tester.cpp | 24 +++++++++++-------- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 187dacbd..4d87c24f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -433,7 +433,7 @@ jobs: --gtest_recreate_environments_when_repeating=1 \ --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml - - name: Run Linux memory regression gate + - name: Run memory regression tests if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' timeout-minutes: 10 shell: bash @@ -445,7 +445,7 @@ jobs: --scenario all \ --iterations 100 \ --warmup 20 \ - --max-rss-growth-kib 8192 \ + --max-rss-growth-mib 8 \ --max-thread-growth 0 - name: Run stress tests diff --git a/docs/testing.md b/docs/testing.md index 2e5f64b4..20b7c673 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -145,10 +145,10 @@ tokens before running any scenario: source scripts/set-test-tokens.sh scripts/memory-regression.sh --scenario audio-track \ --iterations 100 --warmup 20 \ - --max-rss-growth-kib 8192 --max-thread-growth 0 + --max-rss-growth-mib 8 --max-thread-growth 0 scripts/memory-regression.sh --scenario video-track \ --iterations 100 --warmup 20 \ - --max-rss-growth-kib 8192 --max-thread-growth 0 + --max-rss-growth-mib 8 --max-thread-growth 0 ``` Use `--scenario all` to run all six scenarios, each in a fresh tester process: diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index 056baa79..cfee8beb 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -30,14 +30,14 @@ Options: room-server-delete, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) - --max-rss-growth-kib N Allowed RSS growth (default: 8192) + --max-rss-growth-mib N Allowed RSS growth (default: 8) --max-thread-growth N Allowed thread growth (default: 0) -h, --help Show this help Environment: MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_SCENARIO, MEMORY_REGRESSION_ITERATIONS, - MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, + MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB, MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. All scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF @@ -48,7 +48,7 @@ build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" scenario="${MEMORY_REGRESSION_SCENARIO:-audio-track}" iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" warmup="${MEMORY_REGRESSION_WARMUP:-20}" -max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" +max_rss_growth_mib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB:-8}" max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" while (($#)); do @@ -57,7 +57,7 @@ while (($#)); do --scenario) scenario="$2"; shift 2 ;; --iterations) iterations="$2"; shift 2 ;; --warmup) warmup="$2"; shift 2 ;; - --max-rss-growth-kib) max_rss_growth_kib="$2"; shift 2 ;; + --max-rss-growth-mib) max_rss_growth_mib="$2"; shift 2 ;; --max-thread-growth) max_thread_growth="$2"; shift 2 ;; -h|--help) usage; exit 0 ;; *) echo "ERROR: unknown option: $1" >&2; usage >&2; exit 2 ;; @@ -97,19 +97,21 @@ done if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { - echo "## Linux memory regression" + echo "## Memory regression results" echo - echo "Warmup → final limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." + echo "Iterations: ${iterations}" + echo "Warmup: ${warmup}" + echo "Threshold: RSS growth ≤ ${max_rss_growth_mib} MiB; thread growth ≤ ${max_thread_growth}." echo - echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} |" - echo "|---|---:|---:|---:|---:|" + echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} | Status |" + echo "|---|---:|---:|---:|---:|---|" } >> "${GITHUB_STEP_SUMMARY}" fi -report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+) -> warmup ([0-9]+) KiB \((-?[0-9]+) KiB\) -> final ([0-9]+) KiB \((-?[0-9]+) KiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+\.[0-9]+) -> warmup ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\) -> final ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' format_growth() { - if [[ "$1" == "0" || "$1" == -* ]]; then + if [[ "$1" == "0" || "$1" == "0.00" || "$1" == -* ]]; then printf '%s' "$1" else printf '+%s' "$1" @@ -124,7 +126,7 @@ for current_scenario in "${scenarios[@]}"; do --scenario "${current_scenario}" \ --iterations "${iterations}" \ --warmup "${warmup}" \ - --max-rss-growth-kib "${max_rss_growth_kib}" \ + --max-rss-growth-mib "${max_rss_growth_mib}" \ --max-thread-growth "${max_thread_growth}" 2>&1 )" status=$? @@ -133,17 +135,22 @@ for current_scenario in "${scenarios[@]}"; do if [[ "${output}" =~ ${report_regex} ]]; then if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + if [[ "${BASH_REMATCH[12]}" == "PASS" ]]; then + verdict=":white_check_mark: PASS" + else + verdict=":x: FAIL" + fi rss_warmup_growth="$(format_growth "${BASH_REMATCH[4]}")" rss_final_growth="$(format_growth "${BASH_REMATCH[6]}")" thread_warmup_growth="$(format_growth "${BASH_REMATCH[9]}")" thread_final_growth="$(format_growth "${BASH_REMATCH[11]}")" - echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} KiB | ${rss_final_growth} KiB | ${thread_warmup_growth} | ${thread_final_growth} |" \ + echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} MiB | ${rss_final_growth} MiB | ${thread_warmup_growth} | ${thread_final_growth} | ${verdict} |" \ >> "${GITHUB_STEP_SUMMARY}" fi else echo "ERROR: ${current_scenario} did not produce a memory report." >&2 if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - echo "| ${current_scenario} | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" + echo "| ${current_scenario} | — | — | — | — | :x: ERROR |" >> "${GITHUB_STEP_SUMMARY}" fi status=1 fi diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index fb9f0462..b8d279ad 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -53,7 +54,7 @@ struct Options { Scenario scenario{Scenario::AudioTrack}; std::uint64_t iterations{100}; std::uint64_t warmup{20}; - std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; + std::uint64_t max_rss_growth_mib{8}; std::uint64_t max_thread_growth{0}; }; @@ -190,8 +191,8 @@ Options parseOptions(int argc, char* argv[]) { options.iterations = parseUnsigned(value, option); } else if (option == "--warmup") { options.warmup = parseUnsigned(value, option); - } else if (option == "--max-rss-growth-kib") { - options.max_rss_growth_kib = parseUnsigned(value, option); + } else if (option == "--max-rss-growth-mib") { + options.max_rss_growth_mib = parseUnsigned(value, option); } else if (option == "--max-thread-growth") { options.max_thread_growth = parseUnsigned(value, option); } else { @@ -234,6 +235,10 @@ ProcessSample sampleProcess() { return sample; } +double kibToMib(std::uint64_t kib) { return static_cast(kib) / 1024.0; } + +double kibToMib(std::int64_t kib) { return static_cast(kib) / 1024.0; } + RoomCredentials roomCredentials() { const char* url = std::getenv("LIVEKIT_URL"); const char* token = std::getenv("LIVEKIT_TOKEN_A"); @@ -487,13 +492,14 @@ int main(int argc, char* argv[]) { static_cast(warmup.threads) - static_cast(initial.threads); const auto final_thread_growth = static_cast(final.threads) - static_cast(warmup.threads); - const bool passed = final_rss_growth <= static_cast(options.max_rss_growth_kib) && + const bool passed = kibToMib(final_rss_growth) <= static_cast(options.max_rss_growth_mib) && final_thread_growth <= static_cast(options.max_thread_growth); - std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS 0 " << initial.rss_kib - << " -> warmup " << warmup.rss_kib << " KiB (" << warmup_rss_growth << " KiB) -> final " << final.rss_kib - << " KiB (" << final_rss_growth << " KiB), threads 0 " << initial.threads << " -> warmup " - << warmup.threads << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" - << final_thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + std::cout << std::fixed << std::setprecision(2) << "memory lifecycle: scenario=" << scenarioName(options.scenario) + << ", RSS 0 " << kibToMib(initial.rss_kib) << " -> warmup " << kibToMib(warmup.rss_kib) << " MiB (" + << kibToMib(warmup_rss_growth) << " MiB) -> final " << kibToMib(final.rss_kib) << " MiB (" + << kibToMib(final_rss_growth) << " MiB), threads 0 " << initial.threads << " -> warmup " << warmup.threads + << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" << final_thread_growth + << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; return passed ? 0 : 1; } catch (const std::exception& error) { std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; From 9af5d015770febf5ccb76a94c8434e0fcb7b0dd3 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 15 Oct 2026 13:23:38 -0600 Subject: [PATCH 08/13] Rename and update docs --- docs/testing.md | 59 ++++++++++++------- src/tests/CMakeLists.txt | 2 +- ...lifecycle_tester.cpp => memory_tester.cpp} | 0 3 files changed, 38 insertions(+), 23 deletions(-) rename src/tests/memory/{lifecycle_tester.cpp => memory_tester.cpp} (100%) diff --git a/docs/testing.md b/docs/testing.md index 20b7c673..cbf97cc5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,7 +1,21 @@ # Testing -The SDK includes integration and stress tests using -[Google Test](https://github.com/google/googletest). +The SDK includes unit, integration and stress tests using +[Google Test](https://github.com/google/googletest). It also +includes a memory-leak/resource checking tester. + +Unit, integration, and memory tests are run on every PR in CI. +Stress tests are reserved for nightly tests in CI due to their +length. + +## Test binaries + +| Executable | Description | +|------------|-------------| +| `livekit_unit_tests` | Unit tests (no server required) | +| `livekit_integration_tests` | Integration tests (server required) | +| `livekit_stress_tests` | Long-running stress tests (server required) | +| `livekit_memory_lifecycle_tester` | Resource leak tester | ## Building the test binaries @@ -17,34 +31,35 @@ The SDK includes integration and stress tests using .\build.cmd release-tests ``` -## Running tests +After building, test binaries are available at: -After building, run tests using `ctest` or invoke the binaries directly: +```bash +# Linux/macOS +/bin/ + +# Windows +\bin\ +``` + +## Running tests ```bash -# Run all tests via ctest +# Run all tests via ctest (warning, runs unit, ) cd build-debug ctest --output-on-failure # Or run test executables directly +./build-debug/bin/livekit_unit_tests ./build-debug/bin/livekit_integration_tests ./build-debug/bin/livekit_stress_tests # Run specific test suites +./build-debug/bin/livekit_unit_tests --gtest_filter="*Room*" ./build-debug/bin/livekit_integration_tests --gtest_filter="*Rpc*" ./build-debug/bin/livekit_stress_tests --gtest_filter="*MaxPayloadStress*" ``` -__Note:__ The tests require tokens and a running LiveKit server. See the section below for details. - -## Test binaries - -| Executable | Description | -|------------|-------------| -| `livekit_unit_tests` | Pure unit tests (no server required) | -| `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality | -| `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) | -| `livekit_memory_lifecycle_tester` | Linux lifecycle RSS and thread regression scenarios | +__Note:__ The integration and stress tests require tokens and a running LiveKit server. See the section below for details. ## Running a local LiveKit server for tests @@ -53,9 +68,10 @@ path is `livekit-server --dev`, which uses the well-known dev API key/secret (`devkey` / `secret`). Install [`livekit-server`](https://docs.livekit.io/home/self-hosting/local/) -and start it with data tracks enabled: +and start it with data blobs enabled: ```bash +LIVEKIT_CONFIG="enable_participant_data_blob: true" livekit-server --dev ``` @@ -130,12 +146,11 @@ Each scenario reports process RSS and thread count at cycle 0, after warmup cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth and fails on thread growth or more than the configured amount of RSS growth. -The binary can be run directly, or via a helper script at `scripts/memory-regression.sh`, -which invokes the binary but also has GitHub actions support for a results summary table. -The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process. -This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a -controlled CI signal, not evidence that the default allocator returns all -freed pages to the OS. +The binary can be run directly, or via a helper script at `scripts/memory-regression.sh`, +which invokes the binary but also has GitHub actions support for a results summary table. +The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process. +This limits `glibc`'s allocator arenas, reducing Linux thread-churn RSS noise, but it is +not evidence that the default allocator returns all freed pages to the OS. Start `livekit-server --dev`, install the `lk` CLI, and source the local test tokens before running any scenario: diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 1204193c..02e02bed 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -351,7 +351,7 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_executable(livekit_memory_lifecycle_tester - "${CMAKE_CURRENT_SOURCE_DIR}/memory/lifecycle_tester.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/memory/memory_tester.cpp" ) target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit) target_include_directories(livekit_memory_lifecycle_tester PRIVATE diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/memory_tester.cpp similarity index 100% rename from src/tests/memory/lifecycle_tester.cpp rename to src/tests/memory/memory_tester.cpp From c6252078cf7dbd5edf10830f1fdea5cfe80b2436 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 15 Oct 2026 17:42:06 -0600 Subject: [PATCH 09/13] Name alignment --- .github/workflows/ci.yml | 2 +- .github/workflows/tests.yml | 4 +-- docs/testing.md | 22 +++++++------- ...{memory-regression.sh => memory-tester.sh} | 30 +++++++++---------- src/tests/CMakeLists.txt | 14 ++++----- src/tests/memory/memory_tester.cpp | 4 +-- 6 files changed, 38 insertions(+), 38 deletions(-) rename scripts/{memory-regression.sh => memory-tester.sh} (81%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87152589..0b5dd776 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - client-sdk-rust - client-sdk-rust/** - cmake/** - - scripts/memory-regression.sh + - scripts/memory-tester.sh - scripts/set-test-tokens.sh - scripts/run-with-backtrace.sh - CMakeLists.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4d87c24f..1060eeb2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -433,14 +433,14 @@ jobs: --gtest_recreate_environments_when_repeating=1 \ --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml - - name: Run memory regression tests + - name: Run memory tester if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' timeout-minutes: 10 shell: bash run: | set -euo pipefail source scripts/set-test-tokens.sh - scripts/memory-regression.sh \ + scripts/memory-tester.sh \ --build-dir "${{ env.BUILD_DIR }}" \ --scenario all \ --iterations 100 \ diff --git a/docs/testing.md b/docs/testing.md index cbf97cc5..f344c837 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -15,7 +15,7 @@ length. | `livekit_unit_tests` | Unit tests (no server required) | | `livekit_integration_tests` | Integration tests (server required) | | `livekit_stress_tests` | Long-running stress tests (server required) | -| `livekit_memory_lifecycle_tester` | Resource leak tester | +| `livekit_memory_tester` | Resource leak tester | ## Building the test binaries @@ -123,11 +123,11 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls. - **Stress**: high throughput, bidirectional RPC, memory pressure. -## Memory lifecycle tester +## Memory tester > Note: Linux only -The memory lifecycle tester binary catches resource leak regressions in an +The memory tester binary catches resource leak regressions in an automated fashion, and is invoked in CI. This single binary runs individual or groups of scenarios: @@ -146,7 +146,7 @@ Each scenario reports process RSS and thread count at cycle 0, after warmup cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth and fails on thread growth or more than the configured amount of RSS growth. -The binary can be run directly, or via a helper script at `scripts/memory-regression.sh`, +The binary can be run directly, or via a helper script at `scripts/memory-tester.sh`, which invokes the binary but also has GitHub actions support for a results summary table. The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process. This limits `glibc`'s allocator arenas, reducing Linux thread-churn RSS noise, but it is @@ -158,10 +158,10 @@ tokens before running any scenario: ```bash ./build.sh release-tests source scripts/set-test-tokens.sh -scripts/memory-regression.sh --scenario audio-track \ +scripts/memory-tester.sh --scenario audio-track \ --iterations 100 --warmup 20 \ --max-rss-growth-mib 8 --max-thread-growth 0 -scripts/memory-regression.sh --scenario video-track \ +scripts/memory-tester.sh --scenario video-track \ --iterations 100 --warmup 20 \ --max-rss-growth-mib 8 --max-thread-growth 0 ``` @@ -169,11 +169,11 @@ scripts/memory-regression.sh --scenario video-track \ Use `--scenario all` to run all six scenarios, each in a fresh tester process: ```bash -scripts/memory-regression.sh --scenario data-track -scripts/memory-regression.sh --scenario room-reuse -scripts/memory-regression.sh --scenario room-client-leave -scripts/memory-regression.sh --scenario room-server-delete -scripts/memory-regression.sh --scenario all +scripts/memory-tester.sh --scenario data-track +scripts/memory-tester.sh --scenario room-reuse +scripts/memory-tester.sh --scenario room-client-leave +scripts/memory-tester.sh --scenario room-server-delete +scripts/memory-tester.sh --scenario all ``` The limits are command-line options so a longer local run can use different diff --git a/scripts/memory-regression.sh b/scripts/memory-tester.sh similarity index 81% rename from scripts/memory-regression.sh rename to scripts/memory-tester.sh index cfee8beb..936f48e6 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-tester.sh @@ -18,9 +18,9 @@ set -euo pipefail usage() { cat <<'EOF' -Usage: memory-regression.sh [OPTIONS] +Usage: memory-tester.sh [OPTIONS] -Run the Linux memory lifecycle regression tester and append a table to the +Run the Linux memory tester and append a table to the GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: @@ -35,21 +35,21 @@ Options: -h, --help Show this help Environment: - MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_SCENARIO, - MEMORY_REGRESSION_ITERATIONS, - MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB, - MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. + MEMORY_TESTER_BUILD_DIR, MEMORY_TESTER_SCENARIO, + MEMORY_TESTER_ITERATIONS, + MEMORY_TESTER_WARMUP, MEMORY_TESTER_MAX_RSS_GROWTH_MIB, + MEMORY_TESTER_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. All scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF } repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" -build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" -scenario="${MEMORY_REGRESSION_SCENARIO:-audio-track}" -iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" -warmup="${MEMORY_REGRESSION_WARMUP:-20}" -max_rss_growth_mib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB:-8}" -max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" +build_dir="${MEMORY_TESTER_BUILD_DIR:-build-release}" +scenario="${MEMORY_TESTER_SCENARIO:-audio-track}" +iterations="${MEMORY_TESTER_ITERATIONS:-100}" +warmup="${MEMORY_TESTER_WARMUP:-20}" +max_rss_growth_mib="${MEMORY_TESTER_MAX_RSS_GROWTH_MIB:-8}" +max_thread_growth="${MEMORY_TESTER_MAX_THREAD_GROWTH:-0}" while (($#)); do case "$1" in @@ -67,7 +67,7 @@ done if [[ "${build_dir}" != /* ]]; then build_dir="${repo_root}/${build_dir}" fi -tester="${build_dir}/bin/livekit_memory_lifecycle_tester" +tester="${build_dir}/bin/livekit_memory_tester" if [[ ! -x "${tester}" ]]; then echo "ERROR: ${tester} not found. Run ./build.sh release-tests first." >&2 exit 1 @@ -97,7 +97,7 @@ done if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { - echo "## Memory regression results" + echo "## Memory tester results" echo echo "Iterations: ${iterations}" echo "Warmup: ${warmup}" @@ -108,7 +108,7 @@ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then } >> "${GITHUB_STEP_SUMMARY}" fi -report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+\.[0-9]+) -> warmup ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\) -> final ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +report_regex='memory tester: scenario=([a-z-]+), RSS 0 ([0-9]+\.[0-9]+) -> warmup ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\) -> final ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' format_growth() { if [[ "$1" == "0" || "$1" == "0.00" || "$1" == -* ]]; then diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 02e02bed..23d5fc65 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -350,20 +350,20 @@ if(TARGET livekit_stress_tests) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - add_executable(livekit_memory_lifecycle_tester + add_executable(livekit_memory_tester "${CMAKE_CURRENT_SOURCE_DIR}/memory/memory_tester.cpp" ) - target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit) - target_include_directories(livekit_memory_lifecycle_tester PRIVATE + target_link_libraries(livekit_memory_tester PRIVATE livekit) + target_include_directories(livekit_memory_tester PRIVATE ${LIVEKIT_ROOT_DIR}/include ) - add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD + add_custom_command(TARGET livekit_memory_tester POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different "$/liblivekit_ffi.so" - $ - COMMENT "Copying shared libraries next to livekit_memory_lifecycle_tester" + $ + COMMENT "Copying shared libraries next to livekit_memory_tester" ) endif() diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index b8d279ad..ff7b52b0 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -494,7 +494,7 @@ int main(int argc, char* argv[]) { static_cast(final.threads) - static_cast(warmup.threads); const bool passed = kibToMib(final_rss_growth) <= static_cast(options.max_rss_growth_mib) && final_thread_growth <= static_cast(options.max_thread_growth); - std::cout << std::fixed << std::setprecision(2) << "memory lifecycle: scenario=" << scenarioName(options.scenario) + std::cout << std::fixed << std::setprecision(2) << "memory tester: scenario=" << scenarioName(options.scenario) << ", RSS 0 " << kibToMib(initial.rss_kib) << " -> warmup " << kibToMib(warmup.rss_kib) << " MiB (" << kibToMib(warmup_rss_growth) << " MiB) -> final " << kibToMib(final.rss_kib) << " MiB (" << kibToMib(final_rss_growth) << " MiB), threads 0 " << initial.threads << " -> warmup " << warmup.threads @@ -502,7 +502,7 @@ int main(int argc, char* argv[]) { << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; return passed ? 0 : 1; } catch (const std::exception& error) { - std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + std::cerr << "memory tester failed: " << error.what() << '\n'; return 1; } } From 4cf631b092cf90f8aa1c16980da702a1622a5118 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 15 Oct 2026 17:44:32 -0700 Subject: [PATCH 10/13] Actually catch room server delete --- docs/testing.md | 6 ++++-- src/tests/memory/memory_tester.cpp | 31 ++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index f344c837..787a61ed 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -139,8 +139,10 @@ or groups of scenarios: payloads, unpublishes it, and disconnects. - `room-reuse` repeatedly connects and disconnects the same room object. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. -- `room-server-delete` repeatedly connects and uses `lk` to delete the local - test room, then verifies the disconnect and end-of-stream callbacks. +- `room-server-delete` repeatedly connects, publishes one synthetic video + track, captures I420 frames without unpublishing, and uses `lk` to delete + the local test room. It then verifies the disconnect and end-of-stream + callbacks. Each scenario reports process RSS and thread count at cycle 0, after warmup cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index ff7b52b0..7c4a00f3 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -404,8 +404,11 @@ void runRoomClientLeaveCycle(const RoomCredentials& credentials) { delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } -// Exercises server-initiated room deletion and validates disconnect, event-stream -// completion, local-participant cleanup, and destruction of the torn-down Room. +// Exercises server-initiated room deletion while one published video track is +// still attached. Teardown must release that publication after the peer +// connection has already closed; unpublishing first avoids the leak this +// scenario targets. The synthetic track uses the default software-capable +// encoder so the scenario can run in CI. void runRoomServerDeleteCycle(const RoomCredentials& credentials) { if (std::string_view(credentials.url) != kLocalTestLiveKitUrl) { throw std::runtime_error("room-server-delete requires LIVEKIT_URL=ws://localhost:7880"); @@ -416,6 +419,30 @@ void runRoomServerDeleteCycle(const RoomCredentials& credentials) { room.setDelegate(&delegate); connectRoom(room, credentials); + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing video track"); + } + + auto video_source = std::make_shared(640, 360); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); + if (!video_track) { + throw std::runtime_error("failed to create local video track"); + } + livekit::TrackPublishOptions publish_options; + publish_options.source = livekit::TrackSource::SOURCE_CAMERA; + publish_options.simulcast = false; + participant->publishTrack(video_track, publish_options); + if (!video_track->publication()) { + throw std::runtime_error("failed to publish local video track"); + } + + auto frame = livekit::VideoFrame::create(640, 360, livekit::VideoBufferType::I420); + std::fill(frame.data(), frame.data() + frame.dataSize(), 0x7f); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + video_source->captureFrame(frame); + } + // The room name is fixed rather than interpolating untrusted environment data into the command. if (std::system("lk --dev room delete --yes cpp_data_track_test") != 0) { throw std::runtime_error("failed to delete local test room"); From b3381edbdfd397aeb751ebae33fcffda0a846fba Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 15 Oct 2026 17:48:03 -0700 Subject: [PATCH 11/13] Fix typo --- docs/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing.md b/docs/testing.md index 787a61ed..37254e43 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -44,7 +44,7 @@ After building, test binaries are available at: ## Running tests ```bash -# Run all tests via ctest (warning, runs unit, ) +# Run all tests via ctest (warning, runs all GTest-based tests including stress) cd build-debug ctest --output-on-failure From 565be38ac98650468b867a643972a25de6376542 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 15 Oct 2026 18:35:08 -0700 Subject: [PATCH 12/13] Maybe fix instability --- docs/testing.md | 8 +- src/tests/memory/memory_tester.cpp | 138 ++++++++++++++++++++++++++--- 2 files changed, 130 insertions(+), 16 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 37254e43..81a45e39 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -145,8 +145,12 @@ or groups of scenarios: callbacks. Each scenario reports process RSS and thread count at cycle 0, after warmup -cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth -and fails on thread growth or more than the configured amount of RSS growth. +cycle 20, and after final cycle 100. Checkpoint sampling waits for the thread +count to remain stable for 500 ms so short-lived teardown work does not cause a +false failure. The gate applies to warmup-to-final growth and fails on thread +growth or more than the configured amount of RSS growth. If threads grow, the +tester also reports when growth was first observed and groups the warmup and +final threads by their Linux thread names. The binary can be run directly, or via a helper script at `scripts/memory-tester.sh`, which invokes the binary but also has GitHub actions support for a results summary table. diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index 7c4a00f3..00a99661 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -24,11 +24,14 @@ #include #include #include +#include #include #include #include +#include #include #include +#include #include #include #include @@ -40,6 +43,9 @@ using namespace std::chrono_literals; constexpr std::string_view kLocalTestLiveKitUrl = "ws://localhost:7880"; constexpr int kFramesPerCycle = 3; +constexpr auto kThreadQuiescencePeriod = 500ms; +constexpr auto kThreadQuiescencePollInterval = 50ms; +constexpr auto kThreadQuiescenceTimeout = 2s; enum class Scenario { AudioTrack, @@ -63,6 +69,13 @@ struct ProcessSample { std::uint64_t threads{}; }; +using ThreadNameCounts = std::map; + +struct ProcessSnapshot { + ProcessSample process; + ThreadNameCounts thread_names; +}; + struct RoomCredentials { std::string url; std::string token; @@ -235,6 +248,82 @@ ProcessSample sampleProcess() { return sample; } +ThreadNameCounts sampleThreadNames() { + ThreadNameCounts counts; + std::error_code error; + std::filesystem::directory_iterator iterator("/proc/self/task", error); + const std::filesystem::directory_iterator end; + while (!error && iterator != end) { + std::ifstream comm(iterator->path() / "comm"); + std::string name; + if (std::getline(comm, name) && !name.empty()) { + ++counts[name]; + } + iterator.increment(error); + } + return counts; +} + +ProcessSnapshot sampleProcessSnapshot() { return {sampleProcess(), sampleThreadNames()}; } + +ProcessSnapshot sampleQuiescentProcess() { + auto snapshot = sampleProcessSnapshot(); + auto stable_since = std::chrono::steady_clock::now(); + const auto deadline = stable_since + kThreadQuiescenceTimeout; + while (std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(kThreadQuiescencePollInterval); + auto next = sampleProcessSnapshot(); + if (next.process.threads != snapshot.process.threads) { + stable_since = std::chrono::steady_clock::now(); + } else if (std::chrono::steady_clock::now() - stable_since >= kThreadQuiescencePeriod) { + return next; + } + snapshot = std::move(next); + } + std::cerr << "memory tester: thread count did not quiesce within " + << std::chrono::duration_cast(kThreadQuiescenceTimeout).count() + << " ms; using the latest sample\n"; + return snapshot; +} + +std::string formatThreadNames(const ThreadNameCounts& counts) { + std::ostringstream output; + bool first = true; + for (const auto& [name, count] : counts) { + if (!first) { + output << ", "; + } + output << name << '=' << count; + first = false; + } + return output.str(); +} + +std::string formatThreadNameDelta(const ThreadNameCounts& before, const ThreadNameCounts& after) { + ThreadNameCounts names = before; + names.insert(after.begin(), after.end()); + + std::ostringstream output; + bool first = true; + for (const auto& [name, unused] : names) { + static_cast(unused); + const auto before_entry = before.find(name); + const auto after_entry = after.find(name); + const auto before_count = before_entry == before.end() ? 0 : before_entry->second; + const auto after_count = after_entry == after.end() ? 0 : after_entry->second; + const auto delta = static_cast(after_count) - static_cast(before_count); + if (delta == 0) { + continue; + } + if (!first) { + output << ", "; + } + output << name << '=' << (delta > 0 ? "+" : "") << delta; + first = false; + } + return output.str(); +} + double kibToMib(std::uint64_t kib) { return static_cast(kib) / 1024.0; } double kibToMib(std::int64_t kib) { return static_cast(kib) / 1024.0; } @@ -499,34 +588,55 @@ int main(int argc, char* argv[]) { reusable_room = std::make_unique(); } - const ProcessSample initial = sampleProcess(); - ProcessSample warmup; - ProcessSample final; + const ProcessSnapshot initial = sampleProcessSnapshot(); + ProcessSnapshot warmup; + ProcessSnapshot final; + std::uint64_t first_thread_growth_iteration = 0; + ProcessSnapshot first_thread_growth; for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { runCycle(options.scenario, credentials, reusable_room.get()); std::this_thread::sleep_for(100ms); if (iteration == options.warmup) { - warmup = sampleProcess(); + warmup = sampleQuiescentProcess(); } else if (iteration == options.iterations) { - final = sampleProcess(); + final = sampleQuiescentProcess(); + } else if (iteration > options.warmup && first_thread_growth_iteration == 0) { + auto current = sampleProcessSnapshot(); + if (current.process.threads > warmup.process.threads) { + first_thread_growth_iteration = iteration; + first_thread_growth = std::move(current); + } } } const auto warmup_rss_growth = - static_cast(warmup.rss_kib) - static_cast(initial.rss_kib); - const auto final_rss_growth = static_cast(final.rss_kib) - static_cast(warmup.rss_kib); + static_cast(warmup.process.rss_kib) - static_cast(initial.process.rss_kib); + const auto final_rss_growth = + static_cast(final.process.rss_kib) - static_cast(warmup.process.rss_kib); const auto warmup_thread_growth = - static_cast(warmup.threads) - static_cast(initial.threads); + static_cast(warmup.process.threads) - static_cast(initial.process.threads); const auto final_thread_growth = - static_cast(final.threads) - static_cast(warmup.threads); + static_cast(final.process.threads) - static_cast(warmup.process.threads); const bool passed = kibToMib(final_rss_growth) <= static_cast(options.max_rss_growth_mib) && final_thread_growth <= static_cast(options.max_thread_growth); + if (final_thread_growth > 0) { + if (first_thread_growth_iteration == 0) { + first_thread_growth_iteration = options.iterations; + first_thread_growth = final; + } + std::cerr << "memory tester: thread growth first observed after cycle " << first_thread_growth_iteration + << "; warmup={" << formatThreadNames(warmup.thread_names) << "}; first-growth={" + << formatThreadNames(first_thread_growth.thread_names) << "}; final={" + << formatThreadNames(final.thread_names) << "}; warmup-to-final delta={" + << formatThreadNameDelta(warmup.thread_names, final.thread_names) << "}\n"; + } std::cout << std::fixed << std::setprecision(2) << "memory tester: scenario=" << scenarioName(options.scenario) - << ", RSS 0 " << kibToMib(initial.rss_kib) << " -> warmup " << kibToMib(warmup.rss_kib) << " MiB (" - << kibToMib(warmup_rss_growth) << " MiB) -> final " << kibToMib(final.rss_kib) << " MiB (" - << kibToMib(final_rss_growth) << " MiB), threads 0 " << initial.threads << " -> warmup " << warmup.threads - << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" << final_thread_growth - << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + << ", RSS 0 " << kibToMib(initial.process.rss_kib) << " -> warmup " << kibToMib(warmup.process.rss_kib) + << " MiB (" << kibToMib(warmup_rss_growth) << " MiB) -> final " << kibToMib(final.process.rss_kib) + << " MiB (" << kibToMib(final_rss_growth) << " MiB), threads 0 " << initial.process.threads + << " -> warmup " << warmup.process.threads << " (" << warmup_thread_growth << ") -> final " + << final.process.threads << " (" << final_thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") + << '\n'; return passed ? 0 : 1; } catch (const std::exception& error) { std::cerr << "memory tester failed: " << error.what() << '\n'; From 205ed52c657ed26ad89912be2873afaaf0976593 Mon Sep 17 00:00:00 2001 From: Alan George Date: Fri, 16 Oct 2026 08:34:59 -0700 Subject: [PATCH 13/13] Address devin.ai comments --- docs/testing.md | 4 +- src/tests/memory/memory_tester.cpp | 63 ++++++++++++++++++------------ 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 81a45e39..eb66f5da 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -149,8 +149,8 @@ cycle 20, and after final cycle 100. Checkpoint sampling waits for the thread count to remain stable for 500 ms so short-lived teardown work does not cause a false failure. The gate applies to warmup-to-final growth and fails on thread growth or more than the configured amount of RSS growth. If threads grow, the -tester also reports when growth was first observed and groups the warmup and -final threads by their Linux thread names. +tester also reports when the final sustained growth began and groups the warmup +and final threads by their Linux thread names. The binary can be run directly, or via a helper script at `scripts/memory-tester.sh`, which invokes the binary but also has GitHub actions support for a results summary table. diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index 00a99661..54db826a 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -70,10 +70,11 @@ struct ProcessSample { }; using ThreadNameCounts = std::map; +using ThreadIdentities = std::map; struct ProcessSnapshot { ProcessSample process; - ThreadNameCounts thread_names; + ThreadIdentities threads; }; struct RoomCredentials { @@ -248,8 +249,8 @@ ProcessSample sampleProcess() { return sample; } -ThreadNameCounts sampleThreadNames() { - ThreadNameCounts counts; +ThreadIdentities sampleThreads() { + ThreadIdentities threads; std::error_code error; std::filesystem::directory_iterator iterator("/proc/self/task", error); const std::filesystem::directory_iterator end; @@ -257,14 +258,14 @@ ThreadNameCounts sampleThreadNames() { std::ifstream comm(iterator->path() / "comm"); std::string name; if (std::getline(comm, name) && !name.empty()) { - ++counts[name]; + threads.emplace(iterator->path().filename().string(), std::move(name)); } iterator.increment(error); } - return counts; + return threads; } -ProcessSnapshot sampleProcessSnapshot() { return {sampleProcess(), sampleThreadNames()}; } +ProcessSnapshot sampleProcessSnapshot() { return {sampleProcess(), sampleThreads()}; } ProcessSnapshot sampleQuiescentProcess() { auto snapshot = sampleProcessSnapshot(); @@ -273,7 +274,7 @@ ProcessSnapshot sampleQuiescentProcess() { while (std::chrono::steady_clock::now() < deadline) { std::this_thread::sleep_for(kThreadQuiescencePollInterval); auto next = sampleProcessSnapshot(); - if (next.process.threads != snapshot.process.threads) { + if (next.threads != snapshot.threads) { stable_since = std::chrono::steady_clock::now(); } else if (std::chrono::steady_clock::now() - stable_since >= kThreadQuiescencePeriod) { return next; @@ -286,10 +287,19 @@ ProcessSnapshot sampleQuiescentProcess() { return snapshot; } -std::string formatThreadNames(const ThreadNameCounts& counts) { +ThreadNameCounts countThreadNames(const ThreadIdentities& threads) { + ThreadNameCounts counts; + for (const auto& [id, name] : threads) { + static_cast(id); + ++counts[name]; + } + return counts; +} + +std::string formatThreadNames(const ThreadIdentities& threads) { std::ostringstream output; bool first = true; - for (const auto& [name, count] : counts) { + for (const auto& [name, count] : countThreadNames(threads)) { if (!first) { output << ", "; } @@ -299,7 +309,9 @@ std::string formatThreadNames(const ThreadNameCounts& counts) { return output.str(); } -std::string formatThreadNameDelta(const ThreadNameCounts& before, const ThreadNameCounts& after) { +std::string formatThreadNameDelta(const ThreadIdentities& before_threads, const ThreadIdentities& after_threads) { + const auto before = countThreadNames(before_threads); + const auto after = countThreadNames(after_threads); ThreadNameCounts names = before; names.insert(after.begin(), after.end()); @@ -591,8 +603,8 @@ int main(int argc, char* argv[]) { const ProcessSnapshot initial = sampleProcessSnapshot(); ProcessSnapshot warmup; ProcessSnapshot final; - std::uint64_t first_thread_growth_iteration = 0; - ProcessSnapshot first_thread_growth; + std::uint64_t sustained_thread_growth_iteration = 0; + ProcessSnapshot sustained_thread_growth; for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { runCycle(options.scenario, credentials, reusable_room.get()); std::this_thread::sleep_for(100ms); @@ -600,11 +612,14 @@ int main(int argc, char* argv[]) { warmup = sampleQuiescentProcess(); } else if (iteration == options.iterations) { final = sampleQuiescentProcess(); - } else if (iteration > options.warmup && first_thread_growth_iteration == 0) { + } else if (iteration > options.warmup) { auto current = sampleProcessSnapshot(); - if (current.process.threads > warmup.process.threads) { - first_thread_growth_iteration = iteration; - first_thread_growth = std::move(current); + if (current.process.threads <= warmup.process.threads) { + sustained_thread_growth_iteration = 0; + sustained_thread_growth = {}; + } else if (sustained_thread_growth_iteration == 0) { + sustained_thread_growth_iteration = iteration; + sustained_thread_growth = std::move(current); } } } @@ -620,15 +635,15 @@ int main(int argc, char* argv[]) { const bool passed = kibToMib(final_rss_growth) <= static_cast(options.max_rss_growth_mib) && final_thread_growth <= static_cast(options.max_thread_growth); if (final_thread_growth > 0) { - if (first_thread_growth_iteration == 0) { - first_thread_growth_iteration = options.iterations; - first_thread_growth = final; + if (sustained_thread_growth_iteration == 0) { + sustained_thread_growth_iteration = options.iterations; + sustained_thread_growth = final; } - std::cerr << "memory tester: thread growth first observed after cycle " << first_thread_growth_iteration - << "; warmup={" << formatThreadNames(warmup.thread_names) << "}; first-growth={" - << formatThreadNames(first_thread_growth.thread_names) << "}; final={" - << formatThreadNames(final.thread_names) << "}; warmup-to-final delta={" - << formatThreadNameDelta(warmup.thread_names, final.thread_names) << "}\n"; + std::cerr << "memory tester: final thread growth sustained since cycle " << sustained_thread_growth_iteration + << "; warmup={" << formatThreadNames(warmup.threads) << "}; growth-start={" + << formatThreadNames(sustained_thread_growth.threads) << "}; final={" + << formatThreadNames(final.threads) << "}; warmup-to-final delta={" + << formatThreadNameDelta(warmup.threads, final.threads) << "}\n"; } std::cout << std::fixed << std::setprecision(2) << "memory tester: scenario=" << scenarioName(options.scenario) << ", RSS 0 " << kibToMib(initial.process.rss_kib) << " -> warmup " << kibToMib(warmup.process.rss_kib)