From ae9c41438d009faedadb53fd924cca76b6f965e2 Mon Sep 17 00:00:00 2001 From: Harshit Date: Thu, 27 Aug 2026 13:00:32 +0530 Subject: [PATCH] test(recording): add concurrent-recording saturation test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit socket.io-client test that ramps N concurrent server-side recordings (each spawns an FFmpeg process + PlainTransport pair + 2 RTP ports) to find the CPU/disk/FD/port saturation point. - load-test/recording-concurrency.js — concurrent start/stop-recording; start-latency percentiles - load-test/results/RECORDING-CONCURRENCY.md — per-recording cost, saturation sampling Authored by Claude (Anthropic) via Claude Code. Not executed against a live backend — results are placeholders, not fabricated. Co-Authored-By: Claude Opus 4.8 (1M context) --- load-test/recording-concurrency.js | 294 +++++++++++++++++++++ load-test/results/RECORDING-CONCURRENCY.md | 142 ++++++++++ 2 files changed, 436 insertions(+) create mode 100644 load-test/recording-concurrency.js create mode 100644 load-test/results/RECORDING-CONCURRENCY.md diff --git a/load-test/recording-concurrency.js b/load-test/recording-concurrency.js new file mode 100644 index 0000000..e0a4e1c --- /dev/null +++ b/load-test/recording-concurrency.js @@ -0,0 +1,294 @@ +// recording-concurrency.js — CrowdStream server-side recording load test +// +// Ramps N concurrent server-side recordings against a single LIVE room to find +// the saturation point. Each recording makes the server spawn an FFmpeg process, +// create an audio+video mediasoup PlainTransport pair, and allocate 2 RTP UDP +// ports — so this is CPU / disk / file-descriptor / port heavy. The true ceiling +// is the host, not this client: sample the server externally (see README). +// +// Protocol (verified against backend/src/utils/socket.util.ts): +// - auth: JWT cookie `accessToken` via extraHeaders +// - joinRoom(roomId, ack) -> { success, data:{...} } (must join before recording) +// - start-recording(roomId) -> emit with roomId as the single payload arg, NO ack; +// server replies with a `recording-started {recordingId}` +// event to this socket on success. On failure the server +// only logs — the client just never hears back (timeout). +// - stop-recording(roomId, ack) -> server stops FFmpeg / closes transports / releases +// ports, then calls ack(). No active recording or an +// error means ack never fires (timeout). +// +// Authored by Claude (Anthropic), via Claude Code — 2026-08-27. + +const { io } = require("socket.io-client"); + +// ---------- CLI args ---------- +function arg(name, def) { + const i = process.argv.indexOf(`--${name}`); + if (i === -1) return def; + return process.argv[i + 1]; +} + +const URL = arg("url", "http://localhost:3000"); +const ROOM_ID = arg("room", null); +const TOKEN = arg("token", null); +const NUM_RECORDERS = parseInt(arg("recorders", "20"), 10); +const RAMP_MS = parseInt(arg("rampMs", "10000"), 10); +const RECORD_MS = parseInt(arg("recordMs", "30000"), 10); +const ACK_TIMEOUT_MS = parseInt(arg("timeout", "15000"), 10); + +if (!TOKEN) { + console.error( + "Missing --token . The server's JWT middleware rejects unauthenticated sockets." + ); + process.exit(1); +} + +if (!ROOM_ID) { + console.error( + "Missing --room . Pass a LIVE room that has a broadcaster producing audio+video — " + + "recordings consume the room's producers, so an empty room records nothing." + ); + process.exit(1); +} + +// ---------- percentile ---------- +function percentile(sortedArr, p) { + if (sortedArr.length === 0) return NaN; + + const idx = Math.ceil((p / 100) * sortedArr.length) - 1; + + return sortedArr[ + Math.min(Math.max(idx, 0), sortedArr.length - 1) + ]; +} + +function summarize(label, samples) { + const clean = samples + .filter((n) => Number.isFinite(n)) + .sort((a, b) => a - b); + + if (clean.length === 0) { + console.log(`${label}: no samples`); + return; + } + + const avg = + clean.reduce((a, b) => a + b, 0) / clean.length; + + console.log( + `${label.padEnd(32)} ` + + `n=${clean.length.toString().padEnd(5)} ` + + `avg=${avg.toFixed(1)}ms ` + + `p50=${percentile(clean, 50)}ms ` + + `p90=${percentile(clean, 90)}ms ` + + `p95=${percentile(clean, 95)}ms ` + + `p99=${percentile(clean, 99)}ms ` + + `max=${clean[clean.length - 1]}ms` + ); +} + +// ---------- Socket.IO ACK helper ---------- +function ackWithTimeout(socket, event, ...args) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`${event} ack timeout`)); + }, ACK_TIMEOUT_MS); + + const t0 = performance.now(); + + socket.emit(event, ...args, (response) => { + clearTimeout(timer); + + resolve({ + response, + latencyMs: performance.now() - t0, + }); + }); + }); +} + +// ---------- start-recording helper ---------- +// start-recording has NO ack: we emit the roomId and wait for the server to emit +// a `recording-started` event back to this socket. The gap between the emit and +// that event is the "start latency" — the cost of spawning FFmpeg + building the +// PlainTransport pair + allocating RTP ports under whatever load already exists. +function startRecordingAndWait(socket) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + socket.off("recording-started", onStarted); + reject( + new Error( + "recording-started timeout (server likely saturated: CPU/disk/FD/ports)" + ) + ); + }, ACK_TIMEOUT_MS); + + const t0 = performance.now(); + + const onStarted = (payload) => { + clearTimeout(timer); + resolve({ + latencyMs: performance.now() - t0, + recordingId: payload?.recordingId, + }); + }; + + socket.once("recording-started", onStarted); + socket.emit("start-recording", ROOM_ID); + }); +} + +const results = { + connectMs: [], + joinRoomMs: [], + startRecordingMs: [], + stopAckMs: [], + errors: [], +}; + +// ---------- per-recorder ---------- +async function runRecorder(idx) { + const socket = io(URL, { + transports: ["websocket"], + reconnection: false, + forceNew: true, + extraHeaders: { + cookie: `accessToken=${TOKEN}`, + }, + }); + + try { + // 1. CONNECT + const connectStart = performance.now(); + + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error("connect timeout")); + }, ACK_TIMEOUT_MS); + + socket.once("connect", () => { + clearTimeout(timer); + resolve(); + }); + + socket.once("connect_error", (err) => { + clearTimeout(timer); + reject(err); + }); + }); + + results.connectMs.push(performance.now() - connectStart); + + // 2. JOIN ROOM (must be a viewer before recording) + const { + response: joinRes, + latencyMs: joinLatency, + } = await ackWithTimeout(socket, "joinRoom", ROOM_ID); + + results.joinRoomMs.push(joinLatency); + + if (!joinRes?.success) { + throw new Error( + `joinRoom failed: ${joinRes?.code || "unknown error"}` + ); + } + + // 3. START RECORDING (spawns FFmpeg + PlainTransport pair + 2 RTP ports) + const { latencyMs: startLatency } = + await startRecordingAndWait(socket); + + results.startRecordingMs.push(startLatency); + + // 4. HOLD the recording open so concurrent FFmpeg load overlaps + await new Promise((resolve) => + setTimeout(resolve, RECORD_MS) + ); + + // 5. STOP RECORDING (stops FFmpeg, closes transports/consumers, frees ports) + const { latencyMs: stopLatency } = await ackWithTimeout( + socket, + "stop-recording", + ROOM_ID + ); + + results.stopAckMs.push(stopLatency); + } catch (err) { + results.errors.push( + `recorder ${idx}: ${err?.message || String(err)}` + ); + } finally { + socket.disconnect(); + } +} + +// ---------- main ---------- +async function main() { + console.log( + `Starting recording-concurrency load test: ` + + `${NUM_RECORDERS} recorders, ` + + `ramped over ${RAMP_MS}ms, ` + + `hold ${RECORD_MS}ms, ` + + `room=${ROOM_ID}` + ); + console.log( + `Each recording = 1 FFmpeg process + 1 PlainTransport pair + 2 RTP UDP ports on the server.` + ); + + const delayBetween = + NUM_RECORDERS > 0 ? RAMP_MS / NUM_RECORDERS : 0; + + const runs = []; + + for (let i = 0; i < NUM_RECORDERS; i++) { + runs.push(runRecorder(i)); + + if (delayBetween > 0) { + await new Promise((resolve) => + setTimeout(resolve, delayBetween) + ); + } + } + + await Promise.allSettled(runs); + + console.log("\n=== Results ==="); + + console.log( + `start-recording success: ${results.startRecordingMs.length}/${NUM_RECORDERS}` + ); + console.log( + `stop-recording ack success: ${results.stopAckMs.length}/${NUM_RECORDERS}` + ); + + summarize("socket connect", results.connectMs); + summarize("joinRoom ack", results.joinRoomMs); + summarize("start-recording latency", results.startRecordingMs); + summarize("stop-recording ack", results.stopAckMs); + + const timeouts = results.errors.filter((e) => + e.includes("timeout") + ).length; + + console.log( + `\nFailures: ${results.errors.length}/${NUM_RECORDERS} ` + + `(of which timeouts: ${timeouts})` + ); + + if (results.errors.length > 0) { + console.log(results.errors.slice(0, 10).join("\n")); + + if (results.errors.length > 10) { + console.log(`...and ${results.errors.length - 10} more`); + } + } + + console.log( + "\nNOTE: these are client-observed numbers only. The real recording ceiling is " + + "server-side — FFmpeg CPU, disk write throughput, open file descriptors, and RTP " + + "UDP port exhaustion. Start latency climbing and start/stop timeouts appearing are " + + "the client-visible symptoms; sample the server externally to find the true limit " + + "(see RECORDING-CONCURRENCY.md)." + ); +} + +main(); diff --git a/load-test/results/RECORDING-CONCURRENCY.md b/load-test/results/RECORDING-CONCURRENCY.md new file mode 100644 index 0000000..adaa6da --- /dev/null +++ b/load-test/results/RECORDING-CONCURRENCY.md @@ -0,0 +1,142 @@ +# Recording Concurrency Load Test + +> Authored by Claude (Anthropic), via Claude Code — 2026-08-27. + +Roadmap gap: **concurrent server-side recording load is untested.** CrowdStream can +record a room server-side, but every recording spawns an FFmpeg process — an expensive, +CPU/disk/FD/port-heavy operation. Nothing in the load-test suite has yet answered "how +many simultaneous recordings can one server node sustain before it falls over?" This test +fills that gap. + +## What it tests + +Ramps `--recorders` sockets, each of which joins a single LIVE room and asks the server to +record it. The point is to overlap many recordings at once and watch when the server stops +keeping up. + +Per concurrent recording, the server pays (verified against `backend/src/utils/socket.util.ts`): + +- **1 FFmpeg process** — spawned to mux the consumed RTP into an MP4 on disk (CPU + disk write). +- **1 mediasoup PlainTransport pair** — one audio, one video, each consuming the room's producers. +- **2 RTP UDP ports** — allocated from the server's port range (one per transport). + +So N concurrent recordings ≈ N FFmpeg processes + N transport pairs + 2N UDP ports, all +writing MP4s to disk at once. That is the load this test generates. + +The client measures what it can see: + +- **start-recording latency** — from emitting `start-recording` to receiving the + `recording-started {recordingId}` event (i.e. the time to spawn FFmpeg + build the + transport pair + allocate ports). +- **stop-recording ack** — time for the server to stop FFmpeg, close transports/consumers, + release ports, and `ack()`. +- **start/stop success counts and failures/timeouts.** + +## Prerequisites + +- A **LIVE room with an active broadcaster producing audio and video.** Recordings consume + the room's producers; an empty room produces empty/zero-byte files and is not a meaningful test. +- A valid JWT for the `accessToken` cookie (the socket auth middleware rejects unauthenticated + connections). +- `socket.io-client` installed (already present in `load-test/node_modules`). +- Enough **free disk** in the server's recording output directory to hold `--recorders` + simultaneous MP4s for the full `--recordMs` hold (see caveats). + +## How to run + +```bash +node recording-concurrency.js \ + --url http://localhost:3000 \ + --token "$ACCESS_TOKEN" \ + --room \ + --recorders 20 \ + --rampMs 10000 \ + --recordMs 30000 \ + --timeout 15000 +``` + +Never hardcode the token — pass it via an environment variable as shown. + +### Arguments + +| Arg | Default | Description | +| ------------- | ----------------------- | ---------------------------------------------------------------- | +| `--url` | `http://localhost:3000` | Server base URL. | +| `--token` | _(required)_ | JWT for the `accessToken` cookie. Exits 1 if missing. | +| `--room` | _(required)_ | LIVE room id with a broadcaster producing. Exits 1 if missing. | +| `--recorders` | `20` | Number of concurrent recordings to ramp. | +| `--rampMs` | `10000` | Window over which recorders are started (spread evenly). | +| `--recordMs` | `30000` | How long each recorder holds its recording open before stopping. | +| `--timeout` | `15000` | Per-operation timeout (connect, joinRoom ack, recording-started, stop ack). | + +## Server-side saturation sampling + +The client numbers only tell you when the server _stopped responding_. To find the real +ceiling and see _why_, sample the server host while the test holds its recordings open. + +**FFmpeg CPU (per-process and aggregate):** + +```bash +pidstat -C ffmpeg 2 5 # per-ffmpeg %CPU, sampled every 2s +top -b -n1 | grep -c ffmpeg # count of live ffmpeg processes +top -b -n1 | grep ffmpeg # their individual CPU/MEM +``` + +**Disk (fill + write throughput):** + +```bash +df -h /path/to/recording/output # watch free space shrink as MP4s grow +iostat -x 2 5 # %util and w_await on the recording disk +``` + +**File descriptors held by the Node server** (transports, consumers, sockets, MP4 writers): + +```bash +ls /proc//fd | wc -l # total open FDs; compare against `ulimit -n` +``` + +**RTP UDP port-range usage** (2 ports per recording — watch for exhaustion): + +```bash +ss -u -a -n | wc -l # total UDP sockets +ss -u -a -n | grep -c ':4[0-9]{4}' # count within your RTP port range (adjust regex) +``` + +Sample all of these once per second (or via `watch -n1`) across the ramp + hold window. + +## Interpretation + +At the ceiling the client-visible symptoms are: + +- **start-recording latency climbs** — FFmpeg spawns and transport/port allocation queue + behind a CPU/disk-bound server; p90/p95/p99 pull away from p50. +- **start/stop timeouts appear** — `recording-started` never arrives, or `stop-recording` + never acks (the server errors out server-side and only logs). `Failures` and the timeout + subcount rise. + +Cross-reference the moment failures begin with the server samples above: if ffmpeg CPU is +pinned, you are CPU-bound; if `iostat` `%util` is ~100%, you are disk-bound; if FD count +approaches `ulimit -n` or the RTP port range is exhausted, you have hit a resource cap. +The saturation point is the highest `--recorders` at which start latency stays flat and +failures stay at zero. + +## Caveats + +- **Disk fills during long holds.** Every recorder writes a growing MP4 for the entire + `--recordMs`. Large `--recorders` × long `--recordMs` can exhaust the recording disk and + produce disk-full failures that look like server saturation but are just capacity — make + sure the recording output directory has ample free space before a long run, and clean up + MP4s between runs. +- Client-observed latency includes network + event-loop scheduling on the load generator; + run the generator off-box from the server if the client machine is itself a bottleneck. +- Failed `start-recording` attempts emit no client-facing error event — the server only logs + server-side — so a failure surfaces here purely as a `recording-started` timeout. Check + server logs to distinguish causes. + +## Results + +_Pending execution — not fabricated._ + +| recorders | start success | stop success | start p50 (ms) | start p95 (ms) | stop p50 (ms) | failures | server bottleneck | +| --------- | ------------- | ------------ | -------------- | -------------- | ------------- | -------- | ----------------- | +| _TBD_ | _TBD_ | _TBD_ | _TBD_ | _TBD_ | _TBD_ | _TBD_ | _TBD_ |