From a68cb9d881c761ff7117c5d5326a22ede59ec775 Mon Sep 17 00:00:00 2001 From: Harshit Date: Thu, 27 Aug 2026 12:55:54 +0530 Subject: [PATCH] test(mongo): add join-churn write-load test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit socket.io-client test that drives high join/leave churn — each join triggers a Viewer insert + a LiveRoom \$inc — to measure how join-ack latency degrades under MongoDB write pressure. - load-test/mongo-write-load.js — worker-pool join/leave churn; join-ack percentiles + joins/sec - load-test/results/MONGO-WRITE-LOAD.md — writes triggered, mongostat/serverStatus sampling Authored by Claude (Anthropic) via Claude Code. Not executed against a live backend — results are placeholders, not fabricated. Co-Authored-By: Claude Opus 4.8 (1M context) --- load-test/mongo-write-load.js | 299 ++++++++++++++++++++++++++ load-test/results/MONGO-WRITE-LOAD.md | 179 +++++++++++++++ 2 files changed, 478 insertions(+) create mode 100644 load-test/mongo-write-load.js create mode 100644 load-test/results/MONGO-WRITE-LOAD.md diff --git a/load-test/mongo-write-load.js b/load-test/mongo-write-load.js new file mode 100644 index 0000000..27a1adc --- /dev/null +++ b/load-test/mongo-write-load.js @@ -0,0 +1,299 @@ +// mongo-write-load.js +// +// CrowdStream MongoDB write-load test. +// +// Drives high join/leave churn against a live room to stress MongoDB writes and +// measure how joinRoom ack latency degrades under sustained write pressure. +// +// WHY joins create write pressure (see backend registerViewer.handler.ts): +// Each successful joinRoom triggers ~2 Mongo writes: +// 1. Viewer.create({...}) -> insert a viewer doc +// 2. LiveRoom.updateOne({experienceRoomId}, {$inc: -> increment counters +// {totalViewersJoined: 1}}) +// Viewer-session lifecycle + peak-viewer bookkeeping are also persisted. +// So high connect/join/disconnect churn = sustained write pressure. +// +// This is a load generator only. Sample Mongo concurrently (see the printed +// reminder / MONGO-WRITE-LOAD.md) to correlate ack latency with DB pressure. +// +// Authored by Claude (Anthropic), via Claude Code -- 2026-08-27. + +const { io } = require("socket.io-client"); + +// ---------- CLI args ---------- +function arg(name, def) { + const i = process.argv.indexOf(`--${name}`); + if (i === -1) return def; + return process.argv[i + 1]; +} + +const URL = arg("url", "http://localhost:3000"); +const TOKEN = arg("token", null); +const ROOM_ID = arg("room", null); +const CYCLES = parseInt(arg("cycles", "2000"), 10); +const CONCURRENCY = parseInt(arg("concurrency", "50"), 10); +const HOLD_MS = parseInt(arg("holdMs", "0"), 10); +const RAMP_MS = parseInt(arg("rampMs", "0"), 10); +const ACK_TIMEOUT_MS = parseInt(arg("timeout", "8000"), 10); + +// ---------- arg validation ---------- +if (!TOKEN) { + console.error( + "Missing --token . The server's JWT middleware rejects unauthenticated " + + "sockets (auth cookie accessToken via extraHeaders). NEVER hardcode this." + ); + process.exit(1); +} + +if (!ROOM_ID) { + console.error( + "Missing --room . Start a broadcast to create a LIVE room, then pass " + + "its id so joinRoom resolves (otherwise every join returns ROOM_NOT_FOUND)." + ); + process.exit(1); +} + +if (!Number.isFinite(CYCLES) || CYCLES <= 0) { + console.error("Invalid --cycles: must be a positive integer."); + process.exit(1); +} + +if (!Number.isFinite(CONCURRENCY) || CONCURRENCY <= 0) { + console.error("Invalid --concurrency: must be a positive integer."); + process.exit(1); +} + +// ---------- percentile ---------- +function percentile(sortedArr, p) { + if (sortedArr.length === 0) return NaN; + + const idx = Math.ceil((p / 100) * sortedArr.length) - 1; + + return sortedArr[ + Math.min(Math.max(idx, 0), sortedArr.length - 1) + ]; +} + +function summarize(label, samples) { + const clean = samples + .filter((n) => Number.isFinite(n)) + .sort((a, b) => a - b); + + if (clean.length === 0) { + console.log(`${label}: no samples`); + return; + } + + const avg = + clean.reduce((a, b) => a + b, 0) / clean.length; + + console.log( + `${label.padEnd(32)} ` + + `n=${clean.length.toString().padEnd(5)} ` + + `avg=${avg.toFixed(1)}ms ` + + `p50=${percentile(clean, 50).toFixed(1)}ms ` + + `p90=${percentile(clean, 90).toFixed(1)}ms ` + + `p95=${percentile(clean, 95).toFixed(1)}ms ` + + `p99=${percentile(clean, 99).toFixed(1)}ms ` + + `max=${clean[clean.length - 1].toFixed(1)}ms` + ); +} + +// ---------- Socket.IO ACK helper ---------- +function ackWithTimeout(socket, event, ...args) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`${event} ack timeout`)); + }, ACK_TIMEOUT_MS); + + const t0 = performance.now(); + + socket.emit(event, ...args, (response) => { + clearTimeout(timer); + + resolve({ + response, + latencyMs: performance.now() - t0, + }); + }); + }); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +// ---------- shared state ---------- +const results = { + joinRoomMs: [], + joinSuccess: 0, + joinFail: 0, + connectErrors: 0, + cyclesCompleted: 0, + errors: [], +}; + +let nextCycle = 0; // shared cursor consumed by the worker pool + +// ---------- one join/leave cycle ---------- +// connect -> joinRoom (measure ack latency) -> optional hold -> disconnect +async function runCycle(cycleIdx) { + const socket = io(URL, { + transports: ["websocket"], + reconnection: false, + forceNew: true, + extraHeaders: { + cookie: `accessToken=${TOKEN}`, + }, + }); + + try { + // 1. CONNECT + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error("connect timeout")); + }, ACK_TIMEOUT_MS); + + socket.once("connect", () => { + clearTimeout(timer); + resolve(); + }); + + socket.once("connect_error", (err) => { + clearTimeout(timer); + reject(err); + }); + }); + } catch (err) { + results.connectErrors++; + results.errors.push( + `cycle ${cycleIdx}: connect: ${err?.message || String(err)}` + ); + socket.disconnect(); + return; + } + + try { + // 2. JOIN ROOM (this is what forces the Mongo writes) + const { + response: joinRes, + latencyMs: joinLatency, + } = await ackWithTimeout(socket, "joinRoom", ROOM_ID); + + results.joinRoomMs.push(joinLatency); + + if (joinRes?.success) { + results.joinSuccess++; + } else { + results.joinFail++; + results.errors.push( + `cycle ${cycleIdx}: joinRoom failed: ${ + joinRes?.code || "unknown error" + }` + ); + } + + // 3. OPTIONAL HOLD (keep the viewer session alive before churning out) + if (HOLD_MS > 0) { + await sleep(HOLD_MS); + } + } catch (err) { + results.joinFail++; + results.errors.push( + `cycle ${cycleIdx}: joinRoom: ${err?.message || String(err)}` + ); + } finally { + // 4. DISCONNECT (leave) -> next cycle + socket.disconnect(); + } +} + +// ---------- worker pool ---------- +// CONCURRENCY workers each pull the next cycle index off the shared cursor +// until CYCLES total join/leave cycles are done (same pattern as a soak). +async function worker(workerIdx) { + if (RAMP_MS > 0) { + // Stagger worker startup across the ramp window (0 => start immediately). + await sleep((RAMP_MS / CONCURRENCY) * workerIdx); + } + + for (;;) { + const idx = nextCycle++; + if (idx >= CYCLES) break; + + await runCycle(idx); + results.cyclesCompleted++; + } +} + +// ---------- main ---------- +async function main() { + console.log( + `Starting Mongo write-load test: ` + + `${CYCLES} join/leave cycles, ` + + `concurrency=${CONCURRENCY}, ` + + `holdMs=${HOLD_MS}, ` + + `rampMs=${RAMP_MS}, ` + + `room=${ROOM_ID}, ` + + `url=${URL}` + ); + + console.log( + "\n>>> REMINDER: sample MongoDB CONCURRENTLY while this runs so you can\n" + + ">>> correlate join-ack latency with DB write pressure. In another shell:\n" + + ">>> mongostat --rowcount 0 # watch insert / update / dirty %\n" + + ">>> mongosh> db.serverStatus().opcounters # insert & update deltas\n" + + ">>> mongosh> db.currentOp({ active: true }) # in-flight write ops\n" + + ">>> Each join = Viewer.create (insert) + LiveRoom $inc (update). See README.\n" + ); + + const wallStart = performance.now(); + + const workers = []; + for (let w = 0; w < CONCURRENCY; w++) { + workers.push(worker(w)); + } + + await Promise.allSettled(workers); + + const elapsedMs = performance.now() - wallStart; + const elapsedSec = elapsedMs / 1000; + + console.log("\n=== Results ==="); + + summarize("joinRoom ack", results.joinRoomMs); + + console.log( + `\ncycles completed: ${results.cyclesCompleted}/${CYCLES}` + ); + console.log(`join successes: ${results.joinSuccess}`); + console.log(`join failures: ${results.joinFail}`); + console.log(`connect errors: ${results.connectErrors}`); + console.log( + `wall time: ${elapsedSec.toFixed(2)}s` + ); + + // Throughput of successful joins == sustained Mongo write pressure driven. + const joinsPerSec = + elapsedSec > 0 ? results.joinSuccess / elapsedSec : 0; + + console.log( + `throughput: ${joinsPerSec.toFixed(1)} joins/sec ` + + `(~${(joinsPerSec * 2).toFixed(1)} Mongo writes/sec: insert + $inc)` + ); + + if (results.errors.length > 0) { + console.log( + `\nSample errors (${results.errors.length} total):` + ); + console.log(results.errors.slice(0, 10).join("\n")); + + if (results.errors.length > 10) { + console.log( + `...and ${results.errors.length - 10} more` + ); + } + } +} + +main(); diff --git a/load-test/results/MONGO-WRITE-LOAD.md b/load-test/results/MONGO-WRITE-LOAD.md new file mode 100644 index 0000000..f9638c6 --- /dev/null +++ b/load-test/results/MONGO-WRITE-LOAD.md @@ -0,0 +1,179 @@ +# Mongo Write-Load Test + +> Authored by Claude (Anthropic), via Claude Code — 2026-08-27. + +**Roadmap gap this closes:** MongoDB write throughput under viewer join churn is +currently **untested**. The signaling and SFU-capacity tests exercise mediasoup and +socket signaling, but nothing has yet driven the *database* to its write ceiling. +This test does exactly that and measures how join-ack latency degrades under sustained +write pressure. + +--- + +## What it tests + +Every successful `joinRoom` on the server (`backend/src/handlers/registerViewer.handler.ts`) +fans out into MongoDB writes. This test drives thousands of connect → join → disconnect +cycles to keep those writes saturated and watch what happens to ack latency. + +Writes triggered **per join**: + +| Write | Collection | Operation | Trigger | +|-------|-----------|-----------|---------| +| **Viewer doc** | `viewers` | `Viewer.create({...})` — insert | one insert per join (roomId, viewerId, socketId, ipHash, userAgentHash) | +| **LiveRoom `$inc` counter** | `liveRooms` | `LiveRoom.updateOne({ experienceRoomId }, { $inc: { totalViewersJoined: 1 } })` | one indexed update per join | +| **viewer_sessions / peak-viewer bookkeeping** | viewer-session lifecycle | insert/update on join + leave | the viewer-session lifecycle and peak-viewer counters are also persisted as viewers join and churn out | + +So **each join ≈ 2 confirmed Mongo writes (one insert + one `$inc` update)**, plus +session-lifecycle persistence on join/leave. High connect/join/disconnect churn therefore +produces sustained, mostly write-heavy DB load — which is what this generator is for. + +The client measures **`joinRoom` ack latency** (the round trip the real viewer feels) and +reports its distribution alongside join throughput. Because throughput of successful joins +maps directly onto insert+update volume, joins/sec is a proxy for write pressure. + +--- + +## Prerequisites + +- Node 18+ (uses the global `performance` API and `socket.io-client`). +- `socket.io-client` installed (the sibling `load-test/` folder already has it; run from + there or `npm i socket.io-client`). +- The CrowdStream backend running and reachable at `--url`. +- A **LIVE** room id: start a broadcast first, then pass its id as `--room`. Joining a + non-live / unknown room returns `{ success: false, code: 'ROOM_NOT_FOUND' }` and drives + no viewer inserts. +- A valid JWT for the `accessToken` cookie, passed via `--token`. **Never hardcode the + token** — export it from your environment and pass it on the command line. +- Access to the Mongo instance (a `mongosh` shell and/or `mongostat`) so you can sample the + server **while the test runs** (see below). + +--- + +## How to run + +```bash +# from a directory where socket.io-client resolves (e.g. the load-test/ folder) +node /tmp/cs-tests/mongo-write-load/mongo-write-load.js \ + --url http://localhost:3000 \ + --token "$ACCESS_TOKEN" \ + --room \ + --cycles 2000 \ + --concurrency 50 \ + --holdMs 0 \ + --rampMs 0 +``` + +### Arguments + +| Arg | Default | Meaning | +|-----|---------|---------| +| `--url` | `http://localhost:3000` | Backend origin (Socket.IO endpoint). | +| `--token` | *(required)* | JWT for the `accessToken` cookie, sent via `extraHeaders`. Exit(1) if missing. | +| `--room` | *(required)* | Id of a **live** room to join. Exit(1) if missing. | +| `--cycles` | `2000` | Total join/leave cycles to run across the whole test. | +| `--concurrency` | `50` | Worker-pool size = number of simultaneous connect/join/disconnect cycles in flight. | +| `--holdMs` | `0` | Hold the viewer session open this long after a successful join before disconnecting. `0` = maximum churn. | +| `--rampMs` | `0` | Stagger worker startup across this window. `0` = start all workers immediately. | +| `--timeout` | `8000` | Per-connect and per-ack timeout in ms. | + +The `--concurrency` workers each pull the next cycle off a shared cursor until `--cycles` +total cycles are done (a soak-style worker pool), so raising concurrency raises the +instantaneous write pressure while `--cycles` bounds the total work. + +--- + +## Mongo-side sampling (do this WHILE it runs) + +The client alone only shows you the *symptom* (ack latency). To find the *cause* you must +watch Mongo at the same time. Open a second terminal before you start the run. + +**1. `mongostat` — live write rates and contention** + +```bash +mongostat --rowcount 0 # or: mongostat -u -p --authenticationDatabase admin +``` +Watch the `insert`, `update`, `dirty`, `used` (WiredTiger cache), and `qrw` (queued +read/write) columns. Rising `qrw`/`dirty` while `insert`+`update` stop climbing = the write +path is the bottleneck. + +**2. `opcounters` deltas — insert vs update volume** + +```javascript +// in mongosh, sample twice ~10s apart and diff: +db.serverStatus().opcounters // { insert, query, update, delete, ... } +``` +Each join should add ~1 insert (`Viewer.create`) and ~1 update (`LiveRoom $inc`). Confirm +the deltas track your reported joins/sec. + +**3. `db.currentOp()` — in-flight write ops and waits** + +```javascript +db.currentOp({ active: true }) +// or focus on slow/waiting ops: +db.currentOp({ active: true, secs_running: { $gte: 1 } }) +``` +Look for ops parked on `WriteConflict`, collection/global locks, or long `secs_running`. + +**4. Write latency / lock %** + +```javascript +db.serverStatus().wiredTiger.concurrentTransactions // write ticket availability +db.serverStatus().globalLock // currentQueue.writers +db.viewers.stats().wiredTiger["block-manager"] // I/O pressure on the hot collection +``` +Write tickets pinned at 0 available, or a growing `globalLock.currentQueue.writers`, both +mean writes are queuing. + +**5. Index check on the queried fields** + +The `LiveRoom` `$inc` filters on `experienceRoomId`, and viewer inserts/queries key on +`roomId`. Verify supporting indexes actually exist in your deployment: + +```javascript +db.liveRooms.getIndexes() // expect an index whose prefix is experienceRoomId +db.viewers.getIndexes() // expect indexes prefixed by roomId +``` +The schemas declare `{ experienceRoomId: 1, status: 1 }` on `LiveRoom` (so `experienceRoomId` +is a usable index prefix) and `{ roomId: 1, viewerId: 1 }` / `{ roomId: 1, joinedAt: -1 }` on +`Viewer`. Confirm these are present — a missing index turns the per-join `$inc` update into a +collection scan and will dominate latency. + +--- + +## Interpretation + +- **`joinRoom` p99 climbing while `opcounters` insert/update deltas plateau** ⇒ you have hit + the Mongo **write ceiling**: more offered load is queuing, not landing. Corroborate with + rising `qrw`/`globalLock.currentQueue.writers` and shrinking write tickets. First thing to + verify: the indexes above actually exist (an unindexed `$inc` filter is the usual culprit). +- **Latency flat while opcounters scale linearly with joins/sec** ⇒ Mongo is keeping up at + this level; push `--concurrency` (and/or `--holdMs 0`) higher to find the real ceiling. +- **Many `connect errors` / `join failures` before Mongo saturates** ⇒ the bottleneck is + upstream (socket accept, JWT middleware, mediasoup room lookup), not the database. +- Compare `joins/sec` here against the join-latency numbers from `signaling-latency.js` at + the same concurrency to separate DB-write cost from the rest of the join path. + +--- + +## Caveats + +- **The join rate limiter is commented out server-side.** In + `registerViewer.handler.ts` the per-user and per-IP `rateLimiter(...)` checks are disabled + (commented out, per the signaling notes), so joins are **not throttled** — this test can + drive the real write ceiling, but it also means production behaviour with the limiter + enabled will differ. +- This is a load generator, not a benchmark harness: numbers depend on your hardware, Mongo + deployment (standalone vs replica set), network, and whether other traffic is present. +- `--holdMs 0` maximises churn (fastest write turnover) but also maximises connect/disconnect + overhead; use a non-zero `--holdMs` to model realistic viewer dwell time. + +--- + +## Results + +_Pending execution — not fabricated._ + +| Run | cycles | concurrency | holdMs | join p50 | p90 | p95 | p99 | max | joins/sec | join fails | connect errors | Mongo notes (mongostat/opcounters) | +|-----|--------|-------------|--------|----------|-----|-----|-----|-----|-----------|-----------|----------------|------------------------------------| +| _tbd_ | | | | | | | | | | | | |