diff --git a/load-test/reconnection-storm.js b/load-test/reconnection-storm.js new file mode 100644 index 0000000..fa29c6c --- /dev/null +++ b/load-test/reconnection-storm.js @@ -0,0 +1,395 @@ +/** + * CrowdStream — Reconnection-Storm (Thundering-Herd) Load Test + * ============================================================ + * Measures RECOVERY (not cold connect) when a whole signaling pod's clients + * reconnect at once. socket.io auto-reconnection is ENABLED, so the client-side + * backoff + jitter is exercised exactly as it would be in production behind + * sticky sessions + the Socket.IO Redis adapter. + * + * A prior 10k/2s spike test saw 8,139 failures — but that only measured cold + * connect. This test quantifies how the surviving clients recover after a mass + * drop, i.e. the thundering-herd re-join. + * + * Two trigger modes: + * forcedrop (default) — the harness drops every connected client itself + * (io.engine.close ⇒ socket.io auto-reconnect); no infra + * access required. + * killpod — you kill a real signaling pod by hand; the harness does + * NOT drop clients and relies on socket.io's own disconnect + * detection. + * + * Recovery time per client = (successful re-join ack) − (storm trigger). + * + * Never hardcodes secrets: the JWT is passed via --token and sent as the + * `accessToken` cookie, matching CrowdStream's socket auth middleware. + * + * > Authored by Claude (Anthropic), via Claude Code — 2026-08-27. + */ + +const { io } = require("socket.io-client"); + +// ---------- CLI args ---------- +function arg(name, def) { + const i = process.argv.indexOf(`--${name}`); + if (i === -1) return def; + return process.argv[i + 1]; +} + +const URL = arg("url", "http://localhost:3000"); +const URLS_ARG = arg("urls", null); +const URLS = URLS_ARG + ? URLS_ARG.split(",").map((s) => s.trim()).filter(Boolean) + : [URL]; + +const ROOM_ID = arg("room", null); +const TOKEN = arg("token", null); +const NUM_CLIENTS = parseInt(arg("clients", "500"), 10); +const RAMP_MS = parseInt(arg("rampMs", "10000"), 10); +const MODE = arg("mode", "forcedrop"); +const TRIGGER_AFTER_MS = parseInt(arg("triggerAfterMs", "15000"), 10); +const RECOVER_WINDOW_MS = parseInt(arg("recoverWindowMs", "30000"), 10); +const ACK_TIMEOUT_MS = parseInt(arg("timeout", "8000"), 10); + +if (!ROOM_ID) { + console.error( + "Missing --room . Start a broadcast first, then pass its id." + ); + process.exit(1); +} + +if (!TOKEN) { + console.error( + "Missing --token . The socket auth middleware rejects unauthenticated sockets." + ); + process.exit(1); +} + +if (MODE !== "forcedrop" && MODE !== "killpod") { + console.error(`Invalid --mode "${MODE}". Use "forcedrop" or "killpod".`); + process.exit(1); +} + +// ---------- percentile ---------- (identical to signaling-latency.js) +function percentile(sortedArr, p) { + if (sortedArr.length === 0) return NaN; + + const idx = Math.ceil((p / 100) * sortedArr.length) - 1; + + return sortedArr[Math.min(Math.max(idx, 0), sortedArr.length - 1)]; +} + +function summarize(label, samples) { + const clean = samples + .filter((n) => Number.isFinite(n)) + .sort((a, b) => a - b); + + if (clean.length === 0) { + console.log(`${label}: no samples`); + return; + } + + const avg = clean.reduce((a, b) => a + b, 0) / clean.length; + + console.log( + `${label.padEnd(32)} ` + + `n=${clean.length.toString().padEnd(5)} ` + + `avg=${avg.toFixed(1)}ms ` + + `p50=${percentile(clean, 50)}ms ` + + `p90=${percentile(clean, 90)}ms ` + + `p95=${percentile(clean, 95)}ms ` + + `p99=${percentile(clean, 99)}ms ` + + `max=${clean[clean.length - 1]}ms` + ); +} + +// ---------- Socket.IO ACK helper ---------- (same shape as signaling-latency.js) +function ackWithTimeout(socket, event, ...args) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`${event} ack timeout`)); + }, ACK_TIMEOUT_MS); + + const t0 = performance.now(); + + socket.emit(event, ...args, (response) => { + clearTimeout(timer); + + resolve({ + response, + latencyMs: performance.now() - t0, + }); + }); + }); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +// ---------- shared state ---------- +const clients = []; +let stormTriggered = false; +let triggerAt = 0; +const recoverSamples = []; // recovery ms for clients that re-joined within the window +let connectErrors = 0; +let reconnectAttempts = 0; // manager-level auto-reconnect attempts after the trigger + +function urlFor(idx) { + return URLS[idx % URLS.length]; +} + +// ---------- per-client join ---------- +async function joinRoomOnce(client) { + try { + const { response } = await ackWithTimeout( + client.socket, + "joinRoom", + ROOM_ID + ); + + if (response?.success) return true; + + client.lastError = `joinRoom: ${response?.code || "unsuccessful ack"}`; + return false; + } catch (err) { + client.lastError = err?.message || String(err); + return false; + } +} + +async function onConnect(client) { + if (!stormTriggered) { + // Baseline join: initial connect (or a benign pre-trigger reconnect). + client.baselineJoined = await joinRoomOnce(client); + return; + } + + // Post-trigger (re)connect — this is the recovery path we are measuring. + if (client.recovered) return; + + const ok = await joinRoomOnce(client); + + if (ok && !client.recovered) { + client.recovered = true; + client.recoverMs = performance.now() - triggerAt; + + if (client.recoverMs <= RECOVER_WINDOW_MS) { + recoverSamples.push(client.recoverMs); + } + } +} + +function spawnClient(idx) { + const url = urlFor(idx); + + // reconnection ENABLED so socket.io's own backoff + jitter drives recovery. + const socket = io(url, { + transports: ["websocket"], + reconnection: true, + reconnectionAttempts: Infinity, + forceNew: true, + extraHeaders: { + cookie: `accessToken=${TOKEN}`, + }, + }); + + const client = { + idx, + url, + socket, + baselineJoined: false, + droppedAfterTrigger: false, + recovered: false, + recoverMs: NaN, + lastError: null, + }; + + clients.push(client); + + // socket.io re-emits "connect" on every successful (re)connection. + socket.on("connect", () => { + onConnect(client).catch((err) => { + client.lastError = err?.message || String(err); + }); + }); + + socket.on("disconnect", () => { + if (stormTriggered) client.droppedAfterTrigger = true; + }); + + socket.on("connect_error", () => { + connectErrors += 1; + }); + + // Manager-level signal: how many auto-reconnect attempts the herd generated. + socket.io.on("reconnect_attempt", () => { + if (stormTriggered) reconnectAttempts += 1; + }); + + return client; +} + +// ---------- storm triggers ---------- +function triggerForcedrop() { + for (const client of clients) { + // Only drop clients that were actually in steady state; a client that never + // connected was not part of the herd and should not count against recovery. + if (!client.socket.connected) continue; + + client.droppedAfterTrigger = true; + + const engine = client.socket.io && client.socket.io.engine; + + if (engine) { + // Abrupt transport close ⇒ socket.io's automatic reconnection (backoff + + // jitter) re-establishes the socket. This is the path that actually + // smooths — or fails to smooth — the reconnection herd. + engine.close(); + } else { + // Fallback if the engine is unavailable: manual bounce (immediate, no backoff). + client.socket.disconnect(); + client.socket.connect(); + } + } +} + +function triggerKillpod() { + console.log( + "\n *** killpod mode: KILL ONE SIGNALING POD NOW ***\n" + + " The harness will NOT drop clients; it relies on socket.io's own\n" + + " disconnect detection. Recovery timing starts at this instant.\n" + ); +} + +// ---------- main ---------- +async function main() { + console.log( + `Reconnection-storm test: ${NUM_CLIENTS} clients, mode=${MODE}, ` + + `ramp=${RAMP_MS}ms, triggerAfter=${TRIGGER_AFTER_MS}ms, ` + + `recoverWindow=${RECOVER_WINDOW_MS}ms, room=${ROOM_ID}\n` + + `endpoints: ${URLS.join(", ")}` + ); + + const t0 = performance.now(); + const delayBetween = NUM_CLIENTS > 0 ? RAMP_MS / NUM_CLIENTS : 0; + + for (let i = 0; i < NUM_CLIENTS; i++) { + spawnClient(i); + + if (delayBetween > 0) { + await sleep(delayBetween); + } + } + + // Hold until the scheduled trigger time (measured from launch) so the fleet + // can connect, join, and settle into steady state before the storm hits. + const elapsed = performance.now() - t0; + const waitBeforeTrigger = Math.max(0, TRIGGER_AFTER_MS - elapsed); + await sleep(waitBeforeTrigger); + + const stable = clients.filter( + (c) => c.socket.connected && c.baselineJoined + ).length; + + console.log( + `\nPre-storm stability: ${stable}/${NUM_CLIENTS} connected+joined.` + ); + + if (stable < NUM_CLIENTS) { + console.log( + " (Not all clients reached steady state — raise --triggerAfterMs/--timeout " + + "or lower --clients. Recovery % is measured only over clients that dropped.)" + ); + } + + // ---- trigger the storm ---- + stormTriggered = true; + triggerAt = performance.now(); + + if (MODE === "forcedrop") { + triggerForcedrop(); + } else { + triggerKillpod(); + } + + console.log( + `Storm triggered. Watching recovery for ${RECOVER_WINDOW_MS}ms...` + ); + await sleep(RECOVER_WINDOW_MS); + + // ---- tally ---- + const dropped = clients.filter((c) => c.droppedAfterTrigger); + const recovered = dropped.filter( + (c) => c.recovered && c.recoverMs <= RECOVER_WINDOW_MS + ); + const failed = dropped.filter( + (c) => !(c.recovered && c.recoverMs <= RECOVER_WINDOW_MS) + ); + const successPct = dropped.length + ? (recovered.length / dropped.length) * 100 + : 0; + + console.log("\n=== Reconnection-storm results ==="); + console.log( + `mode=${MODE} clients=${NUM_CLIENTS} dropped/affected=${dropped.length}` + ); + + summarize("reconnect + re-join", recoverSamples); + + console.log( + `recovery success within ${RECOVER_WINDOW_MS}ms: ` + + `${recovered.length}/${dropped.length} (${successPct.toFixed(1)}%)` + ); + console.log(`failed to recover in window: ${failed.length}`); + console.log(`auto-reconnect attempts (post-trigger): ${reconnectAttempts}`); + console.log(`connect_error events (whole run): ${connectErrors}`); + + // Data-driven note on herd smoothing — computed from real samples, never fabricated. + const clean = recoverSamples + .filter((n) => Number.isFinite(n)) + .sort((a, b) => a - b); + + if (clean.length >= 2) { + const p50 = percentile(clean, 50); + const p99 = percentile(clean, 99); + const spread = p50 > 0 ? p99 / p50 : Infinity; + + if (spread >= 3) { + console.log( + `\nHerd note: recoveries are spread out (p99/p50=${spread.toFixed(1)}x) — ` + + "backoff + jitter appears to be de-synchronising the herd." + ); + } else { + console.log( + `\nHerd note: recoveries are tightly clustered (p99/p50=${spread.toFixed(1)}x) — ` + + "clients reconnected in a near-simultaneous burst; verify " + + "reconnectionDelayMax / randomizationFactor and LB connection limits." + ); + } + } + + if (failed.length > 0) { + const samples = failed + .slice(0, 10) + .map( + (c) => + `client ${c.idx}@${c.url}: ${ + c.lastError || "no successful re-join in window" + }` + ); + + console.log("\nsample failures:"); + console.log(samples.join("\n")); + + if (failed.length > 10) { + console.log(`...and ${failed.length - 10} more`); + } + } + + // Teardown: reconnection is on (attempts=Infinity), so we must close every + // socket and exit explicitly or the process would never terminate. + for (const client of clients) client.socket.disconnect(); + process.exit(0); +} + +main(); diff --git a/load-test/results/RECONNECTION-STORM.md b/load-test/results/RECONNECTION-STORM.md new file mode 100644 index 0000000..6949167 --- /dev/null +++ b/load-test/results/RECONNECTION-STORM.md @@ -0,0 +1,91 @@ +# Reconnection-Storm (Thundering-Herd) Load Test + +> Authored by Claude (Anthropic), via Claude Code — 2026-08-27. + +**Roadmap gap this closes:** recovery after node loss / thundering-herd reconnect was **untested**. The earlier 10k-in-2s spike test only measured *cold connect* (and recorded 8,139 failures) — it said nothing about how a fleet of already-connected clients recovers when a signaling pod dies underneath them and everyone reconnects at once. + +CrowdStream runs sticky sessions + the Socket.IO Redis adapter, and clients are expected to auto-reconnect and re-join after a signaling pod dies. This test quantifies that **recovery**, not the initial connect. + +## What it tests + +- Bring `--clients` sockets up against the signaling layer with **auto-reconnection enabled** (`reconnection: true`, `reconnectionAttempts: Infinity`), and `joinRoom` each one. +- Let them settle into steady state, then **trigger a simultaneous mass drop** (the "storm"). +- On every client's post-storm `connect`/`reconnect`, re-emit `joinRoom` and record **recovery time = successful re-join − storm trigger**. +- Report reconnect+re-join percentiles, the share of clients that recovered inside `--recoverWindowMs`, failures, and whether socket.io's backoff + jitter de-synchronised the herd. + +## The two modes + +| Mode | What the harness does | When to use | +|------|-----------------------|-------------| +| `forcedrop` (default) | Drops every connected client itself via `io.engine.close()` (falling back to `disconnect()`+`connect()`), triggering socket.io's automatic reconnection. No infra access needed. | Local / CI runs, or any environment where you cannot kill a real pod. Reproducible and self-contained. Note it drops *all* clients, not just one pod's share. | +| `killpod` | Does **not** drop clients. Prints a clear "kill a pod now" instruction and relies on socket.io's own disconnect detection; only the clients on the killed pod actually drop. | Staging/prod-like clusters where you can `kubectl delete pod` a signaling replica. This is the realistic test — it exercises sticky-session failover, the Redis adapter, and the LB re-routing survivors. | + +Recovery `%` is always computed **only over clients that actually dropped**, so `killpod` (where non-victim clients stay connected) is scored fairly. + +## Prerequisites + +- `socket.io-client` v4 available on the module path (see `../../load-test/package.json`; run from a dir where `require("socket.io-client")` resolves). +- A live broadcast so a room exists — pass its id as `--room`. +- A valid JWT for the `accessToken` cookie — pass it as `--token`. **Never hardcode it.** +- For `--urls`, point at your LB VIP(s). On auto-reconnect socket.io reuses each client's own endpoint, so LB endpoints (not direct pod IPs) give realistic re-routing to healthy pods. +- For `killpod`, cluster access to kill a signaling replica during the run. + +## How to run + +```bash +# forcedrop (self-contained): 500 clients, storm at 15s, watch recovery for 30s +node reconnection-storm.js \ + --url https://signal.crowdstream.example \ + --room \ + --token "$ACCESS_TOKEN" \ + --clients 500 + +# killpod against multiple LB endpoints — kill a pod when prompted +node reconnection-storm.js \ + --urls https://lb-a.example,https://lb-b.example \ + --room \ + --token "$ACCESS_TOKEN" \ + --clients 500 \ + --mode killpod \ + --recoverWindowMs 45000 +``` + +### Args + +| Arg | Default | Meaning | +|-----|---------|---------| +| `--url` | `http://localhost:3000` | Single signaling endpoint. | +| `--urls` | *(unset)* | Comma-separated endpoints; clients are spread round-robin across them. Overrides `--url`. | +| `--token` | *(required)* | JWT sent as the `accessToken` cookie via `extraHeaders`. | +| `--room` | *(required)* | Room id to join / re-join. | +| `--clients` | `500` | Number of concurrent sockets. | +| `--rampMs` | `10000` | Initial connect ramp — spawns are paced over this window. | +| `--mode` | `forcedrop` | `forcedrop` or `killpod`. | +| `--triggerAfterMs` | `15000` | When the storm fires, measured from launch (defaults leave ~5s of steady state after the ramp). | +| `--recoverWindowMs` | `30000` | How long to watch for recovery after the trigger. | +| `--timeout` | `8000` | Per-ack timeout for `joinRoom` (ms). | + +## Interpretation + +- **Reconnect+re-join p50/p90/p99/max** — how long survivors take to be usable again. Watch **p99** and **max**: a fat tail means some users stare at a frozen stream for many seconds after a pod loss. +- **Recovery success % within window** — the headline SLO. Anything materially below 100% means clients are permanently stranded (exhausted attempts, auth rejected on re-join, or the surviving pods are saturated). +- **Herd smoothing** — the script prints a `p99/p50` spread ratio: + - **Long tail (spread ≥ 3x)** ⇒ backoff + jitter is de-synchronising reconnects; the herd is being spread over time. Good, as long as p99 stays inside your SLO. + - **Tight cluster (spread < 3x)** ⇒ everyone reconnected in one burst. If success % also dropped, the herd overwhelmed the survivors. Mitigations: enable/raise client `reconnectionDelayMax` and `randomizationFactor` (jitter), and set LB per-pod connection/accept limits so a spike is shed rather than amplified. +- **auto-reconnect attempts (post-trigger)** — high counts relative to `dropped` indicate repeated failed attempts (survivors saturated or slow to accept) before success. + +## Caveats + +- `forcedrop` drops **every** client, so it models a total outage / full rollout, not a single-pod loss. Use `killpod` to model losing one replica's share. +- `io.engine.close()` emulates a transport drop from the client side; it does **not** actually remove server-side room/producer state. Server-side cleanup and Redis-adapter fan-out behaviour are only truly exercised in `killpod`. +- On reconnect socket.io reuses the client's original endpoint URL — real LB re-routing is only observed when `--url`/`--urls` point at an LB VIP, not a pinned pod. +- Timing is client-side wall clock (`performance.now()`); it includes network RTT and the harness host's own scheduling under load. Run the harness off the cluster and with enough CPU that its event loop isn't the bottleneck. +- This measures **signaling recovery** (reconnect + `joinRoom`), not media (transport/consumer) re-establishment. A successful re-join does not by itself prove video resumed. + +## Results + +_Pending execution — not fabricated._ + +| Run (date / mode / clients / endpoints) | Dropped | Reconnect+re-join p50 | p90 | p99 | max | Success % in window | Failures | Notes | +|------------------------------------------|---------|-----------------------|-----|-----|-----|---------------------|----------|-------| +| _pending_ | | | | | | | | |