From 2f8e6417c831ec673b865dd01c96410f3d897a99 Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 10 Sep 2026 19:30:46 -0400 Subject: [PATCH 1/2] feat(browser): render every device of a URL in one job and post one result; v1.23.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A job is now one URL. When the plugin (>= 0.66.0) claims a job carrying `deviceTypes`, the worker renders each device in turn on the job's single concurrency slot — a fresh page per device, the same renderer — and posts ONE result: `{ id, url, deviceTypes, variants: [...] }` followed by the variants' encoded bodies concatenated in order, each variant declaring its own `contentLength`. That is what lets the plugin keep a URL's device variants aligned (same render pass, seconds apart, one scheduling decision) instead of the split pairs every per-device retry lane, render-now and reconcile repair produce today. - `RenderJob.variants()` fans a multi-device job out to one per-device job sharing the claim; a legacy job is its own single variant, so the renderer contract is unchanged. - `sendResult` (legacy, flat shape) and the new `sendVariantsResult` share one `postResult` with the existing retry policy, so the two cannot drift. - A variant is skipped and the result posted PARTIAL when the lease has under 30s left or the worker began draining between variants; the plugin retries the URL for the devices it did not get back. - Stats gain `jobs` (results posted) beside `completed` (renders) and `variantsSkipped`. Compatibility: a job without `deviceTypes` is rendered and posted exactly as before, so this deploys ahead of the plugin. An older renderer handed a multi-device job renders only the first device — degraded, not broken — so the render fleet rolls out first. Co-Authored-By: Claude Fable 5.1 --- package-lock.json | 2 +- packages/browser/README.md | 69 +++++- packages/browser/package.json | 2 +- packages/browser/src/RenderJob.ts | 243 ++++++++++++++------ packages/browser/src/Worker.ts | 88 ++++++- packages/browser/test/jobResult.test.ts | 186 +++++++++++++++ packages/browser/test/variantRender.test.ts | 187 +++++++++++++++ 7 files changed, 695 insertions(+), 82 deletions(-) create mode 100644 packages/browser/test/variantRender.test.ts diff --git a/package-lock.json b/package-lock.json index 9b93a59..df7a8a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8516,7 +8516,7 @@ }, "packages/browser": { "name": "@harperfast/prerender-browser", - "version": "1.21.0", + "version": "1.23.0", "license": "Apache-2.0", "dependencies": { "mqtt": "^5.10.4", diff --git a/packages/browser/README.md b/packages/browser/README.md index a8b3a71..569f306 100644 --- a/packages/browser/README.md +++ b/packages/browser/README.md @@ -58,7 +58,7 @@ Only `harper` is required; everything else has a default. | `bypass` | `{ header: x-harper-renderer-bypass, token: '' }` | Shared origin-bypass header/token (match the plugin) | | `config` | built-in defaults | Rendering config (deep-partial object _or_ JSON file path) | | `concurrency` | ~half the CPUs | Max concurrent page renders | -| `rps` | `8` | Max render starts per second | +| `rps` | `8` | Max job starts per second (a job renders every device of one URL — see the queue protocol) | | `jobClaimLimit` | `concurrency * 2` | Jobs claimed per batch | | `browserExpirationThreshold` | `200` | Pages a browser renders before being retired | | `incognitoPages` | `true` | Render each page in a fresh incognito context | @@ -69,6 +69,73 @@ Only `harper` is required; everything else has a default. | `renderer` | the default renderer | Custom renderer (see below) | | `installSignalHandlers` | `true` | Own SIGTERM/SIGINT (drain in-flight renders, then close Chrome); `false` to own the process | +## Queue protocol + +A **job is one URL.** The plugin (>= 0.66.0) claims one job per URL and names every device to +render on it: + +```jsonc +// POST /render_queue/claim → 200 [ ...jobs ] +{ + "id": "https://site.example.com/product/x", // the schedule row this job stands for — echoed back verbatim + "url": "https://site.example.com/product/x", + "deviceTypes": ["desktop", "mobile"], // every device to render, in this order + "deviceType": "desktop", // the first of them, for renderers that predate the list + "expiresAt": 1757520000000, // lease expiry (epoch ms) + "callbackOrigin": "https://harper-node:9926", + "isFromSitemap": true, +} +``` + +The worker renders the devices **in turn on the job's one concurrency slot** — each on a fresh page, +through the same `renderer` — so `concurrency` still bounds pages in flight and renders-per-slot is +unchanged; a job just holds its slot for as many renders as it has devices. (`rps` therefore paces +_job_ starts.) Every device's snapshot then goes back in **one** result, which is what lets the plugin +keep a URL's variants aligned: same render pass, seconds apart, one scheduling decision. + +```jsonc +// POST /render_queue/job_result (x-metadata-size: ) +{ + "id": "https://site.example.com/product/x", + "url": "https://site.example.com/product/x", + "deviceTypes": ["desktop", "mobile"], // what was asked + "variants": [ + // what was attempted, in order — each followed in the body by `contentLength` bytes of its + // encoded HTML (0 = no content: a redirect, a verdict, or an error) + { + "deviceType": "desktop", + "outcome": "rendered", + "statusCode": 200, + "headers": {}, + "renderTime": 8123, + "isIndexable": true, + "structuredOffers": null, + "contentLength": 41210, + }, + { + "deviceType": "mobile", + "outcome": "error", + "reason": "error", + "error": { "name": "TimeoutError", "message": "…", "phase": "settle" }, + "contentLength": 0, + }, + ], +} +``` + +A variant is **skipped and the result posted partial** when the lease has under 30s left or the +worker began draining between variants: `variants` then lists fewer devices than `deviceTypes`, the +plugin stores what rendered and retries the URL for the rest. (A result that never arrives would cost +the whole lease before anything retried.) The per-window log line reports `jobs` beside `completed` +(renders) and `variantsSkipped`. + +**Compatibility.** A job WITHOUT `deviceTypes` — an older plugin, which claims one job per device +— is rendered as before and posted in the flat legacy shape (`{ id, url, outcome, … }` plus one +body), so this version can be deployed ahead of the plugin. The reverse is degraded, not broken: a +renderer older than this one, handed a multi-device job, renders only `deviceType` (the first) and +posts it flat; the plugin stores that one device and the others go unrendered until the fleet is +upgraded — so **roll the render fleet out first.** + ## Rendering config The `config` option (object or JSON-file path) is **deep-merged over the built-in defaults**, so only diff --git a/packages/browser/package.json b/packages/browser/package.json index f4d667d..cdaee7a 100644 --- a/packages/browser/package.json +++ b/packages/browser/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender-browser", - "version": "1.21.0", + "version": "1.23.0", "type": "module", "description": "Headless-browser render library for Harper Prerender: claims render jobs from the @harperfast/prerender queue, renders pages in headless Chrome (Puppeteer), and posts the HTML back. Embedded by a render service and configured entirely via startWorker() options.", "keywords": [ diff --git a/packages/browser/src/RenderJob.ts b/packages/browser/src/RenderJob.ts index 2d173e7..a1419b2 100644 --- a/packages/browser/src/RenderJob.ts +++ b/packages/browser/src/RenderJob.ts @@ -27,7 +27,17 @@ export type JobConfig = { url: string; expiresAt: number; headers?: Record; + /** + * The ONE device this job renders — a legacy per-device job (plugin < 0.66.0), or one variant + * of a multi-device job (see `variants()`). A multi-device job as claimed also carries this as + * the first entry of `deviceTypes`, for renderers that predate the list. + */ deviceType: string; + /** + * Every device this job must render (plugin >= 0.66.0 claims ONE job per URL and expects every + * variant back in a single result). Absent on a legacy per-device job. + */ + deviceTypes?: string[]; acceptLanguage?: string; renderBudget?: number; callbackOrigin: string; @@ -104,6 +114,8 @@ export default class RenderJob { expiresAt: number; headers?: Record; deviceType: string; + /** See {@link JobConfig.deviceTypes}. Set on the job as CLAIMED; never on a variant. */ + deviceTypes: string[] | undefined; acceptLanguage: string | undefined; renderBudget: number | undefined; callbackOrigin: string; @@ -146,13 +158,42 @@ export default class RenderJob { this.url = config.url; this.headers = config.headers; this.expiresAt = config.expiresAt; - this.deviceType = config.deviceType; + this.deviceTypes = Array.isArray(config.deviceTypes) && config.deviceTypes.length ? config.deviceTypes : undefined; + // A multi-device job names its devices in `deviceTypes`; `deviceType` then only exists for + // renderers that predate the list, and the first entry is what such a renderer would render. + this.deviceType = config.deviceType ?? this.deviceTypes?.[0] ?? ''; this.acceptLanguage = config.acceptLanguage; this.renderBudget = config.renderBudget; this.callbackOrigin = config.callbackOrigin; this.isFromSitemap = config.isFromSitemap; } + /** + * The renders this claimed job stands for, one RenderJob per device. + * + * A legacy per-device job IS its own single variant. A multi-device job (plugin >= 0.66.0 claims + * one job per URL) fans out to one job per entry of `deviceTypes`, each sharing the claim's id, + * lease and callback — the renderer sees a plain per-device job either way, and only the worker + * knows that several of them travel back in one result (`sendVariantsResult`). + */ + variants(): RenderJob[] { + if (!this.deviceTypes) return [this]; + return this.deviceTypes.map( + (deviceType) => + new RenderJob({ + id: this.id, + url: this.url, + expiresAt: this.expiresAt, + headers: this.headers, + deviceType, + acceptLanguage: this.acceptLanguage, + renderBudget: this.renderBudget, + callbackOrigin: this.callbackOrigin, + isFromSitemap: this.isFromSitemap, + }) + ); + } + sanitizeHeaders(headers: Record) { const sanitized: Record = {}; for (const header of allowedResponseHeaders) { @@ -202,24 +243,21 @@ export default class RenderJob { return this.latestAttempt?.error || null; } - /** Returns true if the result was delivered (204), false if it was dropped after retries. */ - async sendResult(): Promise { - const health = getHostHealth(); - let host = ''; - try { - host = new URL(this.callbackOrigin).hostname; - } catch { - // Malformed callbackOrigin — can't track host health, but still attempt the POST. - } - - // Build the payload (incl. the expensive gzip) ONCE; retries re-send the same bytes. + /** + * This render's result as the plugin reads it — every field of the wire metadata EXCEPT the job + * identity (`id`/`url`), which belongs to the result envelope. One variant of a multi-device + * result, or (with the identity added) the whole of a legacy per-device result. + * + * Builds the encoded body too (the expensive gzip) so a caller assembling several variants pays + * it once per variant and retries re-send the same bytes. + */ + async resultMetadata(): Promise<{ metadata: VariantMetadata; contentBuffer: Buffer | null }> { const attemptError = this.error; - const metadata = { - id: this.id, - url: this.url, + const metadata: VariantMetadata = { + deviceType: this.deviceType, statusCode: this.httpResponse?.statusCode, - headers: {} as Record, - renderTime: undefined as number | undefined, + headers: {}, + renderTime: undefined, redirectedTo: this.redirectedTo, isIndexable: this.isIndexable, structuredOffers: this.structuredOffers, @@ -257,62 +295,131 @@ export default class RenderJob { metadata.headers['content-encoding'] = settings.contentEncoding; contentBuffer = await encode(this.content, settings.contentEncoding); } - const metadataBuffer = Buffer.from(JSON.stringify(metadata), 'utf-8'); - const body = contentBuffer - ? Buffer.concat([metadataBuffer, contentBuffer], metadataBuffer.byteLength + contentBuffer.byteLength) - : metadataBuffer; - - // Retry transient failures (503/overload/network) so an expensive render isn't thrown - // away on a blip — bounded by the retry cap AND the job's lease (`expiresAt`), after - // which Harper may have re-leased it, so posting is pointless. - const maxAttempts = Math.max(1, settings.backoff.resultRetries + 1); - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - const res = await request(this.callbackOrigin, { - method: 'POST', - path: '/render_queue/job_result', - body, - headers: { - 'x-metadata-size': metadataBuffer.byteLength.toString(), - 'content-type': 'application/octet-stream', - }, - }); - - if (res.statusCode === 204) { - await res.body.bytes(); - if (host) health.recordSuccess(host); - return true; - } + return { metadata, contentBuffer }; + } - const text = await res.body.text().catch(() => ''); - if (RESULT_RETRIABLE_STATUS.has(res.statusCode)) { - const retryAfterMs = parseRetryAfter(res.headers['retry-after'] as string | string[] | undefined); - if (host) health.recordUnavailable(host, retryAfterMs); - if (attempt < maxAttempts && Date.now() < this.expiresAt) { - await sleep(resultBackoffMs(attempt)); - continue; - } - } else if (host) { - // Non-retriable (4xx bug, auth failure, wrong endpoint) — usually persistent and - // host-wide. Feed the shared circuit (same as the claim path's non-2xx handling) - // so the consumer stops claiming work it can't deliver to this host, instead of - // rendering more results that will only be dropped. - health.recordError(host); - } - logger.error({ id: this.id, statusCode: res.statusCode, body: text, attempt }, 'failed to send job result'); - return false; - } catch (e) { - // Network error — host unreachable. - if (host) health.recordUnavailable(host); - if (attempt < maxAttempts && Date.now() < this.expiresAt) { + /** + * Post THIS render as a legacy per-device result: `{ id, url, ...variant }` followed by the one + * encoded body. The shape every plugin release understands; a multi-device job posts through + * `sendVariantsResult` instead. Returns true if the result was delivered (204), false if it was + * dropped after retries. + */ + async sendResult(): Promise { + const { metadata, contentBuffer } = await this.resultMetadata(); + // `deviceType` is not part of the legacy metadata — the plugin reads it off the cache key — + // but posting it is harmless and lets a newer plugin log the device without parsing. + const envelope = { id: this.id, url: this.url, ...metadata }; + return postResult(this, envelope, contentBuffer ? [contentBuffer] : []); + } + + /** + * Post ONE result for a multi-device job: `{ id, url, deviceTypes, variants: [...] }` followed by + * every variant's encoded body, concatenated in `variants` order — each variant's `contentLength` + * says how many of those bytes are its own (0 = no content). Plugin >= 0.66.0. + * + * `variants` is what was ATTEMPTED, which can be fewer than `deviceTypes` when the lease ran short + * or the worker began draining mid-job; the plugin treats a device it asked for and did not get + * back as a failed render and retries the URL. + */ + static async sendVariantsResult(job: RenderJob, variants: RenderJob[]): Promise { + const encoded = await Promise.all(variants.map((variant) => variant.resultMetadata())); + const bodies: Buffer[] = []; + const envelope = { + id: job.id, + url: job.url, + deviceTypes: job.deviceTypes ?? variants.map((variant) => variant.deviceType), + variants: encoded.map(({ metadata, contentBuffer }) => { + if (contentBuffer) bodies.push(contentBuffer); + return { ...metadata, contentLength: contentBuffer?.byteLength ?? 0 }; + }), + }; + return postResult(job, envelope, bodies); + } +} + +/** One variant's share of a posted result — see `RenderJob.resultMetadata`. */ +export type VariantMetadata = { + deviceType: string; + statusCode: number | undefined; + headers: Record; + renderTime: number | undefined; + redirectedTo: string | undefined; + isIndexable: boolean | undefined; + structuredOffers: Array | null | undefined; + outcome: JobOutcome; + reason: string | undefined; + error: { name: string; message: string; phase: string | undefined } | undefined; +}; + +/** + * POST one result body to the job's callback: the JSON envelope, then `bodies` concatenated, with + * `x-metadata-size` marking where the JSON ends. Shared by the legacy and the multi-device shapes so + * the retry policy cannot drift between them. + * + * Retries transient failures (503/overload/network) so an expensive render isn't thrown away on a + * blip — bounded by the retry cap AND the job's lease (`expiresAt`), after which Harper may have + * re-leased it, so posting is pointless. Returns true on a 204, false when dropped. + */ +async function postResult(job: RenderJob, envelope: object, bodies: Buffer[]): Promise { + const health = getHostHealth(); + let host = ''; + try { + host = new URL(job.callbackOrigin).hostname; + } catch { + // Malformed callbackOrigin — can't track host health, but still attempt the POST. + } + + // Build the payload ONCE; retries re-send the same bytes. + const metadataBuffer = Buffer.from(JSON.stringify(envelope), 'utf-8'); + const body = bodies.length ? Buffer.concat([metadataBuffer, ...bodies]) : metadataBuffer; + + const maxAttempts = Math.max(1, settings.backoff.resultRetries + 1); + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + const res = await request(job.callbackOrigin, { + method: 'POST', + path: '/render_queue/job_result', + body, + headers: { + 'x-metadata-size': metadataBuffer.byteLength.toString(), + 'content-type': 'application/octet-stream', + }, + }); + + if (res.statusCode === 204) { + await res.body.bytes(); + if (host) health.recordSuccess(host); + return true; + } + + const text = await res.body.text().catch(() => ''); + if (RESULT_RETRIABLE_STATUS.has(res.statusCode)) { + const retryAfterMs = parseRetryAfter(res.headers['retry-after'] as string | string[] | undefined); + if (host) health.recordUnavailable(host, retryAfterMs); + if (attempt < maxAttempts && Date.now() < job.expiresAt) { await sleep(resultBackoffMs(attempt)); continue; } - logger.error({ id: this.id, err: e, attempt }, 'failed to send job result'); - return false; + } else if (host) { + // Non-retriable (4xx bug, auth failure, wrong endpoint) — usually persistent and + // host-wide. Feed the shared circuit (same as the claim path's non-2xx handling) + // so the consumer stops claiming work it can't deliver to this host, instead of + // rendering more results that will only be dropped. + health.recordError(host); + } + logger.error({ id: job.id, statusCode: res.statusCode, body: text, attempt }, 'failed to send job result'); + return false; + } catch (e) { + // Network error — host unreachable. + if (host) health.recordUnavailable(host); + if (attempt < maxAttempts && Date.now() < job.expiresAt) { + await sleep(resultBackoffMs(attempt)); + continue; } + logger.error({ id: job.id, err: e, attempt }, 'failed to send job result'); + return false; } - // Exhausted retries without a definitive response (e.g. lease expired mid-backoff). - return false; } + // Exhausted retries without a definitive response (e.g. lease expired mid-backoff). + return false; } diff --git a/packages/browser/src/Worker.ts b/packages/browser/src/Worker.ts index 3a33e2b..c4cb04f 100644 --- a/packages/browser/src/Worker.ts +++ b/packages/browser/src/Worker.ts @@ -16,6 +16,13 @@ export type Renderer = (page: Page, job: RenderJob) => Promise= + // 0.66.0 a job renders every device of one URL, so `completed / jobs` is the variants per job. + jobs: 0, + // Variants a multi-device job asked for and this worker did NOT attempt: the lease ran short + // or the worker began draining between variants. The plugin retries the URL for them. + variantsSkipped: 0, concurrencyBlocked: 0, rpsDelayed: 0, resultPostFailures: 0, @@ -159,7 +172,7 @@ export default class RenderWorker { if (this.shuttingDown) break; const ts = Date.now(); // Do not run expired jobs to prevent double rendering - if (job.expiresAt - ts < 30 * 1000) { + if (job.expiresAt - ts < LEASE_MIN_REMAINING_MS) { this.stats.expiredSkipped++; console.log(`Skipping expired job ${job.id}`); continue; @@ -262,6 +275,8 @@ export default class RenderWorker { throughput: { completed: s.completed, perSec: Number((s.completed / elapsedSec).toFixed(2)), + jobs: s.jobs, + variantsSkipped: s.variantsSkipped, succeeded: s.succeeded, emptyContent: s.emptyContent, redirected: s.redirected, @@ -432,7 +447,67 @@ export default class RenderWorker { this.browser = null; } + /** + * One claimed job, start to posted result. + * + * A job is one URL. With plugin >= 0.66.0 it names every device to render (`deviceTypes`), and + * they are rendered HERE, in turn, on this job's one concurrency slot — so `CONCURRENCY` still + * bounds pages in flight, and a job simply occupies its slot for as many renders as it has + * devices. Every device's snapshot then travels back in ONE result, which is what lets the plugin + * keep a URL's variants aligned: same render pass, seconds apart, one scheduling decision. A + * legacy per-device job (older plugin) is the single-variant case of the same loop and posts the + * shape that plugin understands. + * + * The variants run sequentially rather than in parallel on purpose. Parallel variants would need + * page-level accounting against `CONCURRENCY` and would double this job's burst on the origin; + * sequential keeps every existing capacity number true (renders/hour/slot is unchanged) at the cost + * of a longer per-job wall time, which the lease absorbs (`queue.jobLeaseTime` is minutes, a + * variant is seconds). + * + * A variant is skipped — and the result posted PARTIAL — when the lease has under + * `LEASE_MIN_REMAINING_MS` left or the worker started draining. A partial result is still worth + * posting: the plugin stores the variants that rendered and retries the URL for the rest, and a + * result that never arrives costs the whole lease before anything retries. + */ async render(job: RenderJob) { + const variants = job.variants(); + const attempted: RenderJob[] = []; + + for (const variant of variants) { + if (attempted.length > 0) { + const leaseLeft = job.expiresAt - Date.now(); + if (leaseLeft < LEASE_MIN_REMAINING_MS || this.shuttingDown) { + const skipped = variants.length - attempted.length; + this.stats.variantsSkipped += skipped; + logger.warn( + { id: job.id, skipped, leaseLeftMs: leaseLeft, shuttingDown: this.shuttingDown }, + 'posting a partial result — remaining device variants not attempted' + ); + break; + } + } + await this.renderVariant(variant); + attempted.push(variant); + } + + // sendResult resolves true/false, but can still *reject* on an unexpected pre-POST failure + // (e.g. encode() throwing before the retry loop). Catch it so it's counted as a post + // failure rather than rejecting the whole render() through run()'s generic catch. + // + // The legacy shape for a legacy job, always: an older plugin reads `id` as a cache key and has + // no notion of `variants`, so it must get exactly what it always got. + const posted = await ( + job.deviceTypes ? RenderJob.sendVariantsResult(job, attempted) : attempted[0].sendResult() + ).catch((err) => { + logger.error({ id: job.id, err }, 'failed to send job result'); + return false; + }); + this.stats.jobs++; + if (!posted) this.stats.resultPostFailures++; + } + + /** Render ONE device variant on a page of its own; the result is posted by the caller. */ + private async renderVariant(job: RenderJob) { const browser = await this.getBrowser(); browser.jobRefs++; @@ -514,17 +589,8 @@ export default class RenderWorker { } } - // sendResult resolves true/false, but can still *reject* on an unexpected pre-POST failure - // (e.g. encode() throwing before the retry loop). Catch it so it's counted as a post - // failure rather than rejecting the whole render() through run()'s generic catch. - const sendPromise = job.sendResult().catch((err) => { - logger.error({ id: job.id, err }, 'failed to send job result'); - return false; - }); - const closePromise = page ? browser.closePage(page) : Promise.resolve(); try { - const [posted] = await Promise.all([sendPromise, closePromise]); - if (!posted) this.stats.resultPostFailures++; + if (page) await browser.closePage(page); } finally { // Released only now — not when the render finished. A retired browser is reaped once its // refs hit zero, so dropping the ref before the page is closed let the reaper close the diff --git a/packages/browser/test/jobResult.test.ts b/packages/browser/test/jobResult.test.ts index 9eacede..41cd59d 100644 --- a/packages/browser/test/jobResult.test.ts +++ b/packages/browser/test/jobResult.test.ts @@ -124,3 +124,189 @@ test('a failed render posts outcome=error with the attempt error and derived rea { name: 'Error', message: 'Navigation timeout of 30000 ms exceeded' } ); }); + +// ── multi-device results (plugin >= 0.66.0) ────────────────────────────────────────────────────── +// +// One job per URL, every device rendered in turn, ONE result back: `{ id, url, deviceTypes, +// variants: [...] }` followed by the variants' encoded bodies concatenated in order, each variant +// saying how many of those bytes are its own. These tests post through the real +// `sendVariantsResult` and decode the wire exactly as the plugin does. + +import { gunzipSync } from 'node:zlib'; + +// The raw bytes of every result the fake queue endpoint received, beside `posted`. +const postedBodies: Buffer[] = []; +let metadataSizes: number[] = []; + +before(() => { + // Re-wire the request handler to keep the raw body too — the framing is what these tests are about. + server.removeAllListeners('request'); + server.on('request', (req, res) => { + const chunks: Buffer[] = []; + req.on('data', (c) => chunks.push(c)); + req.on('end', () => { + const body = Buffer.concat(chunks); + const metadataSize = parseInt(String(req.headers['x-metadata-size'])); + posted.push(JSON.parse(body.subarray(0, metadataSize).toString('utf8'))); + postedBodies.push(body); + metadataSizes.push(metadataSize); + res.writeHead(204); + res.end(); + }); + }); +}); + +const makeUrlJob = (deviceTypes = ['desktop', 'mobile']) => + new RenderJob({ + id: 'https://site.example.com/product/x', + url: 'https://site.example.com/product/x', + expiresAt: Date.now() + 60_000, + deviceType: deviceTypes[0], + deviceTypes, + callbackOrigin, + isFromSitemap: true, + }); + +type Variant = { deviceType: string; outcome: string; contentLength: number; statusCode?: number; reason?: string }; + +const sendVariants = async (job: RenderJob, variants: RenderJob[]) => { + posted.length = 0; + postedBodies.length = 0; + metadataSizes = []; + assert.equal(await RenderJob.sendVariantsResult(job, variants), true, 'result must be delivered'); + return { + meta: posted[0] as { id: string; url: string; deviceTypes: string[]; variants: Variant[] }, + body: postedBodies[0], + metadataSize: metadataSizes[0], + }; +}; + +test('variants() fans a multi-device job out to one per-device job sharing the claim', () => { + const job = makeUrlJob(['desktop', 'mobile']); + const variants = job.variants(); + assert.deepEqual( + variants.map((v) => v.deviceType), + ['desktop', 'mobile'] + ); + for (const v of variants) { + assert.equal(v.id, job.id); + assert.equal(v.url, job.url); + assert.equal(v.expiresAt, job.expiresAt); + assert.equal(v.callbackOrigin, callbackOrigin); + assert.equal(v.isFromSitemap, true); + assert.equal(v.deviceTypes, undefined, 'a variant is a plain per-device job, never a group'); + } + assert.notEqual(variants[0], job, 'the group job itself is not rendered'); +}); + +test('a legacy per-device job is its own single variant', () => { + const job = makeJob(); + assert.deepEqual(job.variants(), [job]); + assert.equal(job.deviceTypes, undefined); +}); + +test('a job claimed with deviceTypes but no deviceType still has one (the first)', () => { + const job = new RenderJob({ + id: 'https://site.example.com/p', + url: 'https://site.example.com/p', + expiresAt: Date.now() + 60_000, + deviceTypes: ['mobile', 'desktop'], + callbackOrigin, + isFromSitemap: false, + } as never); + assert.equal(job.deviceType, 'mobile'); +}); + +test('a multi-device result posts one envelope and the bodies concatenated in variant order', async () => { + const job = makeUrlJob(['desktop', 'mobile']); + const [desktop, mobile] = job.variants(); + + desktop.attemptStarted(); + desktop.httpResponse = { statusCode: 200, headers: { 'content-type': 'text/html' } }; + desktop.isIndexable = true; + desktop.attemptEnded(undefined, 'desktop'); + + mobile.attemptStarted(); + mobile.httpResponse = { statusCode: 200, headers: { 'content-type': 'text/html' } }; + mobile.isIndexable = true; + mobile.attemptEnded(undefined, 'mobile — a longer body so the offsets differ'); + + const { meta, body, metadataSize } = await sendVariants(job, [desktop, mobile]); + + assert.equal(meta.id, job.id); + assert.equal(meta.url, job.url); + assert.deepEqual(meta.deviceTypes, ['desktop', 'mobile']); + assert.equal(meta.variants.length, 2); + assert.deepEqual( + meta.variants.map((v) => [v.deviceType, v.outcome]), + [ + ['desktop', 'rendered'], + ['mobile', 'rendered'], + ] + ); + + // Decode exactly as the plugin does: walk the body region by each variant's contentLength. + let offset = metadataSize; + const decoded: string[] = []; + for (const v of meta.variants) { + assert.ok(v.contentLength > 0, 'a rendered variant carries content'); + decoded.push(gunzipSync(body.subarray(offset, offset + v.contentLength)).toString('utf8')); + offset += v.contentLength; + } + assert.equal(offset, body.byteLength, 'the contentLengths account for every body byte'); + assert.deepEqual(decoded, ['desktop', 'mobile — a longer body so the offsets differ']); +}); + +test('a variant without content has contentLength 0 and consumes no body bytes', async () => { + const job = makeUrlJob(['desktop', 'mobile']); + const [desktop, mobile] = job.variants(); + + desktop.attemptStarted(); + desktop.httpResponse = { statusCode: 200, headers: {} }; + desktop.isIndexable = true; + desktop.attemptEnded(undefined, 'desktop'); + + // Mobile failed mid-render: no content, an error, and a derived reason. + mobile.attemptStarted(); + mobile.attemptEnded(new Error('Navigation timeout of 30000 ms exceeded'), undefined); + + const { meta, body, metadataSize } = await sendVariants(job, [desktop, mobile]); + assert.equal(meta.variants[0].contentLength > 0, true); + assert.deepEqual( + [meta.variants[1].outcome, meta.variants[1].contentLength, meta.variants[1].reason], + ['error', 0, 'error'] + ); + assert.equal(metadataSize + meta.variants[0].contentLength, body.byteLength); +}); + +test('a PARTIAL result echoes every device asked for while listing only those attempted', async () => { + const job = makeUrlJob(['desktop', 'mobile']); + const [desktop] = job.variants(); + desktop.attemptStarted(); + desktop.httpResponse = { statusCode: 200, headers: {} }; + desktop.isIndexable = true; + desktop.attemptEnded(undefined, 'desktop'); + + const { meta } = await sendVariants(job, [desktop]); + assert.deepEqual(meta.deviceTypes, ['desktop', 'mobile'], 'what the plugin asked for'); + assert.deepEqual( + meta.variants.map((v) => v.deviceType), + ['desktop'], + 'what this worker actually rendered — the plugin retries the URL for the rest' + ); +}); + +test('a legacy job still posts the legacy envelope: id, url and the variant fields at the top level', async () => { + const job = makeJob(); + job.attemptStarted(); + job.httpResponse = { statusCode: 200, headers: {} }; + job.isIndexable = true; + job.attemptEnded(undefined, 'ok'); + + const meta = (await send(job)) as Record; + assert.equal(meta.id, 'https://site.example.com/product/x|desktop'); + assert.equal(meta.url, 'https://site.example.com/product/x'); + assert.equal(meta.outcome, 'rendered'); + assert.equal('variants' in meta, false, 'an older plugin has no notion of variants'); + assert.equal('deviceTypes' in meta, false); +}); diff --git a/packages/browser/test/variantRender.test.ts b/packages/browser/test/variantRender.test.ts new file mode 100644 index 0000000..c370a12 --- /dev/null +++ b/packages/browser/test/variantRender.test.ts @@ -0,0 +1,187 @@ +import { test, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import RenderWorker from '../dist/Worker.js'; +import RenderJob from '../dist/RenderJob.js'; +import { resolveSettings } from '../dist/settings.js'; + +// The worker's render loop over a multi-device job (plugin >= 0.66.0): every device rendered in +// turn on the job's one slot, each on a page of its own, then ONE result posted — partial when +// the lease runs short between variants. Driven with a stub browser and a fake queue endpoint, so +// no Chrome is involved: what is under test is the loop and the accounting, not rendering. + +let server: http.Server; +let callbackOrigin = ''; +const posted: Array<{ id: string; deviceTypes?: string[]; variants?: Array<{ deviceType: string; outcome: string }> }> = + []; + +before(async () => { + resolveSettings({ harper: {} }, { requireHarper: false }); + server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', (c) => chunks.push(c)); + req.on('end', () => { + const body = Buffer.concat(chunks); + const metadataSize = parseInt(String(req.headers['x-metadata-size'])); + posted.push(JSON.parse(body.subarray(0, metadataSize).toString('utf8'))); + res.writeHead(204); + res.end(); + }); + }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + callbackOrigin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}); + +after(async () => { + await new Promise((r) => server.close(() => r())); +}); + +// A browser that hands out inert pages and counts them. `getBrowser()` returns `worker.browser` +// when set, so no launch happens. +const stubBrowser = () => { + const browser = { + jobRefs: 0, + activePages: 0, + opened: 0, + closed: 0, + maxJobRefs: 0, + closing: false, + getPage: async () => { + browser.opened++; + browser.activePages++; + browser.maxJobRefs = Math.max(browser.maxJobRefs, browser.jobRefs); + return { isClosed: () => false }; + }, + closePage: async () => { + browser.closed++; + browser.activePages--; + }, + close: async () => {}, + }; + return browser; +}; + +const makeWorker = (renderer: (page: unknown, job: RenderJob) => Promise) => { + const worker = new RenderWorker({ renderer: renderer as never, maxConcurrency: 1 }); + const browser = stubBrowser(); + worker.browser = browser as never; + return { worker, browser }; +}; + +const urlJob = (expiresInMs = 60_000, deviceTypes = ['desktop', 'mobile']) => + new RenderJob({ + id: 'https://site.example.com/product/x', + url: 'https://site.example.com/product/x', + expiresAt: Date.now() + expiresInMs, + deviceType: deviceTypes[0], + deviceTypes, + callbackOrigin, + isFromSitemap: false, + }); + +test('a multi-device job renders every device in turn, one page each, and posts ONE result', async () => { + const seen: string[] = []; + const { worker, browser } = makeWorker(async (_page, job) => { + seen.push(job.deviceType); + job.httpResponse = { statusCode: 200, headers: {} }; + job.isIndexable = true; + return `${job.deviceType}`; + }); + posted.length = 0; + try { + await worker.render(urlJob()); + } finally { + await worker.destroy(); + } + + assert.deepEqual(seen, ['desktop', 'mobile'], 'sequential, in the order the plugin asked'); + assert.equal(browser.opened, 2, 'a page per variant'); + assert.equal(browser.closed, 2, 'every page closed'); + assert.equal(browser.jobRefs, 0, 'the job ref is released after each variant'); + assert.equal(browser.maxJobRefs, 1, 'never more than one ref held — variants do not overlap'); + assert.equal(posted.length, 1, 'one result for the whole job'); + assert.deepEqual(posted[0].deviceTypes, ['desktop', 'mobile']); + assert.deepEqual( + posted[0].variants?.map((v) => [v.deviceType, v.outcome]), + [ + ['desktop', 'rendered'], + ['mobile', 'rendered'], + ] + ); +}); + +test('a variant that throws is reported as error and does not stop the others', async () => { + const { worker } = makeWorker(async (_page, job) => { + if (job.deviceType === 'desktop') throw new Error('settle exploded'); + job.httpResponse = { statusCode: 200, headers: {} }; + job.isIndexable = true; + return 'mobile'; + }); + posted.length = 0; + try { + await worker.render(urlJob()); + } finally { + await worker.destroy(); + } + assert.deepEqual( + posted[0].variants?.map((v) => [v.deviceType, v.outcome]), + [ + ['desktop', 'error'], + ['mobile', 'rendered'], + ] + ); +}); + +test('when the lease runs short between variants the result is posted PARTIAL rather than late', async () => { + // 31s of lease: the first variant runs (the run loop already admitted the job), then a + // ~1.5s render leaves ~29.5s, under the 30s floor, so the second variant is skipped. + const seen: string[] = []; + const { worker } = makeWorker(async (_page, job) => { + seen.push(job.deviceType); + await new Promise((r) => setTimeout(r, 1500)); + job.httpResponse = { statusCode: 200, headers: {} }; + job.isIndexable = true; + return `${job.deviceType}`; + }); + posted.length = 0; + try { + await worker.render(urlJob(31_000)); + } finally { + await worker.destroy(); + } + assert.deepEqual(seen, ['desktop']); + assert.equal(posted.length, 1, 'a partial result is still posted — silence would cost the whole lease'); + assert.deepEqual(posted[0].deviceTypes, ['desktop', 'mobile']); + assert.deepEqual( + posted[0].variants?.map((v) => v.deviceType), + ['desktop'] + ); +}); + +test('a legacy per-device job renders once and posts the legacy shape', async () => { + const { worker, browser } = makeWorker(async (_page, job) => { + job.httpResponse = { statusCode: 200, headers: {} }; + job.isIndexable = true; + return 'ok'; + }); + posted.length = 0; + try { + await worker.render( + new RenderJob({ + id: 'https://site.example.com/product/x|desktop', + url: 'https://site.example.com/product/x', + expiresAt: Date.now() + 60_000, + deviceType: 'desktop', + callbackOrigin, + isFromSitemap: false, + }) + ); + } finally { + await worker.destroy(); + } + assert.equal(browser.opened, 1); + assert.equal(posted.length, 1); + assert.equal(posted[0].id, 'https://site.example.com/product/x|desktop'); + assert.equal(posted[0].variants, undefined, 'an older plugin reads the flat shape'); +}); From 7acc2968981e17eff3735d057bad929aa7db1294 Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 10 Sep 2026 19:58:16 -0400 Subject: [PATCH 2/2] fix(browser): start the result post inside the promise chain so a synchronous throw is caught too (review) Co-Authored-By: Claude Fable 5.1 --- packages/browser/src/Worker.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/browser/src/Worker.ts b/packages/browser/src/Worker.ts index c4cb04f..87674c7 100644 --- a/packages/browser/src/Worker.ts +++ b/packages/browser/src/Worker.ts @@ -492,16 +492,19 @@ export default class RenderWorker { // sendResult resolves true/false, but can still *reject* on an unexpected pre-POST failure // (e.g. encode() throwing before the retry loop). Catch it so it's counted as a post - // failure rather than rejecting the whole render() through run()'s generic catch. + // failure rather than rejecting the whole render() through run()'s generic catch. Started + // inside the chain so a synchronous throw lands in the same catch rather than escaping it. + // (`attempted` is never empty: the skip check above only runs once one variant is done, and + // `variants()` always yields at least one.) // // The legacy shape for a legacy job, always: an older plugin reads `id` as a cache key and has // no notion of `variants`, so it must get exactly what it always got. - const posted = await ( - job.deviceTypes ? RenderJob.sendVariantsResult(job, attempted) : attempted[0].sendResult() - ).catch((err) => { - logger.error({ id: job.id, err }, 'failed to send job result'); - return false; - }); + const posted = await Promise.resolve() + .then(() => (job.deviceTypes ? RenderJob.sendVariantsResult(job, attempted) : attempted[0].sendResult())) + .catch((err) => { + logger.error({ id: job.id, err }, 'failed to send job result'); + return false; + }); this.stats.jobs++; if (!posted) this.stats.resultPostFailures++; }