From 8c8ac17d2b5d9e9f0608b61331eff1755ecbaaa1 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Fri, 25 Sep 2026 15:51:38 +0300 Subject: [PATCH 1/3] perf(core): compile the per-request zod schemas (#4123) --- packages/core/src/request.ts | 64 +++++++-------------- packages/core/src/storages/request_queue.ts | 33 ++++++----- packages/utils/src/internals/schemas.ts | 16 +++--- scripts/benchmarks/request-validation.mjs | 50 ++++++++++++++++ 4 files changed, 101 insertions(+), 62 deletions(-) create mode 100644 scripts/benchmarks/request-validation.mjs diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts index 957caba82fc3..da68e1383c40 100644 --- a/packages/core/src/request.ts +++ b/packages/core/src/request.ts @@ -8,7 +8,6 @@ import { z } from 'zod'; import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities'; import { serviceLocator } from './service_locator.js'; -import { keys } from './typedefs.js'; import { parseArgument, schemas } from './validators.js'; /** The `strategy` option accepted by {@apilink ExtractLinksOptions} and {@apilink EnqueueUrlsOptions}. */ @@ -26,31 +25,28 @@ interface CrawleeRequestData { enqueueStrategy?: EnqueueStrategyOption; } -const requestUrlSchema = z.object({ url: z.string() }); - // new properties on the Request object breaks serialization -const requestOptionalSchemaShapes: Record = { - uniqueKey: z.string().optional(), - method: z.string().optional(), - payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(), - noRetry: z.boolean().optional(), - sessionId: z.string().optional(), - maxRetries: schemas.anyNumber.optional(), - headers: z.looseObject({}).optional(), - userData: z.looseObject({}).optional(), - label: z.string().optional(), - keepUrlFragment: z.boolean().optional(), - useExtendedUniqueKey: z.boolean().optional(), - alwaysEnqueue: z.boolean().optional(), - skipNavigation: z.boolean().optional(), - crawlDepth: schemas.anyNumber - .refine((value) => value >= 0, 'Expected a number greater than or equal to 0') - .optional(), -}; - -// Each schema is wrapped in a single-key object so validation errors carry the property name. -const requestOptionalSchemas: Partial> = Object.fromEntries( - Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]), +// Compiled once: every `Request` runs it, and the generated fast path skips zod's interpreter. +const requestOptionsSchema = z.compile( + z.looseObject({ + url: z.string(), + uniqueKey: z.string().optional(), + method: z.string().optional(), + payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(), + noRetry: z.boolean().optional(), + sessionId: z.string().optional(), + maxRetries: schemas.anyNumber.optional(), + headers: z.looseObject({}).optional(), + userData: z.looseObject({}).optional(), + label: z.string().optional(), + keepUrlFragment: z.boolean().optional(), + useExtendedUniqueKey: z.boolean().optional(), + alwaysEnqueue: z.boolean().optional(), + skipNavigation: z.boolean().optional(), + crawlDepth: schemas.anyNumber + .refine((value) => value >= 0, 'Expected a number greater than or equal to 0') + .optional(), + }), ); /** @@ -153,23 +149,7 @@ class CrawleeRequest { } parseArgument(options, schemas.anyObject, 'RequestOptions'); - parseArgument(options, requestUrlSchema, 'RequestOptions'); - // Full-shape validation is slow, because it checks all predicates - // even if the validated object has only 1 property. - // This custom validation loop iterates only over existing - // properties and speeds up the validation cca 3-fold. - keys(options).forEach((prop) => { - // skip url, because it is validated above - if (prop === 'url') { - return; - } - - const schema = requestOptionalSchemas[prop as string]; - const value = options[prop]; - if (schema) { - parseArgument({ [prop]: value }, schema, 'RequestOptions'); - } - }); + parseArgument(options, requestOptionsSchema, 'RequestOptions'); const { url, diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.ts index f213fd0720cb..ce4ea5ef3dd8 100644 --- a/packages/core/src/storages/request_queue.ts +++ b/packages/core/src/storages/request_queue.ts @@ -65,19 +65,26 @@ const addRequestsBatchedOptionsSchema = z.strictObject({ waitBetweenBatchesMillis: schemas.anyNumber.default(1000), maxNewRequests: schemas.anyNumber.optional(), }); -const newRequestLikeSchema = z.looseObject({ - url: z.string(), - id: z.undefined().optional(), -}); -const handledRequestSchema = z.looseObject({ - id: z.string(), - uniqueKey: z.string(), - handledAt: z.string().optional(), -}); -const reclaimedRequestSchema = z.looseObject({ - id: z.string(), - uniqueKey: z.string(), -}); +// Compiled: these run once per request. +const newRequestLikeSchema = z.compile( + z.looseObject({ + url: z.string(), + id: z.undefined().optional(), + }), +); +const handledRequestSchema = z.compile( + z.looseObject({ + id: z.string(), + uniqueKey: z.string(), + handledAt: z.string().optional(), + }), +); +const reclaimedRequestSchema = z.compile( + z.looseObject({ + id: z.string(), + uniqueKey: z.string(), + }), +); const uniqueKeySchema = z.string(); const openOptionsSchema = z.strictObject({ configuration: z.instanceof(Configuration).optional(), diff --git a/packages/utils/src/internals/schemas.ts b/packages/utils/src/internals/schemas.ts index 3f8ea71e1687..cdb6fe74fb42 100644 --- a/packages/utils/src/internals/schemas.ts +++ b/packages/utils/src/internals/schemas.ts @@ -82,11 +82,7 @@ export const plainObject = z.custom>( { message: 'Invalid input: expected an object' }, ); -/** - * Shape of a request stored in a request queue. - * @internal - */ -export const storageRequest = z.looseObject({ +const storageRequestShape = z.looseObject({ id: z.string(), url: z.url({ protocol: /^https?$/ }), uniqueKey: z.string(), @@ -95,11 +91,17 @@ export const storageRequest = z.looseObject({ handledAt: z.union([z.string(), z.date()]).optional(), }); +/** + * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it. + * @internal + */ +export const storageRequest = z.compile(storageRequestShape); + /** * {@link storageRequest} before an id is assigned. * @internal */ -export const storageRequestWithoutId = storageRequest.omit({ id: true }); +export const storageRequestWithoutId = z.compile(storageRequestShape.omit({ id: true })); /** * `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` — @@ -118,7 +120,7 @@ export function arrayOf(item: TItem, elements: string): * Batch of {@link storageRequestWithoutId}. * @internal */ -export const storageRequestBatch = arrayOf(storageRequestWithoutId, 'requests'); +export const storageRequestBatch = z.compile(arrayOf(storageRequestWithoutId, 'requests')); /** * Options of request queue add/update operations. diff --git a/scripts/benchmarks/request-validation.mjs b/scripts/benchmarks/request-validation.mjs new file mode 100644 index 000000000000..2b21081d9985 --- /dev/null +++ b/scripts/benchmarks/request-validation.mjs @@ -0,0 +1,50 @@ +// Throughput of the per-request validation paths. Run against a build: `pnpm build && node scripts/benchmarks/request-validation.mjs`. +import { createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; + +const req = createRequire(new URL('../../packages/crawlee/package.json', import.meta.url)); +const load = (specifier) => import(pathToFileURL(req.resolve(specifier)).href); +const { Request } = await load('@crawlee/core'); +const { schemas } = await load('@crawlee/utils/internal'); +const { z } = await load('zod'); + +const options = { + url: 'https://example.com/products/123?page=2', + uniqueKey: 'https://example.com/products/123?page=2', + method: 'GET', + userData: { label: 'DETAIL', depth: 3 }, + headers: { accept: 'text/html' }, +}; +const stored = { id: 'abc123', ...options, retryCount: 0 }; +const batch = Array.from({ length: 100 }, (_, i) => ({ + ...options, + url: `${options.url}&i=${i}`, + uniqueKey: `${options.uniqueKey}&i=${i}`, +})); + +function opsPerSecond(fn, minMs = 500) { + for (let i = 0; i < 2000; i++) fn(); + let n = 0; + const t0 = performance.now(); + let t; + do { + for (let i = 0; i < 1000; i++) fn(); + n += 1000; + t = performance.now(); + } while (t - t0 < minMs); + return n / ((t - t0) / 1000); +} +const best = (fn) => Math.round(Math.max(opsPerSecond(fn), opsPerSecond(fn), opsPerSecond(fn))); + +console.table([ + { case: 'new Request(options)', 'ops/s': best(() => new Request(options)) }, + { case: 'storageRequest parse', 'ops/s': best(() => z.parse(schemas.storageRequest, stored)) }, + { + case: 'storageRequestBatch parse (100 requests)', + 'ops/s': best(() => z.parse(schemas.storageRequestBatch, batch)), + }, + { + case: 'requestQueueOperationOptions parse', + 'ops/s': best(() => z.parse(schemas.requestQueueOperationOptions, { forefront: true })), + }, +]); From 6a51211911f5c672f4f7f12ae2c8382cb104ffb4 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 25 Sep 2026 15:09:52 +0200 Subject: [PATCH 2/3] chore: Update CLAUDE.md (#4159) --- .claude/CLAUDE.md | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index c9d91a3db5d9..e6865a5248a2 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -13,7 +13,7 @@ Keep changes minimal and scoped. Do not fix unrelated issues, touch unrelated fi ## Pre-Commit Checks -Always run `yarn tsc-check-tests` (or the project's type-check command) before committing any TypeScript changes. Never assume type safety — verify it. +Always run `pnpm tsc-check-tests` (or the project's type-check command) before committing any TypeScript changes. Never assume type safety — verify it. ## Code Editing Rules @@ -31,35 +31,35 @@ When opening PRs, write concise descriptions focused on what changed and why. Av ## Build & Test Commands ```bash -# Setup (uses Yarn v4 via Corepack) +# Setup (pnpm via Corepack) corepack enable -yarn install +pnpm install # Build -yarn build # Build all packages (Turbo + TypeScript) +pnpm build # Build all packages (Turbo + TypeScript) # Test -yarn test # Run all tests (vitest), fast config -yarn test:full # Difficult tests + full firefox/webkit plugin matrix -yarn vitest run path/to/test.ts # Run specific test file +pnpm test # Run all tests (vitest), fast config +pnpm test:full # Difficult tests + full firefox/webkit plugin matrix +pnpm vitest run path/to/test.ts # Run specific test file # Code Quality -yarn lint # ESLint -yarn lint:fix # ESLint with auto-fix -yarn format # Format with Biome -yarn tsc-check-tests # Type-check test files +pnpm lint # oxlint +pnpm lint:fix # oxlint with auto-fix +pnpm format # Format with oxfmt +pnpm tsc-check-tests # Type-check test files ``` ## Architecture -Crawlee is a **Yarn workspaces monorepo** with Turbo build orchestration. All packages are in `/packages/`. +Crawlee is a **pnpm workspaces monorepo** with Turbo build orchestration. All packages are in `/packages/`. ### Package Hierarchy ``` @crawlee/types # Shared TypeScript interfaces @crawlee/utils # Shared utilities -@crawlee/memory-storage # In-memory storage (default for testing) +@crawlee/fs-storage # File-system storage backend (in-memory backend lives in @crawlee/core) ↓ @crawlee/core # Request, RequestQueue, RequestList, Dataset ↓ @@ -67,7 +67,6 @@ Crawlee is a **Yarn workspaces monorepo** with Turbo build orchestration. All pa ↓ @crawlee/http # HttpCrawler ↓ -↓ @crawlee/cheerio (@crawlee/jsdom and @crawlee/linkedom moved to their own repositories) @@ -75,16 +74,28 @@ Crawlee is a **Yarn workspaces monorepo** with Turbo build orchestration. All pa ↓ @crawlee/browser # BrowserCrawler base ↓ +┌──────┴──────┬────────────────────┐ +↓ ↓ ↓ +@crawlee/playwright @crawlee/puppeteer @crawlee/stagehand # stagehand: AI-driven browser crawler + +@crawlee/http-client # Pluggable HTTP client interface + ↓ ┌──────┴──────┐ ↓ ↓ -@crawlee/playwright @crawlee/puppeteer +@crawlee/impit-client @crawlee/got-scraping-client + +@crawlee/otel # OpenTelemetry instrumentation + +@crawlee/templates # Project templates + ↓ +@crawlee/cli # `crawlee` CLI (create/run projects, install Playwright browsers) crawlee # Meta-package re-exporting most @crawlee/* packages ``` ### Test Location -Tests are in `/test/` at the repo root (not inside packages). E2E tests are in `/test/e2e/`. +Most tests are in `/test/` at the repo root; some packages also have their own `packages/*/test/`. E2E tests are in `/test/e2e/`. `tsc-check-tests` only type-checks `/test/`, not `packages/*/test/`. ## Vitest Notes (vs Jest) From 761be275c6ceb9c26f292d7939c9e9eccb88da91 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 25 Sep 2026 16:22:35 +0200 Subject: [PATCH 3/3] fix!: Remove mostly unused Request.userData.forefront field (#4157) - closes #4139 --- packages/basic-crawler/src/internals/basic-crawler.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 3960b3b500c9..4f15f074bbbc 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -2830,13 +2830,13 @@ export class BasicCrawler< source: IRequestManager, ): Promise { if (error instanceof RequestThrottledError) { - // The domain told us to come back later, so the request was never really attempted. Put it back - // without recording a failure - it costs neither a retry nor session reputation. + // The domain told us to come back later, so the request was never really attempted. Put it back where + // it was, without recording a failure - it costs neither a retry nor session reputation. this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, { id: request.id, url: request.url, }); - await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront }); + await source.reclaimRequest(request, { forefront: true }); return; } @@ -2873,7 +2873,7 @@ export class BasicCrawler< retryCount, }); - await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront }); + await source.reclaimRequest(request); return; } }