[pull] master from apify:master

.claude/CLAUDE.md

@@ -13,7 +13,7 @@ Keep changes minimal and scoped. Do not fix unrelated issues, touch unrelated fi
 
 ## Pre-Commit Checks
 
-Always run `yarn tsc-check-tests` (or the project's type-check command) before committing any TypeScript changes. Never assume type safety — verify it.
+Always run `pnpm tsc-check-tests` (or the project's type-check command) before committing any TypeScript changes. Never assume type safety — verify it.
 
 ## Code Editing Rules
 
@@ -31,60 +31,71 @@ When opening PRs, write concise descriptions focused on what changed and why. Av
 ## Build & Test Commands
 
 ```bash
-# Setup (uses Yarn v4 via Corepack)
+# Setup (pnpm via Corepack)
 corepack enable
-yarn install
+pnpm install
 
 # Build
-yarn build                    # Build all packages (Turbo + TypeScript)
+pnpm build                    # Build all packages (Turbo + TypeScript)
 
 # Test
-yarn test                     # Run all tests (vitest), fast config
-yarn test:full                # Difficult tests + full firefox/webkit plugin matrix
-yarn vitest run path/to/test.ts    # Run specific test file
+pnpm test                     # Run all tests (vitest), fast config
+pnpm test:full                # Difficult tests + full firefox/webkit plugin matrix
+pnpm vitest run path/to/test.ts    # Run specific test file
 
 # Code Quality
-yarn lint                     # ESLint
-yarn lint:fix                 # ESLint with auto-fix
-yarn format                   # Format with Biome
-yarn tsc-check-tests          # Type-check test files
+pnpm lint                     # oxlint
+pnpm lint:fix                 # oxlint with auto-fix
+pnpm format                   # Format with oxfmt
+pnpm tsc-check-tests          # Type-check test files
 ```
 
 ## Architecture
 
-Crawlee is a **Yarn workspaces monorepo** with Turbo build orchestration. All packages are in `/packages/`.
+Crawlee is a **pnpm workspaces monorepo** with Turbo build orchestration. All packages are in `/packages/`.
 
 ### Package Hierarchy
 
 ```
 @crawlee/types          # Shared TypeScript interfaces
 @crawlee/utils          # Shared utilities
-@crawlee/memory-storage # In-memory storage (default for testing)
+@crawlee/fs-storage     # File-system storage backend (in-memory backend lives in @crawlee/core)
        ↓
 @crawlee/core           # Request, RequestQueue, RequestList, Dataset
        ↓
 @crawlee/basic          # BasicCrawler (foundation for all crawlers)
        ↓
 @crawlee/http           # HttpCrawler
        ↓
-↓
 @crawlee/cheerio
 (@crawlee/jsdom and @crawlee/linkedom moved to their own repositories)
 
 @crawlee/browser-pool   # Browser instance management
        ↓
 @crawlee/browser        # BrowserCrawler base
        ↓
+┌──────┴──────┬────────────────────┐
+↓             ↓                    ↓
+@crawlee/playwright  @crawlee/puppeteer  @crawlee/stagehand   # stagehand: AI-driven browser crawler
+
+@crawlee/http-client    # Pluggable HTTP client interface
+       ↓
 ┌──────┴──────┐
 ↓             ↓
-@crawlee/playwright  @crawlee/puppeteer
+@crawlee/impit-client  @crawlee/got-scraping-client
+
+@crawlee/otel           # OpenTelemetry instrumentation
+
+@crawlee/templates      # Project templates
+       ↓
+@crawlee/cli            # `crawlee` CLI (create/run projects, install Playwright browsers)
 
 crawlee                 # Meta-package re-exporting most @crawlee/* packages
 ```
 
 ### Test Location
 
-Tests are in `/test/` at the repo root (not inside packages). E2E tests are in `/test/e2e/`.
+Most tests are in `/test/` at the repo root; some packages also have their own `packages/*/test/`. E2E tests are in `/test/e2e/`. `tsc-check-tests` only type-checks `/test/`, not `packages/*/test/`.
 
 ## Vitest Notes (vs Jest)
 

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -2830,13 +2830,13 @@ export class BasicCrawler<
         source: IRequestManager,
     ): Promise<void> {
         if (error instanceof RequestThrottledError) {
-            // The domain told us to come back later, so the request was never really attempted. Put it back
-            // without recording a failure - it costs neither a retry nor session reputation.
+            // The domain told us to come back later, so the request was never really attempted. Put it back where
+            // it was, without recording a failure - it costs neither a retry nor session reputation.
             this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
                 id: request.id,
                 url: request.url,
             });
-            await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+            await source.reclaimRequest(request, { forefront: true });
             return;
         }
 
@@ -2873,7 +2873,7 @@ export class BasicCrawler<
                     retryCount,
                 });
 
-                await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+                await source.reclaimRequest(request);
                 return;
             }
         }

packages/core/src/request.ts

@@ -8,7 +8,6 @@ import { z } from 'zod';
 import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
 
 import { serviceLocator } from './service_locator.js';
-import { keys } from './typedefs.js';
 import { parseArgument, schemas } from './validators.js';
 
 /** The `strategy` option accepted by {@apilink ExtractLinksOptions} and {@apilink EnqueueUrlsOptions}. */
@@ -26,31 +25,28 @@ interface CrawleeRequestData {
     enqueueStrategy?: EnqueueStrategyOption;
 }
 
-const requestUrlSchema = z.object({ url: z.string() });
-
 // new properties on the Request object breaks serialization
-const requestOptionalSchemaShapes: Record<string, z.ZodType> = {
-    uniqueKey: z.string().optional(),
-    method: z.string().optional(),
-    payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
-    noRetry: z.boolean().optional(),
-    sessionId: z.string().optional(),
-    maxRetries: schemas.anyNumber.optional(),
-    headers: z.looseObject({}).optional(),
-    userData: z.looseObject({}).optional(),
-    label: z.string().optional(),
-    keepUrlFragment: z.boolean().optional(),
-    useExtendedUniqueKey: z.boolean().optional(),
-    alwaysEnqueue: z.boolean().optional(),
-    skipNavigation: z.boolean().optional(),
-    crawlDepth: schemas.anyNumber
-        .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
-        .optional(),
-};
-
-// Each schema is wrapped in a single-key object so validation errors carry the property name.
-const requestOptionalSchemas: Partial<Record<string, z.ZodType>> = Object.fromEntries(
-    Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]),
+// Compiled once: every `Request` runs it, and the generated fast path skips zod's interpreter.
+const requestOptionsSchema = z.compile(
+    z.looseObject({
+        url: z.string(),
+        uniqueKey: z.string().optional(),
+        method: z.string().optional(),
+        payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
+        noRetry: z.boolean().optional(),
+        sessionId: z.string().optional(),
+        maxRetries: schemas.anyNumber.optional(),
+        headers: z.looseObject({}).optional(),
+        userData: z.looseObject({}).optional(),
+        label: z.string().optional(),
+        keepUrlFragment: z.boolean().optional(),
+        useExtendedUniqueKey: z.boolean().optional(),
+        alwaysEnqueue: z.boolean().optional(),
+        skipNavigation: z.boolean().optional(),
+        crawlDepth: schemas.anyNumber
+            .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
+            .optional(),
+    }),
 );
 
 /**
@@ -153,23 +149,7 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
         }
 
         parseArgument(options, schemas.anyObject, 'RequestOptions');
-        parseArgument(options, requestUrlSchema, 'RequestOptions');
-        // Full-shape validation is slow, because it checks all predicates
-        // even if the validated object has only 1 property.
-        // This custom validation loop iterates only over existing
-        // properties and speeds up the validation cca 3-fold.
-        keys(options).forEach((prop) => {
-            // skip url, because it is validated above
-            if (prop === 'url') {
-                return;
-            }
-
-            const schema = requestOptionalSchemas[prop as string];
-            const value = options[prop];
-            if (schema) {
-                parseArgument({ [prop]: value }, schema, 'RequestOptions');
-            }
-        });
+        parseArgument(options, requestOptionsSchema, 'RequestOptions');
 
         const {
             url,

packages/core/src/storages/request_queue.ts

@@ -65,19 +65,26 @@ const addRequestsBatchedOptionsSchema = z.strictObject({
     waitBetweenBatchesMillis: schemas.anyNumber.default(1000),
     maxNewRequests: schemas.anyNumber.optional(),
 });
-const newRequestLikeSchema = z.looseObject({
-    url: z.string(),
-    id: z.undefined().optional(),
-});
-const handledRequestSchema = z.looseObject({
-    id: z.string(),
-    uniqueKey: z.string(),
-    handledAt: z.string().optional(),
-});
-const reclaimedRequestSchema = z.looseObject({
-    id: z.string(),
-    uniqueKey: z.string(),
-});
+// Compiled: these run once per request.
+const newRequestLikeSchema = z.compile(
+    z.looseObject({
+        url: z.string(),
+        id: z.undefined().optional(),
+    }),
+);
+const handledRequestSchema = z.compile(
+    z.looseObject({
+        id: z.string(),
+        uniqueKey: z.string(),
+        handledAt: z.string().optional(),
+    }),
+);
+const reclaimedRequestSchema = z.compile(
+    z.looseObject({
+        id: z.string(),
+        uniqueKey: z.string(),
+    }),
+);
 const uniqueKeySchema = z.string();
 const openOptionsSchema = z.strictObject({
     configuration: z.instanceof(Configuration).optional(),

packages/utils/src/internals/schemas.ts

@@ -82,11 +82,7 @@ export const plainObject = z.custom<Record<string, unknown>>(
     { message: 'Invalid input: expected an object' },
 );
 
-/**
- * Shape of a request stored in a request queue.
- * @internal
- */
-export const storageRequest = z.looseObject({
+const storageRequestShape = z.looseObject({
     id: z.string(),
     url: z.url({ protocol: /^https?$/ }),
     uniqueKey: z.string(),
@@ -95,11 +91,17 @@ export const storageRequest = z.looseObject({
     handledAt: z.union([z.string(), z.date()]).optional(),
 });
 
+/**
+ * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
+ * @internal
+ */
+export const storageRequest = z.compile(storageRequestShape);
+
 /**
  * {@link storageRequest} before an id is assigned.
  * @internal
  */
-export const storageRequestWithoutId = storageRequest.omit({ id: true });
+export const storageRequestWithoutId = z.compile(storageRequestShape.omit({ id: true }));
 
 /**
  * `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
@@ -118,7 +120,7 @@ export function arrayOf<TItem extends z.ZodType>(item: TItem, elements: string):
  * Batch of {@link storageRequestWithoutId}.
  * @internal
  */
-export const storageRequestBatch = arrayOf(storageRequestWithoutId, 'requests');
+export const storageRequestBatch = z.compile(arrayOf(storageRequestWithoutId, 'requests'));
 
 /**
  * Options of request queue add/update operations.

scripts/benchmarks/request-validation.mjs

@@ -0,0 +1,50 @@
+// Throughput of the per-request validation paths. Run against a build: `pnpm build && node scripts/benchmarks/request-validation.mjs`.
+import { createRequire } from 'node:module';
+import { pathToFileURL } from 'node:url';
+
+const req = createRequire(new URL('../../packages/crawlee/package.json', import.meta.url));
+const load = (specifier) => import(pathToFileURL(req.resolve(specifier)).href);
+const { Request } = await load('@crawlee/core');
+const { schemas } = await load('@crawlee/utils/internal');
+const { z } = await load('zod');
+
+const options = {
+    url: 'https://example.com/products/123?page=2',
+    uniqueKey: 'https://example.com/products/123?page=2',
+    method: 'GET',
+    userData: { label: 'DETAIL', depth: 3 },
+    headers: { accept: 'text/html' },
+};
+const stored = { id: 'abc123', ...options, retryCount: 0 };
+const batch = Array.from({ length: 100 }, (_, i) => ({
+    ...options,
+    url: `${options.url}&i=${i}`,
+    uniqueKey: `${options.uniqueKey}&i=${i}`,
+}));
+
+function opsPerSecond(fn, minMs = 500) {
+    for (let i = 0; i < 2000; i++) fn();
+    let n = 0;
+    const t0 = performance.now();
+    let t;
+    do {
+        for (let i = 0; i < 1000; i++) fn();
+        n += 1000;
+        t = performance.now();
+    } while (t - t0 < minMs);
+    return n / ((t - t0) / 1000);
+}
+const best = (fn) => Math.round(Math.max(opsPerSecond(fn), opsPerSecond(fn), opsPerSecond(fn)));
+
+console.table([
+    { case: 'new Request(options)', 'ops/s': best(() => new Request(options)) },
+    { case: 'storageRequest parse', 'ops/s': best(() => z.parse(schemas.storageRequest, stored)) },
+    {
+        case: 'storageRequestBatch parse (100 requests)',
+        'ops/s': best(() => z.parse(schemas.storageRequestBatch, batch)),
+    },
+    {
+        case: 'requestQueueOperationOptions parse',
+        'ops/s': best(() => z.parse(schemas.requestQueueOperationOptions, { forefront: true })),
+    },
+]);