From fadc53fb5043eae2fefbf515f840bed90627e579 Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Wed, 15 Jul 2026 23:36:38 +0800 Subject: [PATCH 01/38] fix(docker): tolerate unavailable IPv6 loopback --- deploy/docker/supervisord.conf | 4 ++-- deploy/docker/tests/test_security_container_posture.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/docker/supervisord.conf b/deploy/docker/supervisord.conf index 91b4b2538..d726e2c6c 100644 --- a/deploy/docker/supervisord.conf +++ b/deploy/docker/supervisord.conf @@ -6,7 +6,7 @@ logfile_maxbytes=0 [program:redis] ; Loopback-only and password-protected. REDIS_PASSWORD is exported by ; entrypoint.sh from the mounted secret; the app must supply it to connect. -command=/usr/bin/redis-server --loglevel notice --bind 127.0.0.1 ::1 --requirepass "%(ENV_REDIS_PASSWORD)s" --dir /var/lib/redis +command=/usr/bin/redis-server --loglevel notice --bind 127.0.0.1 -::1 --requirepass "%(ENV_REDIS_PASSWORD)s" --dir /var/lib/redis user=appuser ; Run redis as our non-root user autorestart=true priority=10 @@ -31,4 +31,4 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr ; Redirect gunicorn stderr to container stderr stderr_logfile_maxbytes=0 -# Optional: Add filebeat or other logging agents here if needed \ No newline at end of file +# Optional: Add filebeat or other logging agents here if needed diff --git a/deploy/docker/tests/test_security_container_posture.py b/deploy/docker/tests/test_security_container_posture.py index 871e642de..3399d6bd1 100644 --- a/deploy/docker/tests/test_security_container_posture.py +++ b/deploy/docker/tests/test_security_container_posture.py @@ -78,8 +78,8 @@ class TestSupervisord: def test_redis_requires_password(self, supervisord): assert "--requirepass" in supervisord - def test_redis_bound_loopback(self, supervisord): - assert "--bind 127.0.0.1" in supervisord + def test_redis_bound_loopback_with_optional_ipv6(self, supervisord): + assert "--bind 127.0.0.1 -::1" in supervisord def test_gunicorn_bind_is_env_driven(self, supervisord): # entrypoint.sh resolves GUNICORN_BIND (loopback unless a credential). From a659914ab3784a75ae9ef1804a0acddbb12f24c3 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Wed, 22 Jul 2026 13:22:12 +0530 Subject: [PATCH 02/38] docs(docker): update self-hosting & migration guides for 0.9.x The self-hosting guide predated the 0.9.0 secure-by-default release and its quickstarts produced an unreachable server. Verified against a fresh 0.9.2 deployment. self-hosting.md: - Require CRAWL4AI_API_TOKEN in every quickstart (docker run, compose, manual build); explain the loopback-only bind and the resulting "connection reset" on published ports, plus startup-delay troubleshooting - Compose: .llm.env is required; the token must be set inside it (host `export` is not forwarded to the container) - Replace the removed inline-Python hooks API (~600 lines) with the declarative hooks reference: CRAWL4AI_HOOKS_ENABLED flag, action table, /hooks/info, and 0.8.x migration notes - Rewrite /screenshot and /pdf for the artifact flow; note output_path is removed and currently silently ignored - Fix dashboard URL (/dashboard, not /monitor); document the token bar in the playground and dashboard UIs - Fix stale versions (0.8.0 -> 0.9.2); drop the SDK function-hooks example MIGRATION.md: - Token setup instructions for compose (.llm.env) and docker run (explicit -e form); clarify the loopback failure symptom - Document CRAWL4AI_HOOKS_ENABLED and the silent failure modes of legacy hooks.code and output_path .llm.env.example: - Add CRAWL4AI_API_TOKEN with guidance; add CRAWL4AI_HOOKS_ENABLED --- deploy/docker/.llm.env.example | 11 + deploy/docker/MIGRATION.md | 31 +- docs/md_v2/core/self-hosting.md | 897 ++++++++------------------------ 3 files changed, 245 insertions(+), 694 deletions(-) diff --git a/deploy/docker/.llm.env.example b/deploy/docker/.llm.env.example index 012435d83..2d7a2ea25 100644 --- a/deploy/docker/.llm.env.example +++ b/deploy/docker/.llm.env.example @@ -1,3 +1,14 @@ +# REQUIRED for a reachable server: API token for the Docker server (0.9.0+). +# Without it the server binds loopback inside the container and the published +# port answers with "connection reset". Any non-empty value works, but treat it +# as a password — use a long random string (e.g. from: openssl rand -hex 32). +# Note: with docker compose, the token MUST be set here — exporting it in your +# shell does not reach the container. +CRAWL4AI_API_TOKEN= + +# Optional: enable declarative hooks support (disabled by default) +# CRAWL4AI_HOOKS_ENABLED=true + # LLM Provider Keys OPENAI_API_KEY=your_openai_key_here DEEPSEEK_API_KEY=your_deepseek_key_here diff --git a/deploy/docker/MIGRATION.md b/deploy/docker/MIGRATION.md index 753823060..b895ae588 100644 --- a/deploy/docker/MIGRATION.md +++ b/deploy/docker/MIGRATION.md @@ -25,11 +25,23 @@ loopback by default and will not expose itself without a credential. export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)" ``` +> ⚠️ **Docker Compose users:** `export` alone does **not** work — the shipped +> `docker-compose.yml` does not forward host environment variables. Set the +> token in the `.llm.env` file at the project root instead (the example file +> ships an empty `CRAWL4AI_API_TOKEN=` line — fill it in). +> +> For plain `docker run`, pass it explicitly: +> `-e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN"` (the value-less shorthand +> `-e CRAWL4AI_API_TOKEN` silently passes empty from a shell where the variable +> isn't set). + - With a token set, you may expose the server (put a TLS-terminating reverse proxy in front) and must send `Authorization: Bearer ` on every request except `GET /health`. -- With **no** token set, the server binds `127.0.0.1` only and prints a one-off - token at startup for local use. +- With **no** token set, the server binds `127.0.0.1` only (the **container's** + loopback — published ports answer with *connection reset* even though the + container reports healthy) and prints a one-off token at startup for + in-container use. WebSocket clients (MCP, monitor) that can't set headers may pass `?token=...`. @@ -60,6 +72,10 @@ safe maximums. ### Hooks: declarative actions instead of code +Hooks are now **disabled by default** — enable them with +`CRAWL4AI_HOOKS_ENABLED=true` in the container environment, or any request +containing `hooks` returns HTTP 403. + `hooks.code` (Python strings) is replaced by a fixed set of declarative actions: ```jsonc @@ -77,6 +93,13 @@ Available actions: `block_resources`, `add_cookies`, `set_headers`, `scroll_to_bottom`, `wait_for_timeout`. Call `GET /hooks/info` for the parameter schemas. Arbitrary hook code is available in a self-hosted in-process build. +> ⚠️ **Legacy `hooks.code` requests fail silently.** With hooks enabled, a +> request in the old format returns HTTP 200 with +> `"hooks": {"status": "success", "attached": []}` — the inline code is +> dropped without error. If `attached` is empty, your hooks did not run. +> (With hooks disabled, the same request returns the generic 403, whose +> "enable hooks" hint will not make code hooks work either.) + ### Screenshot / PDF: artifact id instead of `output_path` `output_path` is removed. The server stores the result and returns an id + URL: @@ -89,6 +112,10 @@ schemas. Arbitrary hook code is available in a self-hosted in-process build. Fetch the file with `GET /artifacts/{artifact_id}` (authenticated). Artifacts have a TTL and a storage quota. +> ⚠️ A request that still includes `output_path` is **silently ignored** — it +> returns `success: true` with an artifact id, but no file is written to the +> requested path. Update your code to fetch from `/artifacts/{artifact_id}`. + ### LLM endpoints: provider by name `base_url` is removed from `/md`, `/llm`, and `/llm/job`. Select a provider by diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.md index 1420774d1..966672f04 100644 --- a/docs/md_v2/core/self-hosting.md +++ b/docs/md_v2/core/self-hosting.md @@ -1,13 +1,17 @@ # Self-Hosting Crawl4AI 🚀 -> **🔐 0.9.0 is secure-by-default (breaking changes).** The self-hosted Docker -> server now requires authentication by default, binds to loopback unless you +> **🔐 0.9.0+ is secure-by-default (breaking changes).** The self-hosted Docker +> server requires authentication by default, binds to loopback unless you > set a token, validates request bodies against a strict trust boundary, uses > declarative hooks instead of inline Python, and returns artifact ids for > screenshot/pdf. If you are upgrading from 0.8.x, read the > [migration guide](https://github.com/unclecode/crawl4ai/blob/main/deploy/docker/MIGRATION.md) -> first. Some examples below are being updated for 0.9.0; the migration guide is -> the authoritative reference for the new defaults. +> first. +> +> **The single most important thing to know:** without a `CRAWL4AI_API_TOKEN`, +> the server binds loopback **inside the container** — published ports will +> answer with *connection reset* even though the container reports healthy. +> Every quickstart below therefore starts by setting a token. **Take Control of Your Web Crawling Infrastructure** @@ -76,13 +80,13 @@ Pull and run images directly from Docker Hub without building locally. #### 1. Pull the Image -Our latest release is `0.8.0`. Images are built with multi-arch manifests, so Docker automatically pulls the correct version for your system. +Our latest release is `0.9.2`. Images are built with multi-arch manifests, so Docker automatically pulls the correct version for your system. -> 💡 **Note**: The `latest` tag points to the stable `0.8.0` version. +> 💡 **Note**: The `latest` tag points to the most recent stable version. ```bash # Pull the latest version -docker pull unclecode/crawl4ai:0.8.0 +docker pull unclecode/crawl4ai:0.9.2 # Or pull using the latest tag docker pull unclecode/crawl4ai:latest @@ -121,7 +125,23 @@ EOL ``` > 🔑 **Note**: Keep your API keys secure! Never commit `.llm.env` to version control. -#### 3. Run the Container +#### 3. Set an API Token (Required to Reach the Server) + +Since 0.9.0 the server is secure-by-default: **without a token it binds +loopback inside the container**, so the published port answers with +*connection reset* — even though `docker ps` shows the container healthy and +the port mapped. Generate a token first: + +```bash +export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)" +``` + +> ⚠️ Use the explicit form `-e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN"` below. +> The shorthand `-e CRAWL4AI_API_TOKEN` (no value) forwards the variable from +> the *current shell* — in a shell where it isn't set, it silently passes an +> empty value and the server starts in loopback mode with no host-side warning. + +#### 4. Run the Container * **Basic run:** ```bash @@ -129,6 +149,7 @@ EOL -p 11235:11235 \ --name crawl4ai \ --shm-size=1g \ + -e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN" \ unclecode/crawl4ai:latest ``` @@ -140,12 +161,30 @@ EOL --name crawl4ai \ --env-file .llm.env \ --shm-size=1g \ + -e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN" \ unclecode/crawl4ai:latest ``` -> The server will be available at `http://localhost:11235`. Visit `/playground` to access the interactive testing interface. +The server will be available at `http://localhost:11235` after a short startup +(allow ~10 seconds; the healthcheck allows up to 40). Verify: + +```bash +curl http://localhost:11235/health # no auth needed for /health +``` + +Every other endpoint requires `Authorization: Bearer $CRAWL4AI_API_TOKEN`. +Visit `/playground` for the interactive testing interface and `/dashboard` for +the monitoring UI — both have an **API token** bar in the top navigation; paste +your token there and click **Set**. + +> 💡 **Troubleshooting "connection reset":** during the ~10s startup window the +> published port also answers with connection reset — the same symptom as the +> missing-token failure. If resets persist after 20s, check +> `docker logs crawl4ai` for `binding loopback only`: that means no token +> reached the container (note the `127.0.0.1` in that log line is the +> *container's* loopback, not your host's — the port mapping cannot reach it). -#### 4. Stopping the Container +#### 5. Stopping the Container ```bash docker stop crawl4ai && docker rm crawl4ai @@ -154,7 +193,7 @@ docker stop crawl4ai && docker rm crawl4ai #### Docker Hub Versioning Explained * **Image Name:** `unclecode/crawl4ai` -* **Tag Format:** `LIBRARY_VERSION[-SUFFIX]` (e.g., `0.8.0`) +* **Tag Format:** `LIBRARY_VERSION[-SUFFIX]` (e.g., `0.9.2`) * `LIBRARY_VERSION`: The semantic version of the core `crawl4ai` Python library * `SUFFIX`: Optional tag for release candidates (``) and revisions (`r1`) * **`latest` Tag:** Points to the most recent stable version @@ -171,17 +210,32 @@ git clone https://github.com/unclecode/crawl4ai.git cd crawl4ai ``` -#### 2. Environment Setup (API Keys) +#### 2. Environment Setup (Required) -If you plan to use LLMs, copy the example environment file and add your API keys. This file should be in the **project root directory**. +The compose file loads `.llm.env` from the **project root directory** — the +file must exist even if you don't use LLMs, or compose will fail with +"env file .llm.env not found". Create it from the example and add an API token: ```bash # Make sure you are in the 'crawl4ai' root directory cp deploy/docker/.llm.env.example .llm.env +``` + +Then open `.llm.env` and fill in the `CRAWL4AI_API_TOKEN=` line at the top — +**required**, or the server will be unreachable (loopback-only). Any long +random string works, e.g. from `openssl rand -hex 32`. One-liner: -# Now edit .llm.env and add your API keys +```bash +sed -i.bak "s|^CRAWL4AI_API_TOKEN=.*|CRAWL4AI_API_TOKEN=$(openssl rand -hex 32)|" .llm.env && rm .llm.env.bak ``` +Optionally add your LLM API keys in the same file. + +> ⚠️ **The token must go inside `.llm.env`.** `export CRAWL4AI_API_TOKEN=...` +> in your shell does **not** work with compose — the compose file does not +> forward host environment variables, and the server silently starts in +> loopback-only mode (published port → connection reset). + **Flexible LLM Provider Configuration:** The Docker setup now supports flexible LLM provider configuration through a hierarchical system: @@ -249,7 +303,9 @@ The `docker-compose.yml` file in the project root provides a simplified approach ENABLE_GPU=true docker compose up --build -d ``` -> The server will be available at `http://localhost:11235`. +> The server will be available at `http://localhost:11235` (allow ~10 seconds +> for startup). All endpoints except `GET /health` require +> `Authorization: Bearer `. #### 4. Stopping the Service @@ -287,12 +343,20 @@ docker buildx build \ #### 3. Run the Container +Set a token first (required — without it the server is loopback-only and the +published port answers with connection reset): + +```bash +export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)" +``` + * **Basic run (no LLM support):** ```bash docker run -d \ -p 11235:11235 \ --name crawl4ai-standalone \ --shm-size=1g \ + -e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN" \ crawl4ai-local:latest ``` @@ -304,10 +368,13 @@ docker buildx build \ --name crawl4ai-standalone \ --env-file .llm.env \ --shm-size=1g \ + -e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN" \ crawl4ai-local:latest ``` -> The server will be available at `http://localhost:11235`. +> The server will be available at `http://localhost:11235` (allow ~10 seconds +> for startup). All endpoints except `GET /health` require +> `Authorization: Bearer $CRAWL4AI_API_TOKEN`. #### 4. Stopping the Manual Container @@ -402,13 +469,32 @@ Captures a full-page PNG screenshot of the specified URL. ```json { "url": "https://example.com", - "screenshot_wait_for": 2, - "output_path": "/path/to/save/screenshot.png" + "screenshot_wait_for": 2 } ``` - `screenshot_wait_for`: Optional delay in seconds before capture (default: 2) -- `output_path`: Optional path to save the screenshot (recommended) + +The response contains the image inline (base64) **and** an artifact id you can +fetch later: + +```json +{"success": true, "screenshot": "", + "artifact_id": "…", "url": "/artifacts/…", "mime": "image/png", "size": 16668} +``` + +Download the stored file with an authenticated request (artifacts have a TTL +and a storage quota): + +```bash +curl -H "Authorization: Bearer $CRAWL4AI_API_TOKEN" \ + -o screenshot.png http://localhost:11235/artifacts/ +``` + +> **⚠️ Changed in 0.9.0:** `output_path` was removed (server-side file writes +> were a path-traversal risk). A request that still includes `output_path` +> currently returns `success: true` but **silently ignores the field** — no +> file is written. Use the artifact flow above instead. ### PDF Export Endpoint @@ -420,12 +506,13 @@ Generates a PDF document of the specified URL. ```json { - "url": "https://example.com", - "output_path": "/path/to/save/document.pdf" + "url": "https://example.com" } ``` -- `output_path`: Optional path to save the PDF (recommended) +Like `/screenshot`, the response returns the document plus an `artifact_id`; +fetch the file via `GET /artifacts/{artifact_id}` with your Bearer token. +`output_path` was removed in 0.9.0 and is silently ignored if sent. ### JavaScript Execution Endpoint @@ -449,684 +536,112 @@ Executes JavaScript snippets on the specified URL and returns the full crawl res --- -## User-Provided Hooks API - -> **⚠️ Changed in 0.9.0.** The inline-Python hook API described below was removed. -> Sending arbitrary Python code to the server is no longer accepted (it was an -> unauthenticated code-execution surface). 0.9.0 replaces it with **declarative -> hooks**: a fixed set of safe, server-validated actions (for example -> `add_cookies`, `set_headers`, `block_resources`) supplied as JSON, with no code -> execution. See the [migration guide](https://github.com/unclecode/crawl4ai/blob/main/deploy/docker/MIGRATION.md) -> for the declarative hook format. The inline-code examples in this section apply -> to 0.8.x only and are kept for reference until this page is fully rewritten. +## Hooks (Declarative Actions) -The Docker API supports user-provided hook functions, allowing you to customize the crawling behavior by injecting your own Python code at specific points in the crawling pipeline. This powerful feature enables authentication, performance optimization, and custom content extraction without modifying the server code. +> **⚠️ Changed in 0.9.0.** The previous hooks API — sending Python code strings in +> `hooks.code` — was removed. It was an unauthenticated remote-code-execution +> surface. The server now accepts **declarative hooks**: a fixed set of safe, +> server-validated actions supplied as JSON. No code execution. If you need +> arbitrary hook code, use the in-process Python SDK (`AsyncWebCrawler`), where +> you keep full control. -> ⚠️ **IMPORTANT SECURITY WARNING**: -> - **Never use hooks with untrusted code or on untrusted websites** -> - **Be extremely careful when crawling sites that might be phishing or malicious** -> - **Hook code has access to page context and can interact with the website** -> - **Always validate and sanitize any data extracted through hooks** -> - **Never expose credentials or sensitive data in hook code** -> - **Consider running the Docker container in an isolated network when testing** - -### Hook Information Endpoint - -``` -GET /hooks/info -``` +### Enabling hooks -Returns information about available hook points and their signatures: +Hooks are **disabled by default**. Enable them with an environment variable when +starting the container: ```bash -curl http://localhost:11235/hooks/info +docker run -d \ + -p 11235:11235 \ + --name crawl4ai \ + --shm-size=1g \ + -e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN" \ + -e CRAWL4AI_HOOKS_ENABLED=true \ + unclecode/crawl4ai:latest ``` -### Available Hook Points - -The API supports 8 hook points that match the local SDK: - -| Hook Point | Parameters | Description | Best Use Cases | -|------------|------------|-------------|----------------| -| `on_browser_created` | `browser` | After browser instance creation | Light setup tasks | -| `on_page_context_created` | `page, context` | After page/context creation | **Authentication, cookies, route blocking** | -| `before_goto` | `page, context, url` | Before navigating to URL | Custom headers, logging | -| `after_goto` | `page, context, url, response` | After navigation completes | Verification, waiting for elements | -| `on_user_agent_updated` | `page, context, user_agent` | When user agent changes | UA-specific logic | -| `on_execution_started` | `page, context` | When JS execution begins | JS-related setup | -| `before_retrieve_html` | `page, context` | Before getting final HTML | **Scrolling, lazy loading** | -| `before_return_html` | `page, context, html` | Before returning HTML | Final modifications, metrics | - -### Using Hooks in Requests - -Add hooks to any crawl request by including the `hooks` parameter: +Without the flag, any request containing `hooks` returns: ```json -{ - "urls": ["https://httpbin.org/html"], - "hooks": { - "code": { - "hook_point_name": "async def hook(...): ...", - "another_hook": "async def hook(...): ..." - }, - "timeout": 30 // Optional, default 30 seconds (max 120) - } -} +{"detail": "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable."} ``` -### Hook Examples with Real URLs +### Available actions -#### 1. Authentication with Cookies (GitHub) +| Action | Hook point | Description | +|--------|-----------|-------------| +| `block_resources` | `on_page_context_created` | Abort matching resource types (`image`, `stylesheet`, `font`, `media`) | +| `add_cookies` | `on_page_context_created` | Add cookies to the browser context before navigation (auth) | +| `set_headers` | `before_goto` | Set extra HTTP request headers before navigating | +| `scroll_to_bottom` | `before_retrieve_html` | Scroll to the page bottom in bounded steps (lazy-load), `max_steps` 1–50, `delay_ms` 0–5000 | +| `wait_for_timeout` | `before_retrieve_html` | Wait a bounded number of milliseconds (0–60000) before retrieving HTML | -```python -import requests +Get the full parameter schemas at runtime: -# Example: Setting GitHub session cookie (use your actual session) -hooks_code = { - "on_page_context_created": """ -async def hook(page, context, **kwargs): - # Add authentication cookies for GitHub - # WARNING: Never hardcode real credentials! - await context.add_cookies([ - { - 'name': 'user_session', - 'value': 'your_github_session_token', # Replace with actual token - 'domain': '.github.com', - 'path': '/', - 'httpOnly': True, - 'secure': True, - 'sameSite': 'Lax' - } - ]) - return page -""" -} - -response = requests.post("http://localhost:11235/crawl", json={ - "urls": ["https://github.com/settings/profile"], # Protected page - "hooks": {"code": hooks_code, "timeout": 30} -}) -``` - -#### 2. Basic Authentication (httpbin.org for testing) - -```python -# Safe testing with httpbin.org (a service designed for HTTP testing) -hooks_code = { - "before_goto": """ -async def hook(page, context, url, **kwargs): - import base64 - # httpbin.org/basic-auth expects username="user" and password="passwd" - credentials = base64.b64encode(b"user:passwd").decode('ascii') - - await page.set_extra_http_headers({ - 'Authorization': f'Basic {credentials}' - }) - return page -""" -} - -response = requests.post("http://localhost:11235/crawl", json={ - "urls": ["https://httpbin.org/basic-auth/user/passwd"], - "hooks": {"code": hooks_code, "timeout": 15} -}) -``` - -#### 3. Performance Optimization (News Sites) - -```python -# Example: Optimizing crawling of news sites like CNN or BBC -hooks_code = { - "on_page_context_created": """ -async def hook(page, context, **kwargs): - # Block images, fonts, and media to speed up crawling - await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort()) - await context.route("**/*.{woff,woff2,ttf,otf,eot}", lambda route: route.abort()) - await context.route("**/*.{mp4,webm,ogg,mp3,wav,flac}", lambda route: route.abort()) - - # Block common tracking and ad domains - await context.route("**/googletagmanager.com/*", lambda route: route.abort()) - await context.route("**/google-analytics.com/*", lambda route: route.abort()) - await context.route("**/doubleclick.net/*", lambda route: route.abort()) - await context.route("**/facebook.com/tr/*", lambda route: route.abort()) - await context.route("**/amazon-adsystem.com/*", lambda route: route.abort()) - - # Disable CSS animations for faster rendering - await page.add_style_tag(content=''' - *, *::before, *::after { - animation-duration: 0s !important; - transition-duration: 0s !important; - } - ''') - - return page -""" -} - -response = requests.post("http://localhost:11235/crawl", json={ - "urls": ["https://www.bbc.com/news"], # Heavy news site - "hooks": {"code": hooks_code, "timeout": 30} -}) +```bash +curl -H "Authorization: Bearer $CRAWL4AI_API_TOKEN" \ + http://localhost:11235/hooks/info ``` -#### 4. Handling Infinite Scroll (Twitter/X) +### Using hooks in a request -```python -# Example: Scrolling on Twitter/X (requires authentication) -hooks_code = { - "before_retrieve_html": """ -async def hook(page, context, **kwargs): - # Scroll to load more tweets - previous_height = 0 - for i in range(5): # Limit scrolls to avoid infinite loop - current_height = await page.evaluate("document.body.scrollHeight") - if current_height == previous_height: - break # No more content to load - - await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") - await page.wait_for_timeout(2000) # Wait for content to load - previous_height = current_height - - return page -""" -} +Add a `hooks` object with a list of actions (maximum 10 per request): -# Note: Twitter requires authentication for most content -response = requests.post("http://localhost:11235/crawl", json={ - "urls": ["https://twitter.com/nasa"], # Public profile - "hooks": {"code": hooks_code, "timeout": 30} -}) -``` - -#### 5. E-commerce Login (Example Pattern) - -```python -# SECURITY WARNING: This is a pattern example. -# Never use real credentials in code! -# Always use environment variables or secure vaults. - -hooks_code = { - "on_page_context_created": """ -async def hook(page, context, **kwargs): - # Example pattern for e-commerce sites - # DO NOT use real credentials here! - - # Navigate to login page first - await page.goto("https://example-shop.com/login") - - # Wait for login form to load - await page.wait_for_selector("#email", timeout=5000) - - # Fill login form (use environment variables in production!) - await page.fill("#email", "test@example.com") # Never use real email - await page.fill("#password", "test_password") # Never use real password - - # Handle "Remember Me" checkbox if present - try: - await page.uncheck("#remember_me") # Don't remember on shared systems - except: - pass - - # Submit form - await page.click("button[type='submit']") - - # Wait for redirect after login - await page.wait_for_url("**/account/**", timeout=10000) - - return page -""" -} +```bash +curl -X POST http://localhost:11235/crawl \ + -H "Authorization: Bearer $CRAWL4AI_API_TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{ + "urls": ["https://example.com"], + "hooks": { + "hooks": [ + {"action": "block_resources", "params": {"resource_types": ["image", "font"]}}, + {"action": "scroll_to_bottom", "params": {"max_steps": 10, "delay_ms": 500}} + ] + } + }' ``` -#### 6. Extracting Structured Data (Wikipedia) - -```python -# Safe example using Wikipedia -hooks_code = { - "after_goto": """ -async def hook(page, context, url, response, **kwargs): - # Wait for Wikipedia content to load - await page.wait_for_selector("#content", timeout=5000) - return page -""", - - "before_retrieve_html": """ -async def hook(page, context, **kwargs): - # Extract structured data from Wikipedia infobox - metadata = await page.evaluate('''() => { - const infobox = document.querySelector('.infobox'); - if (!infobox) return null; - - const data = {}; - const rows = infobox.querySelectorAll('tr'); - - rows.forEach(row => { - const header = row.querySelector('th'); - const value = row.querySelector('td'); - if (header && value) { - data[header.innerText.trim()] = value.innerText.trim(); - } - }); - - return data; - }''') - - if metadata: - print("Extracted metadata:", metadata) - - return page -""" -} +The response reports what was attached and executed: -response = requests.post("http://localhost:11235/crawl", json={ - "urls": ["https://en.wikipedia.org/wiki/Python_(programming_language)"], - "hooks": {"code": hooks_code, "timeout": 20} -}) +```json +{"success": true, "results": [...], "hooks": {"status": "success", "attached": ["before_retrieve_html"]}} ``` -### Security Best Practices - -> 🔒 **Critical Security Guidelines**: - -1. **Never Trust User Input**: If accepting hook code from users, always validate and sandbox it -2. **Avoid Phishing Sites**: Never use hooks on suspicious or unverified websites -3. **Protect Credentials**: - - Never hardcode passwords, tokens, or API keys in hook code - - Use environment variables or secure secret management - - Rotate credentials regularly -4. **Network Isolation**: Run the Docker container in an isolated network when testing -5. **Audit Hook Code**: Always review hook code before execution -6. **Limit Permissions**: Use the least privileged access needed -7. **Monitor Execution**: Check hook execution logs for suspicious behavior -8. **Timeout Protection**: Always set reasonable timeouts (default 30s) - -### Hook Response Information - -When hooks are used, the response includes detailed execution information: +Cookie example (authentication): ```json { - "success": true, - "results": [...], + "urls": ["https://example.com/account"], "hooks": { - "status": { - "status": "success", // or "partial" or "failed" - "attached_hooks": ["on_page_context_created", "before_retrieve_html"], - "validation_errors": [], - "successfully_attached": 2, - "failed_validation": 0 - }, - "execution_log": [ - { - "hook_point": "on_page_context_created", - "status": "success", - "execution_time": 0.523, - "timestamp": 1234567890.123 - } - ], - "errors": [], // Any runtime errors - "summary": { - "total_executions": 2, - "successful": 2, - "failed": 0, - "timed_out": 0, - "success_rate": 100.0 - } + "hooks": [ + {"action": "add_cookies", "params": {"cookies": [ + {"name": "session", "value": "your-session-token", + "domain": ".example.com", "path": "/", "secure": true} + ]}}, + {"action": "set_headers", "params": {"headers": {"Accept-Language": "en-US"}}} + ] } } ``` -### Error Handling - -The hooks system is designed to be resilient: - -1. **Validation Errors**: Caught before execution (syntax errors, wrong parameters) -2. **Runtime Errors**: Handled gracefully - crawl continues with original page object -3. **Timeout Protection**: Hooks automatically terminated after timeout (configurable 1-120s) - -### Complete Example: Safe Multi-Hook Crawling - -```python -import requests -import json -import os - -# Safe example using httpbin.org for testing -hooks_code = { - "on_page_context_created": """ -async def hook(page, context, **kwargs): - # Set viewport and test cookies - await page.set_viewport_size({"width": 1920, "height": 1080}) - await context.add_cookies([ - {"name": "test_cookie", "value": "test_value", "domain": ".httpbin.org", "path": "/"} - ]) - - # Block unnecessary resources for httpbin - await context.route("**/*.{png,jpg,jpeg}", lambda route: route.abort()) - return page -""", - - "before_goto": """ -async def hook(page, context, url, **kwargs): - # Add custom headers for testing - await page.set_extra_http_headers({ - "X-Test-Header": "crawl4ai-test", - "Accept-Language": "en-US,en;q=0.9" - }) - print(f"[HOOK] Navigating to: {url}") - return page -""", - - "before_retrieve_html": """ -async def hook(page, context, **kwargs): - # Simple scroll for any lazy-loaded content - await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") - await page.wait_for_timeout(1000) - return page -""" -} - -# Make the request to safe testing endpoints -response = requests.post("http://localhost:11235/crawl", json={ - "urls": [ - "https://httpbin.org/html", - "https://httpbin.org/json" - ], - "hooks": { - "code": hooks_code, - "timeout": 30 - }, - "crawler_config": { - "cache_mode": "bypass" - } -}) - -# Check results -if response.status_code == 200: - data = response.json() - - # Check hook execution - if data['hooks']['status']['status'] == 'success': - print(f"✅ All {len(data['hooks']['status']['attached_hooks'])} hooks executed successfully") - print(f"Execution stats: {data['hooks']['summary']}") - - # Process crawl results - for result in data['results']: - print(f"Crawled: {result['url']} - Success: {result['success']}") -else: - print(f"Error: {response.status_code}") -``` - -> 💡 **Remember**: Always test your hooks on safe, known websites first before using them on production sites. Never crawl sites that you don't have permission to access or that might be malicious. - -### Hooks Utility: Function-Based Approach (Python) - -For Python developers, Crawl4AI provides a more convenient way to work with hooks using the `hooks_to_string()` utility function and Docker client integration. - -#### Why Use Function-Based Hooks? - -**String-Based Approach (shown above)**: -```python -hooks_code = { - "on_page_context_created": """ -async def hook(page, context, **kwargs): - await page.set_viewport_size({"width": 1920, "height": 1080}) - return page -""" -} -``` - -**Function-Based Approach (recommended for Python)**: -```python -from crawl4ai import Crawl4aiDockerClient - -async def my_hook(page, context, **kwargs): - await page.set_viewport_size({"width": 1920, "height": 1080}) - return page - -async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client: - result = await client.crawl( - ["https://example.com"], - hooks={"on_page_context_created": my_hook} - ) -``` - -**Benefits**: -- ✅ Write hooks as regular Python functions -- ✅ Full IDE support (autocomplete, syntax highlighting, type checking) -- ✅ Easy to test and debug -- ✅ Reusable hook libraries -- ✅ Automatic conversion to API format - -#### Using the Hooks Utility - -The `hooks_to_string()` utility converts Python function objects to the string format required by the API: - -```python -from crawl4ai import hooks_to_string - -# Define your hooks as functions -async def setup_hook(page, context, **kwargs): - await page.set_viewport_size({"width": 1920, "height": 1080}) - await context.add_cookies([{ - "name": "session", - "value": "token", - "domain": ".example.com" - }]) - return page - -async def scroll_hook(page, context, **kwargs): - await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") - return page - -# Convert to string format -hooks_dict = { - "on_page_context_created": setup_hook, - "before_retrieve_html": scroll_hook -} -hooks_string = hooks_to_string(hooks_dict) - -# Now use with REST API or Docker client -# hooks_string contains the string representations -``` - -#### Docker Client with Automatic Conversion - -The Docker client automatically detects and converts function objects: - -```python -from crawl4ai import Crawl4aiDockerClient - -async def auth_hook(page, context, **kwargs): - """Add authentication cookies""" - await context.add_cookies([{ - "name": "auth_token", - "value": "your_token", - "domain": ".example.com" - }]) - return page - -async def performance_hook(page, context, **kwargs): - """Block unnecessary resources""" - await context.route("**/*.{png,jpg,gif}", lambda r: r.abort()) - await context.route("**/analytics/*", lambda r: r.abort()) - return page - -async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client: - # Pass functions directly - automatic conversion! - result = await client.crawl( - ["https://example.com"], - hooks={ - "on_page_context_created": performance_hook, - "before_goto": auth_hook - }, - hooks_timeout=30 # Optional timeout in seconds (1-120) - ) - - print(f"Success: {result.success}") - print(f"HTML: {len(result.html)} chars") -``` - -#### Creating Reusable Hook Libraries - -Build collections of reusable hooks: - -```python -# hooks_library.py -class CrawlHooks: - """Reusable hook collection for common crawling tasks""" - - @staticmethod - async def block_images(page, context, **kwargs): - """Block all images to speed up crawling""" - await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda r: r.abort()) - return page - - @staticmethod - async def block_analytics(page, context, **kwargs): - """Block analytics and tracking scripts""" - tracking_domains = [ - "**/google-analytics.com/*", - "**/googletagmanager.com/*", - "**/facebook.com/tr/*", - "**/doubleclick.net/*" - ] - for domain in tracking_domains: - await context.route(domain, lambda r: r.abort()) - return page - - @staticmethod - async def scroll_infinite(page, context, **kwargs): - """Handle infinite scroll to load more content""" - previous_height = 0 - for i in range(5): # Max 5 scrolls - current_height = await page.evaluate("document.body.scrollHeight") - if current_height == previous_height: - break - await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") - await page.wait_for_timeout(1000) - previous_height = current_height - return page - - @staticmethod - async def wait_for_dynamic_content(page, context, url, response, **kwargs): - """Wait for dynamic content to load""" - await page.wait_for_timeout(2000) - try: - # Click "Load More" if present - load_more = await page.query_selector('[class*="load-more"]') - if load_more: - await load_more.click() - await page.wait_for_timeout(1000) - except: - pass - return page - -# Use in your application -from hooks_library import CrawlHooks -from crawl4ai import Crawl4aiDockerClient - -async def crawl_with_optimizations(url): - async with Crawl4aiDockerClient() as client: - result = await client.crawl( - [url], - hooks={ - "on_page_context_created": CrawlHooks.block_images, - "before_retrieve_html": CrawlHooks.scroll_infinite - } - ) - return result -``` - -#### Choosing the Right Approach +### Migrating from 0.8.x `hooks.code` -| Approach | Best For | IDE Support | Language | -|----------|----------|-------------|----------| -| **String-based** | Non-Python clients, REST APIs, other languages | ❌ None | Any | -| **Function-based** | Python applications, local development | ✅ Full | Python only | -| **Docker Client** | Python apps with automatic conversion | ✅ Full | Python only | +Requests using the removed `hooks.code` (Python strings) format are **not +executed**. Be aware of the current behavior: -**Recommendation**: -- **Python applications**: Use Docker client with function objects (easiest) -- **Non-Python or REST API**: Use string-based hooks (most flexible) -- **Manual control**: Use `hooks_to_string()` utility (middle ground) +- With hooks disabled (default): HTTP 403 "Hooks are disabled…" — note that + enabling hooks will *not* make code hooks work. +- With hooks enabled: the request succeeds (HTTP 200) but the inline code is + **silently ignored** — the response shows `"hooks": {"status": "success", + "attached": []}`. If you see an empty `attached` list, your hooks did not run. -#### Complete Example with Function Hooks +Map your old hook code to declarative actions where possible (resource blocking, +cookies, headers, scrolling, waits). For anything beyond the fixed action set — +custom JavaScript, form logins, conditional logic — use the in-process Python +SDK, which retains the full 8-hook-point API described in the +[hooks documentation](../advanced/hooks-auth.md). -```python -from crawl4ai import Crawl4aiDockerClient, BrowserConfig, CrawlerRunConfig, CacheMode - -# Define hooks as regular Python functions -async def setup_environment(page, context, **kwargs): - """Setup crawling environment""" - # Set viewport - await page.set_viewport_size({"width": 1920, "height": 1080}) - - # Block resources for speed - await context.route("**/*.{png,jpg,gif}", lambda r: r.abort()) - - # Add custom headers - await page.set_extra_http_headers({ - "Accept-Language": "en-US", - "X-Custom-Header": "Crawl4AI" - }) - - print("[HOOK] Environment configured") - return page - -async def extract_content(page, context, **kwargs): - """Extract and prepare content""" - # Scroll to load lazy content - await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") - await page.wait_for_timeout(1000) - - # Extract metadata - metadata = await page.evaluate('''() => ({ - title: document.title, - links: document.links.length, - images: document.images.length - })''') - - print(f"[HOOK] Page metadata: {metadata}") - return page - -async def main(): - async with Crawl4aiDockerClient(base_url="http://localhost:11235", verbose=True) as client: - # Configure crawl - browser_config = BrowserConfig(headless=True) - crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS) - - # Crawl with hooks - result = await client.crawl( - ["https://httpbin.org/html"], - browser_config=browser_config, - crawler_config=crawler_config, - hooks={ - "on_page_context_created": setup_environment, - "before_retrieve_html": extract_content - }, - hooks_timeout=30 - ) - - if result.success: - print(f"✅ Crawl successful!") - print(f" URL: {result.url}") - print(f" HTML: {len(result.html)} chars") - print(f" Markdown: {len(result.markdown)} chars") - else: - print(f"❌ Crawl failed: {result.error_message}") - -if __name__ == "__main__": - import asyncio - asyncio.run(main()) -``` - -#### Additional Resources - -- **Comprehensive Examples**: See `/docs/examples/hooks_docker_client_example.py` for Python function-based examples -- **REST API Examples**: See `/docs/examples/hooks_rest_api_example.py` for string-based examples -- **Comparison Guide**: See `/docs/examples/README_HOOKS.md` for detailed comparison -- **Utility Documentation**: See `/docs/hooks-utility-guide.md` for complete guide --- @@ -1634,7 +1149,14 @@ Communicate with the running Docker server via its REST API (defaulting to `http ### Playground Interface -A built-in web playground is available at `http://localhost:11235/playground` for testing and generating API requests. The playground allows you to: +A built-in web playground is available at `http://localhost:11235/playground` for testing and generating API requests. + +> 🔑 Before running requests, paste your API token into the **API token** bar in +> the top navigation and click **Set** — otherwise every request returns +> `{"detail": "Authentication required"}` (note: the status banner may still +> show "Success" for such error responses; check the response body). + +The playground allows you to: 1. Configure `CrawlerRunConfig` and `BrowserConfig` using the main library's Python syntax 2. Test crawling operations directly from the interface @@ -1646,7 +1168,13 @@ This is the easiest way to translate Python configuration to JSON requests when Install the SDK: `pip install crawl4ai` -The Python SDK provides a convenient way to interact with the Docker API, including **automatic hook conversion** when using function objects. +The Python SDK provides a convenient way to interact with the Docker API. + +> **⚠️ Changed in 0.9.0:** the SDK's function-based hooks (`hooks={...}` with +> Python functions) no longer work against the Docker server — they were +> converted to code strings server-side, and request-supplied hook code was +> removed. Use [declarative hooks](#hooks-declarative-actions) over the REST +> API, or run the in-process SDK (`AsyncWebCrawler`) for full hook support. ```python import asyncio @@ -1687,25 +1215,6 @@ async def main(): except Exception as e: print(f"Streaming crawl failed: {e}") - # Example with hooks (Python function objects) - print("\n--- Crawl with Hooks ---") - - async def my_hook(page, context, **kwargs): - """Custom hook to optimize performance""" - await page.set_viewport_size({"width": 1920, "height": 1080}) - await context.route("**/*.{png,jpg}", lambda r: r.abort()) - print("[HOOK] Page optimized") - return page - - result = await client.crawl( - ["https://httpbin.org/html"], - browser_config=BrowserConfig(headless=True), - crawler_config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS), - hooks={"on_page_context_created": my_hook}, # Pass function directly! - hooks_timeout=30 - ) - print(f"Crawl with hooks success: {result.success}") - # Example Get schema print("\n--- Getting Schema ---") schema = await client.get_schema() @@ -1730,8 +1239,6 @@ The Docker client supports the following parameters: - `urls` (List[str]): List of URLs to crawl - `browser_config` (Optional[BrowserConfig]): Browser configuration - `crawler_config` (Optional[CrawlerRunConfig]): Crawler configuration -- `hooks` (Optional[Dict]): Hook functions or strings - **automatically converts function objects!** -- `hooks_timeout` (int): Timeout for each hook execution in seconds (default: 30) **Returns**: - Single URL: `CrawlResult` object @@ -1978,9 +1485,15 @@ One of the key advantages of self-hosting is complete visibility into your infra Access the **built-in real-time monitoring dashboard** for complete operational visibility: ``` -http://localhost:11235/monitor +http://localhost:11235/dashboard ``` +> ⚠️ The dashboard UI lives at `/dashboard` — **not** `/monitor`, which is the +> API namespace (`/monitor/health`, `/monitor/ws`, …) and returns +> `{"detail": "Authentication required"}` in a browser. On the dashboard, paste +> your API token into the **API token** bar (top right) and click **Set**; the +> WebSocket then connects and live stats appear. + ![Monitoring Dashboard](https://via.placeholder.com/800x400?text=Crawl4AI+Monitoring+Dashboard) **Dashboard Features:** @@ -2425,7 +1938,7 @@ Returns: ```json { "status": "healthy", - "version": "0.7.4" + "version": "0.9.2" } ``` @@ -2622,14 +2135,14 @@ By self-hosting Crawl4AI, you: **Next Steps:** 1. **Start Simple**: Deploy with Docker Hub image and test with the playground -2. **Monitor Everything**: Open `http://localhost:11235/monitor` to watch your server +2. **Monitor Everything**: Open `http://localhost:11235/dashboard` to watch your server 3. **Integrate**: Connect your applications using the Python SDK or REST API 4. **Scale Smart**: Use the monitoring data to optimize your deployment 5. **Go Production**: Set up alerting, log aggregation, and automated cleanup **Key Resources:** - 🎮 **Playground**: `http://localhost:11235/playground` - Interactive testing -- 📊 **Monitor Dashboard**: `http://localhost:11235/monitor` - Real-time visibility +- 📊 **Monitor Dashboard**: `http://localhost:11235/dashboard` - Real-time visibility - 📖 **Architecture Docs**: `deploy/docker/ARCHITECTURE.md` - Deep technical dive - 💬 **Discord Community**: Get help and share experiences - ⭐ **GitHub**: Report issues, contribute, show support From 588de042fcc86ec7017673b11e5c2151b5b4fad0 Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Sat, 1 Aug 2026 09:08:39 +0800 Subject: [PATCH 03/38] fix(docker): preserve single-url crawl failure details --- deploy/docker/api.py | 25 +++++++++++-------- .../docker/tests/test_api_crawl_failures.py | 24 ++++++++++++++++++ 2 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 deploy/docker/tests/test_api_crawl_failures.py diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 1756b925f..22e061096 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -92,6 +92,15 @@ def _attach_declarative_hooks(crawler, hooks_config: dict) -> dict: logger = logging.getLogger(__name__) + +def _raise_for_crawl_failure(result): + if not result.success: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=result.error_message, + ) + + # --- Helper to get memory --- def _get_memory_mb(): try: @@ -147,11 +156,7 @@ async def handle_llm_qa( enforce_egress(browser_cfg) crawler = await get_crawler(browser_cfg) result = await crawler.arun(url) - if not result.success: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=result.error_message - ) + _raise_for_crawl_failure(result) content = result.markdown.fit_markdown or result.markdown.raw_markdown # Create prompt and get LLM response @@ -179,6 +184,8 @@ async def handle_llm_qa( ) return response.choices[0].message.content + except HTTPException: + raise except LLMProviderNotAllowed as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: @@ -389,11 +396,7 @@ async def handle_markdown_request( ) ) - if not result.success: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=result.error_message - ) + _raise_for_crawl_failure(result) return (result.markdown.raw_markdown if filter_type == FilterType.RAW @@ -1023,4 +1026,4 @@ async def _runner(): except HTTPException: await redis.delete(f"task:{task_id}") raise - return {"task_id": task_id} \ No newline at end of file + return {"task_id": task_id} diff --git a/deploy/docker/tests/test_api_crawl_failures.py b/deploy/docker/tests/test_api_crawl_failures.py new file mode 100644 index 000000000..6c8fa817f --- /dev/null +++ b/deploy/docker/tests/test_api_crawl_failures.py @@ -0,0 +1,24 @@ +import inspect +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException + +from api import _raise_for_crawl_failure, handle_llm_qa, handle_markdown_request + + +def test_crawl_failure_is_reported_as_bad_gateway(): + result = SimpleNamespace(success=False, error_message="Blocked by anti-bot protection: challenge") + + with pytest.raises(HTTPException) as raised: + _raise_for_crawl_failure(result) + + assert raised.value.status_code == 502 + assert raised.value.detail == result.error_message + + +@pytest.mark.parametrize("handler", [handle_llm_qa, handle_markdown_request]) +def test_single_url_handlers_use_crawl_failure_mapping(handler): + source = inspect.getsource(handler) + assert "_raise_for_crawl_failure(result)" in source + assert handler is not handle_llm_qa or "except HTTPException:" in source From 05de127942e0b5255fa079bc4fbffef5466da39b Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Fri, 7 Aug 2026 23:11:40 +0800 Subject: [PATCH 04/38] fix(docker): support PDF scraping by default --- deploy/docker/api.py | 15 +++++++++----- deploy/docker/requirements.txt | 1 + tests/test_issue_2127_docker_pdf.py | 31 +++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 tests/test_issue_2127_docker_pdf.py diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 1756b925f..55d9bdeb6 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -865,15 +865,20 @@ async def handle_stream_crawl_request( # mirroring handle_crawl_request. The streaming path previously skipped # this, leaving /crawl/stream (and /crawl with stream=true) unguarded. urls = _normalize_and_validate_seeds(urls) - browser_config = BrowserConfig.load(browser_config, provenance=Provenance.UNTRUSTED) + browser_config = BrowserConfig.load( + browser_config, provenance=Provenance.UNTRUSTED + ) # browser_config.verbose = True # Set to False or remove for production stress testing browser_config.verbose = False from egress_broker import enforce_egress + enforce_egress(browser_config) - crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED) + crawler_config = CrawlerRunConfig.load( + crawler_config, provenance=Provenance.UNTRUSTED + ) from governor import clamp_deep_crawl + clamp_deep_crawl(crawler_config) - crawler_config.scraping_strategy = LXMLWebScrapingStrategy() crawler_config.stream = True # Deep crawl streaming supports exactly one start URL @@ -941,7 +946,7 @@ async def handle_stream_crawl_request( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) ) - + async def handle_crawl_job( redis, background_tasks: BackgroundTasks, @@ -1023,4 +1028,4 @@ async def _runner(): except HTTPException: await redis.delete(f"task:{task_id}") raise - return {"task_id": task_id} \ No newline at end of file + return {"task_id": task_id} diff --git a/deploy/docker/requirements.txt b/deploy/docker/requirements.txt index 212fcf036..e06cef78d 100644 --- a/deploy/docker/requirements.txt +++ b/deploy/docker/requirements.txt @@ -14,3 +14,4 @@ PyJWT==2.10.1 mcp>=1.18.0 websockets>=15.0.1 httpx[http2]>=0.27.2 +pypdf diff --git a/tests/test_issue_2127_docker_pdf.py b/tests/test_issue_2127_docker_pdf.py new file mode 100644 index 000000000..3fe0e5694 --- /dev/null +++ b/tests/test_issue_2127_docker_pdf.py @@ -0,0 +1,31 @@ +import ast +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def test_default_docker_dependencies_include_pypdf(): + requirements = ( + (ROOT / "deploy" / "docker" / "requirements.txt").read_text().splitlines() + ) + + assert "pypdf" in requirements + + +def test_stream_handler_preserves_requested_scraping_strategy(): + tree = ast.parse((ROOT / "deploy" / "docker" / "api.py").read_text()) + handler = next( + node + for node in tree.body + if isinstance(node, ast.AsyncFunctionDef) + and node.name == "handle_stream_crawl_request" + ) + assigned_attributes = { + target.attr + for node in ast.walk(handler) + if isinstance(node, ast.Assign) + for target in node.targets + if isinstance(target, ast.Attribute) + } + + assert "scraping_strategy" not in assigned_attributes From 0a78f071c0e58a289a808f4391048a9cc292fe9e Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Sat, 8 Aug 2026 11:09:05 +0800 Subject: [PATCH 05/38] fix(config): make body visibility timeout configurable --- crawl4ai/async_configs.py | 6 ++++++ crawl4ai/async_crawler_strategy.py | 2 +- tests/test_config_defaults.py | 6 +++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 27320fd4b..084060f9f 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -244,6 +244,7 @@ class UntrustedConfigError(ValueError): "fetch_ssl_certificate", # timing / waiting "wait_until", "page_timeout", "wait_for", "wait_for_timeout", + "body_visibility_timeout", "wait_for_images", "delay_before_return_html", "mean_delay", "max_range", # scrolling / rendering "ignore_body_visibility", "scan_full_page", "scroll_delay", @@ -1463,6 +1464,8 @@ class CrawlerRunConfig(): Default: False. ignore_body_visibility (bool): If True, ignore whether the body is visible before proceeding. Default: True. + body_visibility_timeout (int): Maximum time in ms to wait for the body to become visible. + Default: 30000. scan_full_page (bool): If True, scroll through the entire page to load all content. Default: False. scroll_delay (float): Delay in seconds between scroll steps if scan_full_page is True. @@ -1640,6 +1643,7 @@ def __init__( c4a_script: Union[str, List[str]] = None, js_only: bool = False, ignore_body_visibility: bool = True, + body_visibility_timeout: int = 30000, scan_full_page: bool = False, scroll_delay: float = 0.2, max_scroll_steps: Optional[int] = None, @@ -1770,6 +1774,7 @@ def __init__( self.c4a_script = c4a_script self.js_only = js_only self.ignore_body_visibility = ignore_body_visibility + self.body_visibility_timeout = body_visibility_timeout self.scan_full_page = scan_full_page self.scroll_delay = scroll_delay self.max_scroll_steps = max_scroll_steps @@ -2137,6 +2142,7 @@ def to_dict(self): "js_code_before_wait": self.js_code_before_wait, "js_only": self.js_only, "ignore_body_visibility": self.ignore_body_visibility, + "body_visibility_timeout": self.body_visibility_timeout, "scan_full_page": self.scan_full_page, "scroll_delay": self.scroll_delay, "max_scroll_steps": self.max_scroll_steps, diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index 265c376e9..9202124ab 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -823,7 +823,7 @@ async def handle_request_failed_capture(request): style.opacity !== '0'; return isVisible; }""", - timeout=30000, + timeout=config.body_visibility_timeout, ) if not is_visible and not config.ignore_body_visibility: diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.py index 700886aa9..93a9484ca 100644 --- a/tests/test_config_defaults.py +++ b/tests/test_config_defaults.py @@ -226,13 +226,17 @@ def test_dump_load_survives_reset(self): assert loaded.headless is False def test_crawler_run_config_dump_load(self): - CrawlerRunConfig.set_defaults(verbose=False, scan_full_page=True) + assert CrawlerRunConfig().body_visibility_timeout == 30000 + CrawlerRunConfig.set_defaults( + verbose=False, scan_full_page=True, body_visibility_timeout=2000 + ) cfg = CrawlerRunConfig() data = cfg.dump() CrawlerRunConfig.reset_defaults() loaded = CrawlerRunConfig.load(data) assert loaded.verbose is False assert loaded.scan_full_page is True + assert loaded.body_visibility_timeout == 2000 def test_to_dict_includes_user_default_values(self): BrowserConfig.set_defaults(headless=False) From 130a3829b28284be67c121c29b0304523c4bd3ad Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Tue, 11 Aug 2026 11:10:42 +0800 Subject: [PATCH 06/38] fix(docker): preserve failed crawl results --- deploy/docker/server.py | 3 -- .../tests/test_crawl_failure_response.py | 28 +++++++++++++++++++ tests/docker/test_server_requests.py | 6 ++-- 3 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 deploy/docker/tests/test_crawl_failure_response.py diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 410b8be38..ce1403b87 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -906,9 +906,6 @@ async def crawl( hooks_config=hooks_config, crawler_configs=crawl_request.crawler_configs, ) - # check if all of the results are not successful - if all(not result["success"] for result in results["results"]): - raise HTTPException(500, f"Crawl request failed: {results['results'][0]['error_message']}") return JSONResponse(results) diff --git a/deploy/docker/tests/test_crawl_failure_response.py b/deploy/docker/tests/test_crawl_failure_response.py new file mode 100644 index 000000000..188da944a --- /dev/null +++ b/deploy/docker/tests/test_crawl_failure_response.py @@ -0,0 +1,28 @@ +def test_all_failed_crawl_returns_results(stock_client, server_module, monkeypatch): + async def failed_crawl(**kwargs): + return { + "success": True, + "results": [ + { + "url": "https://example.com", + "success": False, + "error_message": "Wait condition failed: selector not found", + } + ], + } + + monkeypatch.setattr(server_module, "handle_crawl_request", failed_crawl) + + from auth import create_access_token + + token = create_access_token({"sub": "test@example.com"}) + response = stock_client.post( + "/crawl", + json={"urls": ["https://example.com"]}, + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 200 + result = response.json()["results"][0] + assert result["success"] is False + assert "Wait condition failed" in result["error_message"] diff --git a/tests/docker/test_server_requests.py b/tests/docker/test_server_requests.py index ae838c058..cb7bcbe4f 100644 --- a/tests/docker/test_server_requests.py +++ b/tests/docker/test_server_requests.py @@ -683,9 +683,9 @@ async def test_invalid_url_handling(self, async_client: httpx.AsyncClient): # Should return 200 with failed results, not 500 print(f"Status code: {response.status_code}") print(f"Response: {response.text}") - assert response.status_code == 500 + assert response.status_code == 200 data = response.json() - assert data["detail"].startswith("Crawl request failed:") + assert all(not result["success"] for result in data["results"]) async def test_mixed_success_failure_urls(self, async_client: httpx.AsyncClient): """Test handling of mixed success/failure URLs.""" @@ -887,4 +887,4 @@ async def test_malformed_request_handling(self, async_client: httpx.AsyncClient) # Execute pytest exit_code = pytest.main(pytest_args) - print(f"Pytest finished with exit code: {exit_code}") \ No newline at end of file + print(f"Pytest finished with exit code: {exit_code}") From 4fc16cae698e8389db0063c3e620a9b47b266d3f Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Tue, 11 Aug 2026 23:56:27 +0800 Subject: [PATCH 07/38] test(docker): exercise crawl failures through endpoints --- .../docker/tests/test_api_crawl_failures.py | 48 ++++++++++++++++--- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/deploy/docker/tests/test_api_crawl_failures.py b/deploy/docker/tests/test_api_crawl_failures.py index 6c8fa817f..04b7037c1 100644 --- a/deploy/docker/tests/test_api_crawl_failures.py +++ b/deploy/docker/tests/test_api_crawl_failures.py @@ -1,10 +1,9 @@ -import inspect from types import SimpleNamespace import pytest from fastapi import HTTPException -from api import _raise_for_crawl_failure, handle_llm_qa, handle_markdown_request +from api import _raise_for_crawl_failure def test_crawl_failure_is_reported_as_bad_gateway(): @@ -17,8 +16,43 @@ def test_crawl_failure_is_reported_as_bad_gateway(): assert raised.value.detail == result.error_message -@pytest.mark.parametrize("handler", [handle_llm_qa, handle_markdown_request]) -def test_single_url_handlers_use_crawl_failure_mapping(handler): - source = inspect.getsource(handler) - assert "_raise_for_crawl_failure(result)" in source - assert handler is not handle_llm_qa or "except HTTPException:" in source +@pytest.mark.parametrize( + ("method", "path", "payload"), + [ + ("post", "/md", {"url": "https://example.com", "f": "raw"}), + ("get", "/llm/example.com?q=summarize", None), + ], +) +def test_single_url_crawl_failure_reaches_client( + stock_client, server_module, monkeypatch, method, path, payload +): + error_message = "Blocked by anti-bot protection: challenge" + failed_result = SimpleNamespace(success=False, error_message=error_message) + + class FailedCrawler: + async def arun(self, *args, **kwargs): + return failed_result + + async def get_failed_crawler(*args, **kwargs): + return FailedCrawler() + + async def release_crawler(*args, **kwargs): + return None + + import api + import crawler_pool + from auth import create_access_token + + monkeypatch.setattr(api, "validate_url_destination", lambda url: None) + monkeypatch.setattr(crawler_pool, "get_crawler", get_failed_crawler) + monkeypatch.setattr(crawler_pool, "release_crawler", release_crawler) + + token = create_access_token({"sub": "test@example.com"}) + request = getattr(stock_client, method) + kwargs = {"json": payload} if payload is not None else {} + response = request( + path, headers={"Authorization": f"Bearer {token}"}, **kwargs + ) + + assert response.status_code == 502 + assert response.json() == {"detail": error_message} From 64bf678e13844e95cbd55a57c255fd627fb1b4f5 Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Tue, 11 Aug 2026 23:56:27 +0800 Subject: [PATCH 08/38] fix(config): validate body visibility timeout --- crawl4ai/async_configs.py | 8 ++++- docs/md_v2/api/parameters.md | 1 + docs/md_v2/complete-sdk-reference.md | 1 + tests/test_config_defaults.py | 45 ++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 084060f9f..3efb0d380 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -300,7 +300,7 @@ def _cap_timeout(v): return min(int(v), _MAX_TIMEOUT_MS) if type_name == "CrawlerRunConfig": - for f in ("page_timeout", "wait_for_timeout"): + for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"): if f in params: params[f] = _cap_timeout(params[f]) if isinstance(params.get("max_scroll_steps"), int): @@ -1774,6 +1774,12 @@ def __init__( self.c4a_script = c4a_script self.js_only = js_only self.ignore_body_visibility = ignore_body_visibility + if ( + not isinstance(body_visibility_timeout, (int, float)) + or isinstance(body_visibility_timeout, bool) + or body_visibility_timeout <= 0 + ): + raise ValueError("body_visibility_timeout must be a positive number") self.body_visibility_timeout = body_visibility_timeout self.scan_full_page = scan_full_page self.scroll_delay = scroll_delay diff --git a/docs/md_v2/api/parameters.md b/docs/md_v2/api/parameters.md index 568e14c30..f9f759a58 100644 --- a/docs/md_v2/api/parameters.md +++ b/docs/md_v2/api/parameters.md @@ -159,6 +159,7 @@ Use these for controlling whether you read or write from a local content cache. | **`c4a_script`** | `str or list[str]` (None) | C4A script that compiles to JavaScript. Alternative to writing raw JS. | | **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. | | **`ignore_body_visibility`** | `bool` (True) | Skip checking if `` is visible. Usually best to keep `True`. | +| **`body_visibility_timeout`** | `int` (30000) | Maximum time in milliseconds to wait for `` to become visible. Must be positive. | | **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). | | **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. | | **`max_scroll_steps`** | `int or None` (None) | Maximum number of scroll steps during full page scan. If None, scrolls until entire page is loaded. | diff --git a/docs/md_v2/complete-sdk-reference.md b/docs/md_v2/complete-sdk-reference.md index aa0517b2d..1f3299075 100644 --- a/docs/md_v2/complete-sdk-reference.md +++ b/docs/md_v2/complete-sdk-reference.md @@ -1791,6 +1791,7 @@ run_cfg = CrawlerRunConfig( | **`js_code_before_wait`** | `str or list[str]` (None) | JavaScript to run **before** `wait_for`. Use for triggering loading that `wait_for` then checks. | | **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. | | **`ignore_body_visibility`** | `bool` (True) | Skip checking if `` is visible. Usually best to keep `True`. | +| **`body_visibility_timeout`** | `int` (30000) | Maximum time in milliseconds to wait for `` to become visible. Must be positive. | | **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). | | **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. | | **`process_iframes`** | `bool` (False) | Inlines iframe content for single-page extraction. | diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.py index 93a9484ca..3a7fba641 100644 --- a/tests/test_config_defaults.py +++ b/tests/test_config_defaults.py @@ -1,7 +1,12 @@ """Tests for BrowserConfig.set_defaults / CrawlerRunConfig.set_defaults.""" +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + import pytest + from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig +from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy @pytest.fixture(autouse=True) @@ -238,6 +243,46 @@ def test_crawler_run_config_dump_load(self): assert loaded.scan_full_page is True assert loaded.body_visibility_timeout == 2000 + @pytest.mark.parametrize("timeout", [None, 0, -1, "1000", True]) + def test_body_visibility_timeout_must_be_positive_number(self, timeout): + with pytest.raises(ValueError, match="must be a positive number"): + CrawlerRunConfig(body_visibility_timeout=timeout) + + def test_untrusted_body_visibility_timeout_is_clamped(self): + from crawl4ai.async_configs import Provenance + + config = CrawlerRunConfig.load( + {"body_visibility_timeout": 500_000}, provenance=Provenance.UNTRUSTED + ) + assert config.body_visibility_timeout == 60_000 + + @pytest.mark.asyncio + async def test_body_visibility_timeout_reaches_wait(self): + page = MagicMock() + page.evaluate = AsyncMock() + page.set_content = AsyncMock() + page.wait_for_selector = AsyncMock() + page.content = AsyncMock(return_value="visible") + + strategy = AsyncPlaywrightCrawlerStrategy.__new__( + AsyncPlaywrightCrawlerStrategy + ) + strategy.browser_config = SimpleNamespace( + use_persistent_context=False, accept_downloads=False, text_mode=True + ) + strategy.browser_manager = SimpleNamespace( + get_page=AsyncMock(return_value=(page, MagicMock())) + ) + strategy.execute_hook = AsyncMock() + strategy.csp_compliant_wait = AsyncMock(return_value=True) + + config = CrawlerRunConfig( + session_id="body-timeout-test", body_visibility_timeout=1234 + ) + await strategy._crawl_web("raw:visible", config) + + assert strategy.csp_compliant_wait.await_args.kwargs["timeout"] == 1234 + def test_to_dict_includes_user_default_values(self): BrowserConfig.set_defaults(headless=False) cfg = BrowserConfig() From 10130de3d72b2fd2c593e17b2e52aa248493679f Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Wed, 12 Aug 2026 23:10:53 +0800 Subject: [PATCH 09/38] test(docker): address PDF review feedback Signed-off-by: nightcityblade --- deploy/docker/requirements.txt | 2 +- tests/test_issue_2127_docker_pdf.py | 80 +++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 21 deletions(-) diff --git a/deploy/docker/requirements.txt b/deploy/docker/requirements.txt index e06cef78d..7ae06a543 100644 --- a/deploy/docker/requirements.txt +++ b/deploy/docker/requirements.txt @@ -14,4 +14,4 @@ PyJWT==2.10.1 mcp>=1.18.0 websockets>=15.0.1 httpx[http2]>=0.27.2 -pypdf +pypdf>=6.0.0 diff --git a/tests/test_issue_2127_docker_pdf.py b/tests/test_issue_2127_docker_pdf.py index 3fe0e5694..1fe187622 100644 --- a/tests/test_issue_2127_docker_pdf.py +++ b/tests/test_issue_2127_docker_pdf.py @@ -1,31 +1,71 @@ -import ast +import importlib from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest +from packaging.requirements import InvalidRequirement, Requirement + +from crawl4ai.processors.pdf import PDFContentScrapingStrategy ROOT = Path(__file__).resolve().parent.parent def test_default_docker_dependencies_include_pypdf(): - requirements = ( - (ROOT / "deploy" / "docker" / "requirements.txt").read_text().splitlines() - ) + lines = (ROOT / "deploy" / "docker" / "requirements.txt").read_text().splitlines() + names = set() + for line in lines: + line = line.strip() + if not line or line.startswith(("#", "-")): + continue + try: + names.add(Requirement(line).name) + except InvalidRequirement: + continue - assert "pypdf" in requirements + assert "pypdf" in names -def test_stream_handler_preserves_requested_scraping_strategy(): - tree = ast.parse((ROOT / "deploy" / "docker" / "api.py").read_text()) - handler = next( - node - for node in tree.body - if isinstance(node, ast.AsyncFunctionDef) - and node.name == "handle_stream_crawl_request" - ) - assigned_attributes = { - target.attr - for node in ast.walk(handler) - if isinstance(node, ast.Assign) - for target in node.targets - if isinstance(target, ast.Attribute) +@pytest.mark.asyncio +async def test_stream_handler_preserves_requested_scraping_strategy(monkeypatch): + docker_dir = ROOT / "deploy" / "docker" + monkeypatch.syspath_prepend(str(docker_dir)) + + api = importlib.import_module("api") + crawler_pool = importlib.import_module("crawler_pool") + egress_broker = importlib.import_module("egress_broker") + governor = importlib.import_module("governor") + + crawler = MagicMock() + crawler.arun_many = AsyncMock(return_value=MagicMock()) + monkeypatch.setattr(api, "_normalize_and_validate_seeds", lambda urls: urls) + monkeypatch.setattr(egress_broker, "enforce_egress", lambda _: None) + monkeypatch.setattr(governor, "clamp_deep_crawl", lambda _: None) + monkeypatch.setattr(crawler_pool, "get_crawler", AsyncMock(return_value=crawler)) + + crawler_config = { + "type": "CrawlerRunConfig", + "params": { + "cache_mode": "bypass", + "stream": False, + "scraping_strategy": { + "type": "PDFContentScrapingStrategy", + "params": {"extract_images": False, "batch_size": 8}, + }, + }, + } + config = { + "crawler": { + "memory_threshold_percent": 90, + "rate_limiter": {"base_delay": [0.1, 0.3]}, + } } - assert "scraping_strategy" not in assigned_attributes + await api.handle_stream_crawl_request( + urls=["https://example.com/document.pdf"], + browser_config={"type": "BrowserConfig", "params": {}}, + crawler_config=crawler_config, + config=config, + ) + + effective_config = crawler.arun_many.await_args.kwargs["config"] + assert isinstance(effective_config.scraping_strategy, PDFContentScrapingStrategy) From 2e30e54d7580d816412d6c15afe9257b663f167b Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Wed, 12 Aug 2026 22:18:42 +0530 Subject: [PATCH 10/38] fix: Don't veto PDFCrawlerStrategy placeholder responses (#2135) PDFCrawlerStrategy returns stub html (real content comes from PDFContentScrapingStrategy), which the anti-bot checks misread as a block. Add AsyncCrawlResponse.placeholder_html so strategies can declare stub html and both anti-bot call sites skip it. --- crawl4ai/async_webcrawler.py | 5 +- crawl4ai/models.py | 1 + crawl4ai/processors/pdf/__init__.py | 12 ++- tests/test_placeholder_html_antibot.py | 134 +++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 tests/test_placeholder_html_antibot.py diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index 8216d19bc..a2aabcc41 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -505,7 +505,7 @@ async def arun( # Check if blocked (skip for raw: URLs — # caller-provided content, anti-bot N/A) - if _is_raw_url: + if _is_raw_url or async_response.placeholder_html: _blocked = False _block_reason = "" else: @@ -625,7 +625,8 @@ async def arun( # empty by design, and is_blocked() would misread "0 bytes # html" as a block. _has_download = bool(getattr(crawl_result, "downloaded_files", None)) - if not _fallback_succeeded and not _is_raw_url and not _has_download: + _placeholder = bool(getattr(async_response, "placeholder_html", False)) + if not _fallback_succeeded and not _is_raw_url and not _has_download and not _placeholder: _blocked, _block_reason = is_blocked( crawl_result.status_code, crawl_result.html or "") if _blocked: diff --git a/crawl4ai/models.py b/crawl4ai/models.py index 506538970..7ab3794cc 100644 --- a/crawl4ai/models.py +++ b/crawl4ai/models.py @@ -338,6 +338,7 @@ class AsyncCrawlResponse(BaseModel): mhtml_data: Optional[str] = None get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None downloaded_files: Optional[List[str]] = None + placeholder_html: bool = False # True if html is a stand-in for the content, actual content is produced by the scraping strategy (eg. using PDFCrawlerStrategy) ssl_certificate: Optional[SSLCertificate] = None redirected_url: Optional[str] = None redirected_status_code: Optional[int] = None diff --git a/crawl4ai/processors/pdf/__init__.py b/crawl4ai/processors/pdf/__init__.py index 69a6f75a2..ba61e946e 100644 --- a/crawl4ai/processors/pdf/__init__.py +++ b/crawl4ai/processors/pdf/__init__.py @@ -8,6 +8,15 @@ from .processor import NaivePDFProcessorStrategy # Assuming your current PDF code is in pdf_processor.py class PDFCrawlerStrategy(AsyncCrawlerStrategy): + """Crawler strategy for PDF documents. + + This strategy does not fetch or parse anything itself — it returns a + placeholder response (``placeholder_html=True``). It MUST be paired with + ``PDFContentScrapingStrategy`` (via ``CrawlerRunConfig.scraping_strategy``), + which performs the actual PDF download and content extraction. With any + other scraping strategy the result will contain only the placeholder text. + """ + def __init__(self, logger: AsyncLogger = None): self.logger = logger @@ -16,7 +25,8 @@ async def crawl(self, url: str, **kwargs) -> AsyncCrawlResponse: return AsyncCrawlResponse( html="Scraper will handle the real work", # Scraper will handle the real work response_headers={"Content-Type": "application/pdf"}, - status_code=200 + status_code=200, + placeholder_html=True, # HTML is a placeholder for the actual content, which will be produced by the PDFContentScrapingStrategy ) async def close(self): diff --git a/tests/test_placeholder_html_antibot.py b/tests/test_placeholder_html_antibot.py new file mode 100644 index 000000000..caed34a68 --- /dev/null +++ b/tests/test_placeholder_html_antibot.py @@ -0,0 +1,134 @@ +"""Regression tests for the anti-bot false positive on PDFCrawlerStrategy. + +PDFCrawlerStrategy returns a 33-byte placeholder as `html` (the real content +is produced later by PDFContentScrapingStrategy). The post-crawl anti-bot +veto used to read that placeholder and mark every PDF crawl as +"Blocked by anti-bot protection: Near-empty content", even though the PDF +was extracted successfully. The `placeholder_html` flag on AsyncCrawlResponse +lets a crawler strategy declare its html is a stand-in so the anti-bot +content heuristics skip it. +""" + +import asyncio + +import pytest + +from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig +from crawl4ai.antibot_detector import is_blocked +from crawl4ai.models import AsyncCrawlResponse + +PDF_TEXT = "Hello crawl4ai PDF fixture" + + +def _build_minimal_pdf(text: str = PDF_TEXT) -> bytes: + """Build a tiny single-page PDF containing `text`, valid for pypdf.""" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792]" + b" /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>", + None, # content stream, filled below + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + stream = f"BT /F1 18 Tf 72 720 Td ({text}) Tj ET".encode() + objects[3] = ( + b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + + stream + b"\nendstream" + ) + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode() + body + b"\nendobj\n" + xref_pos = len(out) + out += f"xref\n0 {len(objects) + 1}\n".encode() + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode() + out += b"trailer\n<< /Size " + str(len(objects) + 1).encode() + b" /Root 1 0 R >>\n" + out += b"startxref\n" + str(xref_pos).encode() + b"\n%%EOF\n" + return bytes(out) + + +@pytest.fixture +def pdf_file(tmp_path): + pytest.importorskip("pypdf", reason="requires the crawl4ai[pdf] extra") + path = tmp_path / "fixture.pdf" + path.write_bytes(_build_minimal_pdf()) + return path + + +def _run_pdf_crawl(pdf_file, **config_kwargs): + """Crawl the fixture PDF with the documented strategy pairing.""" + from crawl4ai.processors.pdf import ( + PDFContentScrapingStrategy, + PDFCrawlerStrategy, + ) + + async def run(): + config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + scraping_strategy=PDFContentScrapingStrategy(extract_images=False), + **config_kwargs, + ) + async with AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()) as crawler: + return await crawler.arun(f"file://{pdf_file}", config=config) + + return asyncio.run(run()) + + +def test_placeholder_html_defaults_false(): + """No strategy opts in implicitly — every existing strategy is unaffected.""" + response = AsyncCrawlResponse( + html="", response_headers={}, status_code=200 + ) + assert response.placeholder_html is False + + +def test_is_blocked_still_flags_near_empty_html(): + """The anti-bot heuristic itself is unchanged; only the PDF path opts out.""" + blocked, reason = is_blocked(200, "Scraper will handle the real work") + assert blocked + assert "Near-empty content" in reason + + +def test_pdf_pairing_is_not_vetoed_by_antibot(pdf_file): + """The documented PDFCrawlerStrategy + PDFContentScrapingStrategy pairing + (docs/md_v2/advanced/pdf-parsing.md) must report success, not an anti-bot + block, when extraction succeeds.""" + result = _run_pdf_crawl(pdf_file) + + assert result.success, f"crawl failed: {result.error_message}" + assert "anti-bot" not in (result.error_message or "") + markdown = ( + result.markdown.raw_markdown + if hasattr(result.markdown, "raw_markdown") + else result.markdown + ) + assert PDF_TEXT in (markdown or "") + + +def test_pdf_crawl_does_not_burn_retries_or_fallback(pdf_file): + """The attempt loop must not classify a placeholder response as blocked. + + Pre-fix, the phantom "blocked" verdict exhausted every retry attempt and + then invoked the fallback fetch, all on a crawl that had already + succeeded. With max_retries=2 a pre-fix run burns all 3 attempts and + calls the fallback; post-fix the first attempt resolves directly. + """ + fallback_calls = [] + + async def fake_fallback(url): + fallback_calls.append(url) + return "should not be used" + + result = _run_pdf_crawl( + pdf_file, max_retries=2, fallback_fetch_function=fake_fallback + ) + + stats = result.crawl_stats or {} + assert stats.get("resolved_by") == "direct" + assert stats.get("attempts") == 1 + assert all(not p.get("blocked") for p in stats.get("proxies_used", [])) + assert not fallback_calls + assert stats.get("fallback_fetch_used") is False From b536bf57faf5c5e4dc563da026e0380ca27f5f7e Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Thu, 13 Aug 2026 20:47:03 +0530 Subject: [PATCH 11/38] fix: remove unconditional setTimeout waits from overlay/consent removal scripts In-page timers never fire on CSP-sandboxed pages (GitHub/HuggingFace raw), hanging crawls 30s-to-forever; waits now run Python-side or only after a consent action actually fired. --- crawl4ai/async_crawler_strategy.py | 2 +- crawl4ai/js_snippet/remove_consent_popups.js | 9 +++++++-- .../js_snippet/remove_overlay_elements.js | 2 -- tests/regression/test_reg_browser.py | 20 +++++++++++++++++++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index 265c376e9..843463aa6 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -1534,7 +1534,7 @@ async def remove_overlay_elements(self, page: Page) -> None: }})() """ ) - await page.wait_for_timeout(500) # Wait for any animations to complete + await page.wait_for_timeout(600) # Wait for any animations to complete except Exception as e: self.logger.warning( message="Failed to remove overlay elements: {error}", diff --git a/crawl4ai/js_snippet/remove_consent_popups.js b/crawl4ai/js_snippet/remove_consent_popups.js index 9aac8d345..2f7eb0f34 100644 --- a/crawl4ai/js_snippet/remove_consent_popups.js +++ b/crawl4ai/js_snippet/remove_consent_popups.js @@ -292,6 +292,7 @@ async () => { // ========================================================================= // Phase 2: Try CMP JavaScript APIs // ========================================================================= + let apiCalled = false; // IAB TCF v2 API if (typeof window.__tcfapi === 'function') { @@ -304,6 +305,7 @@ async () => { if (typeof window.Didomi !== 'undefined') { try { window.Didomi.setUserAgreeToAll(); + apiCalled = true; } catch (e) { /* continue */ } } @@ -311,6 +313,7 @@ async () => { if (typeof window.Cookiebot !== 'undefined') { try { window.Cookiebot.submitCustomConsent(true, true, true); + apiCalled = true; } catch (e) { /* continue */ } } @@ -318,6 +321,7 @@ async () => { if (typeof window.Osano !== 'undefined') { try { window.Osano.cm.acceptAll(); + apiCalled = true; } catch (e) { /* continue */ } } @@ -325,11 +329,12 @@ async () => { if (typeof window.klaro !== 'undefined') { try { window.klaro.getManager().acceptAll(); + apiCalled = true; } catch (e) { /* continue */ } } - // Wait for CMP animations/transitions - await new Promise(r => setTimeout(r, 500)); + // Wait for CMP animations/transitions - only when a consent action actually fired + if (accepted || apiCalled) await new Promise(r => setTimeout(r, 500)); // ========================================================================= // Phase 3: Remove known CMP containers by selector diff --git a/crawl4ai/js_snippet/remove_overlay_elements.js b/crawl4ai/js_snippet/remove_overlay_elements.js index a50d94274..36de9d971 100644 --- a/crawl4ai/js_snippet/remove_overlay_elements.js +++ b/crawl4ai/js_snippet/remove_overlay_elements.js @@ -114,7 +114,5 @@ async () => { document.body.style.paddingRight = "0px"; document.body.style.overflow = "auto"; - // Wait a bit for any animations to complete document.body.scrollIntoView(false); - await new Promise((resolve) => setTimeout(resolve, 50)); }; diff --git a/tests/regression/test_reg_browser.py b/tests/regression/test_reg_browser.py index ba901178b..2d9f87fb4 100644 --- a/tests/regression/test_reg_browser.py +++ b/tests/regression/test_reg_browser.py @@ -289,6 +289,26 @@ async def test_remove_overlay_elements(local_server): assert len(result.html) > 0, "HTML should still be present after overlay removal" +@pytest.mark.asyncio +@pytest.mark.network +async def test_overlay_removal_on_csp_sandbox_page(): + """raw.githubusercontent.com serves CSP `sandbox`, which disables page + timers; overlay/consent removal must not stall waiting on in-page + setTimeout (used to hang ~30s per page). URL is commit-pinned so the + response never changes.""" + url = "https://raw.githubusercontent.com/unclecode/crawl4ai/055e2ecdc702228a80363e2c51ca79e473222072/README.md" + config = CrawlerRunConfig( + remove_overlay_elements=True, remove_consent_popups=True, verbose=False + ) + async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler: + start = time.perf_counter() + result = await crawler.arun(url=url, config=config) + elapsed = time.perf_counter() - start + assert result.success, f"Crawl failed on CSP-sandboxed page: {result.error_message}" + assert "Crawl4AI" in result.html, "Page content should be captured" + assert elapsed < 20, f"Overlay removal stalled on CSP-sandboxed page ({elapsed:.1f}s)" + + # --------------------------------------------------------------------------- # Stealth mode # --------------------------------------------------------------------------- From 890af3d4e6f866a0e1aedd9fd4207f8e4e9b53b8 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Sat, 15 Aug 2026 17:50:08 +0530 Subject: [PATCH 12/38] fix(docker): chain egress proxy through upstream HTTP(S)_PROXY Dial via the corporate proxy by CONNECT-to-the-pinned-IP when proxy env vars are set, restoring crawls on proxy-only hosts (discussion #2041) without weakening SSRF/rebinding guarantees. --- deploy/docker/README.md | 11 ++ deploy/docker/egress_proxy.py | 132 ++++++++++++++- .../tests/test_security_egress_proxy.py | 156 ++++++++++++++++++ 3 files changed, 292 insertions(+), 7 deletions(-) diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 7ce0dcd00..a57c857b9 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -116,6 +116,17 @@ EOL > The server will be available at `http://localhost:11235`. Visit `/playground` to access the interactive testing interface. +* **Behind a corporate proxy:** if the host reaches the internet only through + an HTTP proxy, set the standard `HTTP_PROXY` / `HTTPS_PROXY` / `NO_PROXY` + env vars (Docker's `proxies` config injects them automatically) — the + server's egress proxy chains through it while keeping its SSRF protections + (the upstream is asked to CONNECT to an already-validated, pinned IP). + `CRAWL4AI_UPSTREAM_PROXY` overrides the env vars. Basic auth via + `http://user:pass@proxy:port` is supported; for NTLM/Kerberos proxies, + front them with a local translator (e.g. `cntlm`, `px`) and point + `CRAWL4AI_UPSTREAM_PROXY` at it. Proxies that refuse CONNECT-to-an-IP, or + containers with no DNS at all, are not yet supported. + #### 4. Stopping the Container ```bash diff --git a/deploy/docker/egress_proxy.py b/deploy/docker/egress_proxy.py index 4cdb56bd1..1aa1bcc2f 100644 --- a/deploy/docker/egress_proxy.py +++ b/deploy/docker/egress_proxy.py @@ -13,13 +13,21 @@ against the real host - no MITM). Bound to 127.0.0.1 on an ephemeral port; started at server boot. + +If HTTP_PROXY/HTTPS_PROXY (or CRAWL4AI_UPSTREAM_PROXY) is set, we still +resolve-and-pin locally but dial via the upstream proxy, asking it to CONNECT +to the PINNED IP — never the hostname — so the rebinding guarantee holds. +NO_PROXY bypasses it; with no proxy env set, behavior is unchanged. """ from __future__ import annotations import asyncio +import base64 +import ipaddress import logging -from urllib.parse import urlsplit +import os +from urllib.parse import unquote, urlsplit from egress_broker import EgressBlocked, resolve_and_pin @@ -31,6 +39,63 @@ _MAX_HEADER_BYTES = 64 * 1024 +def _env(*names: str) -> str: + return next((os.environ[n] for n in names if os.environ.get(n)), "") + + +def upstream_proxy(scheme: str = "https"): + """(host, port, auth_header_bytes|None) of the upstream proxy, or None. + + Read per-call (not at import) so operators and tests see env changes. + The target scheme picks HTTP(S)_PROXY per convention, falling back to + the other pair when only one is set. + """ + order = ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") if scheme == "http" \ + else ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy") + raw = _env("CRAWL4AI_UPSTREAM_PROXY", *order).strip() + if not raw: + return None + sp = urlsplit(raw if "://" in raw else "http://" + raw) + if not sp.hostname: + return None + auth = None + if sp.username: + cred = f"{unquote(sp.username)}:{unquote(sp.password or '')}".encode("utf-8") + auth = b"Proxy-Authorization: Basic " + base64.b64encode(cred) + b"\r\n" + return sp.hostname, sp.port or 80, auth + + +def _no_proxy_match(host: str, ip: str) -> bool: + """True if NO_PROXY says this target must bypass the upstream proxy.""" + entries = [e.strip() for e in _env("NO_PROXY", "no_proxy").split(",") if e.strip()] + for entry in entries: + if entry == "*": + return True + try: + if ipaddress.ip_address(ip) in ipaddress.ip_network(entry, strict=False): + return True + continue + except ValueError: + pass + suffix = entry.lower().lstrip(".") + low = host.lower() + if low == suffix or low.endswith("." + suffix): + return True + return False + + +def _use_upstream(pin): + """The upstream (host, port, auth) to route `pin` through, or None for direct.""" + up = upstream_proxy(pin.scheme) + if up is None or _no_proxy_match(pin.host, pin.ip): + return None + return up + + +def _bracket(ip: str) -> str: + return f"[{ip}]" if ":" in ip else ip + + class PinningProxy: """Async HTTP forward-proxy that connects only to pinned, global IPs.""" @@ -52,6 +117,12 @@ async def start(self) -> str: sock = self._server.sockets[0] self.bound_host, self.bound_port = sock.getsockname()[:2] logger.info("egress pinning proxy listening on %s", self.url) + up = upstream_proxy() + if up is not None: + logger.info( + "egress pinning proxy chaining through upstream proxy %s:%s", + up[0], up[1], + ) return self.url async def stop(self) -> None: @@ -101,9 +172,7 @@ async def _handle_connect(self, target, client_reader, client_writer): await self._drain_headers(client_reader) try: - up_reader, up_writer = await asyncio.wait_for( - asyncio.open_connection(pin.ip, int(port_s)), timeout=30 - ) + up_reader, up_writer = await self._dial(pin, int(port_s)) except Exception: await self._reply(client_writer, _BLOCKED) return @@ -129,15 +198,31 @@ async def _handle_absolute(self, method, target, request_line, client_reader, cl path = sp.path or "/" if sp.query: path += "?" + sp.query + upstream = _use_upstream(pin) + dst = (upstream[0], upstream[1]) if upstream else (pin.ip, port) try: up_reader, up_writer = await asyncio.wait_for( - asyncio.open_connection(pin.ip, port), timeout=30 + asyncio.open_connection(*dst), timeout=30 ) except Exception: await self._reply(client_writer, _BLOCKED) return - # Re-issue in origin form with Host preserved. - out = f"{method} {path} HTTP/1.1\r\n".encode("latin-1") + # Re-issue with Host preserved: origin form when dialing the pinned IP + # directly, absolute form against the pinned IP when going through the + # upstream proxy (which then needs no DNS lookup of its own). + if upstream is None: + out = f"{method} {path} HTTP/1.1\r\n".encode("latin-1") + else: + out = f"{method} http://{_bracket(pin.ip)}:{port}{path} HTTP/1.1\r\n".encode("latin-1") + if upstream[2]: + out += upstream[2] + # One validated request per upstream connection: only this first + # request is pinned/rewritten, so force close to keep a reused + # client connection from smuggling unvalidated requests upstream. + headers = b"".join( + ln + b"\r\n" for ln in headers.split(b"\r\n") + if ln and not ln.lower().startswith(b"connection:") + ) + b"Connection: close\r\n" out += b"Host: " + sp.hostname.encode("latin-1") if sp.port: out += f":{sp.port}".encode("latin-1") @@ -147,6 +232,39 @@ async def _handle_absolute(self, method, target, request_line, client_reader, cl await self._splice(client_reader, client_writer, up_reader, up_writer) # ─────────────────────────── helpers ─────────────────────────── + async def _dial(self, pin, port: int): + """Open a byte pipe to the pinned IP: direct, or tunneled through the + upstream proxy via CONNECT-to-the-pinned-IP (no upstream DNS lookup).""" + upstream = _use_upstream(pin) + if upstream is None: + return await asyncio.wait_for( + asyncio.open_connection(pin.ip, port), timeout=30 + ) + p_host, p_port, auth = upstream + reader, writer = await asyncio.wait_for( + asyncio.open_connection(p_host, p_port), timeout=30 + ) + try: + dst = f"{_bracket(pin.ip)}:{port}" + req = f"CONNECT {dst} HTTP/1.1\r\nHost: {dst}\r\n".encode("latin-1") + if auth: + req += auth + req += b"\r\n" + writer.write(req) + await writer.drain() + status = await asyncio.wait_for(reader.readline(), timeout=30) + parts = status.split() + if len(parts) < 2 or parts[1] != b"200": + logger.warning("upstream proxy refused CONNECT: %r", status[:64]) + raise ConnectionError("upstream proxy refused CONNECT") + # Drain the upstream's response headers so none of them leak into + # the tunneled byte stream. + await self._drain_headers(reader) + except Exception: + await self._safe_close(writer) + raise + return reader, writer + async def _drain_headers(self, reader): read = 0 while True: diff --git a/deploy/docker/tests/test_security_egress_proxy.py b/deploy/docker/tests/test_security_egress_proxy.py index 03ebae1ea..f53767f35 100644 --- a/deploy/docker/tests/test_security_egress_proxy.py +++ b/deploy/docker/tests/test_security_egress_proxy.py @@ -20,6 +20,18 @@ pytestmark = pytest.mark.posture +_PROXY_ENV = ( + "CRAWL4AI_UPSTREAM_PROXY", "HTTP_PROXY", "http_proxy", + "HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy", +) + + +@pytest.fixture(autouse=True) +def _clear_proxy_env(monkeypatch): + # Keep the suite deterministic on dev machines that sit behind a proxy. + for name in _PROXY_ENV: + monkeypatch.delenv(name, raising=False) + async def _fake_upstream(): async def handle(reader, writer): @@ -121,6 +133,150 @@ async def test_malformed_connect_400(self): await proxy.stop() +async def _fake_corporate_proxy(seen): + """Minimal HTTP proxy: records the CONNECT request line, replies 200, then + answers any tunneled bytes with TUNNEL-OK.""" + async def handle(reader, writer): + line = await reader.readline() + seen.append(line) + while True: # drain CONNECT headers + h = await reader.readline() + if h in (b"\r\n", b"\n", b""): + break + writer.write(b"HTTP/1.1 200 Connection established\r\nVia: fake\r\n\r\n") + await writer.drain() + await reader.read(65536) + writer.write(b"TUNNEL-OK") + await writer.drain() + writer.close() + server = await asyncio.start_server(handle, "127.0.0.1", 0) + return server, server.sockets[0].getsockname()[1] + + +@pytest.mark.asyncio +class TestUpstreamChaining: + async def test_chained_connect_pins_ip_and_blocks_before_upstream(self, monkeypatch): + """The chained-CONNECT security contract: the upstream receives the + PINNED IP (never a hostname to resolve), its response headers do not + leak into the tunnel, and a blocked target produces an opaque 403 + with zero upstream traffic.""" + seen = [] + corp, corp_port = await _fake_corporate_proxy(seen) + monkeypatch.setenv("HTTPS_PROXY", f"http://127.0.0.1:{corp_port}") + + def fake_pin(url): + if "internal.example" in url: + raise EgressBlocked() + return PinnedTarget("https", "good.example", 443, "203.0.113.7") + monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin) + + proxy = PinningProxy() + await proxy.start() + try: + r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port) + w.write(b"CONNECT good.example:443 HTTP/1.1\r\n\r\n") + await w.drain() + status = await asyncio.wait_for(r.readline(), timeout=5) + assert b"200" in status + await r.readline() # blank line after the 200 + w.write(b"hello") + await w.drain() + body = await asyncio.wait_for(r.read(100), timeout=5) + # Upstream's Via header must NOT leak into the tunnel. + assert body == b"TUNNEL-OK" + w.close() + + r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port) + w.write(b"CONNECT internal.example:443 HTTP/1.1\r\n\r\n") + await w.drain() + status = await asyncio.wait_for(r.readline(), timeout=5) + assert b"403" in status + w.close() + finally: + await proxy.stop() + corp.close() + # The upstream saw ONLY the pinned IP of the allowed target. + assert seen == [b"CONNECT 203.0.113.7:443 HTTP/1.1\r\n"] + + async def test_chained_plain_http_pinned_absolute_form_no_smuggling(self, monkeypatch): + """Plain HTTP via upstream: the request is re-issued in absolute form + against the PINNED IP (no name for the upstream to resolve), carries + Connection: close, and a reused client connection cannot smuggle a + second, unvalidated request upstream.""" + lines = [] + + async def handle(reader, writer): + req = b"" + while b"\r\n\r\n" not in req: + req += await reader.read(4096) + lines.append(req) + writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nhi") + await writer.drain() + writer.close() + corp = await asyncio.start_server(handle, "127.0.0.1", 0) + corp_port = corp.sockets[0].getsockname()[1] + monkeypatch.setenv("HTTP_PROXY", f"http://127.0.0.1:{corp_port}") + + def fake_pin(url): + return PinnedTarget("http", "plain.example", 80, "203.0.113.7") + monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin) + + proxy = PinningProxy() + await proxy.start() + try: + r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port) + w.write(b"GET http://plain.example/ HTTP/1.1\r\n" + b"Host: plain.example\r\nConnection: keep-alive\r\n\r\n") + await w.drain() + first = await asyncio.wait_for(r.read(200), timeout=5) + assert b"200" in first + # Attempt to smuggle an unvalidated request on the same connection. + w.write(b"GET http://rebind.evil/ HTTP/1.1\r\nHost: rebind.evil\r\n\r\n") + await w.drain() + leftover = await asyncio.wait_for(r.read(200), timeout=5) + assert leftover == b"" # upstream closed; nothing came back + w.close() + finally: + await proxy.stop() + corp.close() + sent = b"".join(lines) + assert sent.startswith(b"GET http://203.0.113.7:80/ HTTP/1.1\r\n") + assert b"Connection: close" in sent + assert b"keep-alive" not in sent + assert b"rebind.evil" not in sent # the smuggled request never got upstream + + +def test_upstream_proxy_env_parsing(monkeypatch): + assert egress_proxy.upstream_proxy() is None + monkeypatch.setenv("HTTP_PROXY", "http://192.168.180.254:56560") + assert egress_proxy.upstream_proxy() == ("192.168.180.254", 56560, None) + monkeypatch.setenv("HTTPS_PROXY", "http://user:p%40ss@10.0.0.1:8080") + host, port, auth = egress_proxy.upstream_proxy() + assert (host, port) == ("10.0.0.1", 8080) + import base64 + assert base64.b64decode(auth.split(b" ")[-1].strip()) == b"user:p@ss" + # scheme-aware selection: http targets prefer HTTP_PROXY + assert egress_proxy.upstream_proxy("http") == ("192.168.180.254", 56560, None) + monkeypatch.setenv("CRAWL4AI_UPSTREAM_PROXY", "proxy.corp:3128") + assert egress_proxy.upstream_proxy() == ("proxy.corp", 3128, None) + # whitespace-only env var means unset, not a proxy named " " + monkeypatch.setenv("CRAWL4AI_UPSTREAM_PROXY", " ") + monkeypatch.delenv("HTTP_PROXY") + monkeypatch.delenv("HTTPS_PROXY") + assert egress_proxy.upstream_proxy() is None + # non-latin-1 credentials must not raise (encoded as UTF-8) + monkeypatch.delenv("CRAWL4AI_UPSTREAM_PROXY") + monkeypatch.setenv("HTTPS_PROXY", "http://u:%E5%AF%86%E7%A0%81@10.0.0.1:8080") + assert egress_proxy.upstream_proxy()[2] is not None + # NO_PROXY routing: suffix and CIDR entries force a direct dial + pin = PinnedTarget("https", "site.corp.example", 443, "203.0.113.7") + assert egress_proxy._use_upstream(pin) is not None + monkeypatch.setenv("NO_PROXY", ".corp.example") + assert egress_proxy._use_upstream(pin) is None + monkeypatch.setenv("NO_PROXY", "203.0.113.0/24") + assert egress_proxy._use_upstream(pin) is None + + class TestEnforceEgressWiring: def test_enforce_egress_sets_proxy(self, monkeypatch): import egress_broker From caa34f1ed6cd42f1eeb4812e9d8c9b0084205e89 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 17 Aug 2026 09:43:55 +0200 Subject: [PATCH 13/38] docs(pdf): document PDFContentScrapingStrategy pairing requirement PDFCrawlerStrategy returns a placeholder response and relies on the scraping strategy for the actual download and extraction. Since #2138 the placeholder is flagged with placeholder_html=True so the anti-bot heuristics skip it, which means an unpaired crawl now succeeds quietly with the placeholder text as content instead of failing loudly. State the pairing as required, document the placeholder_html flag, and note that custom strategies deferring extraction should set it too. --- docs/md_v2/advanced/pdf-parsing.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/md_v2/advanced/pdf-parsing.md b/docs/md_v2/advanced/pdf-parsing.md index 909c0dd17..3d2e7cf71 100644 --- a/docs/md_v2/advanced/pdf-parsing.md +++ b/docs/md_v2/advanced/pdf-parsing.md @@ -7,6 +7,8 @@ Crawl4AI provides specialized strategies for handling and extracting content fro ### Overview `PDFCrawlerStrategy` is an implementation of `AsyncCrawlerStrategy` designed specifically for PDF documents. Instead of interpreting the input URL as an HTML webpage, this strategy treats it as a pointer to a PDF file. It doesn't perform deep crawling or HTML parsing itself but rather prepares the PDF source for a dedicated PDF scraping strategy. Its primary role is to identify the PDF source (web URL or local file) and pass it along the processing pipeline in a way that `AsyncWebCrawler` can handle. +> **Must be paired with `PDFContentScrapingStrategy`.** `PDFCrawlerStrategy` fetches and parses nothing on its own — it returns a placeholder response and lets the scraping strategy do the download and extraction. Always set `CrawlerRunConfig(scraping_strategy=PDFContentScrapingStrategy())`. With any other scraping strategy the crawl still reports `success=True`, but the result contains only the placeholder text instead of your PDF content. + ### When to Use Use `PDFCrawlerStrategy` when you need to: - Process PDF files using the `AsyncWebCrawler`. @@ -20,8 +22,9 @@ Use `PDFCrawlerStrategy` when you need to: - **`async crawl(self, url: str, **kwargs) -> AsyncCrawlResponse`**: - This method is called by the `AsyncWebCrawler` during the `arun` process. - It takes the `url` (which should point to a PDF) and creates a minimal `AsyncCrawlResponse`. - - The `html` attribute of this response is typically empty or a placeholder, as the actual PDF content processing is deferred to the `PDFContentScrapingStrategy` (or a similar PDF-aware scraping strategy). + - The `html` attribute of this response is a short placeholder string, as the actual PDF content processing is deferred to the `PDFContentScrapingStrategy` (or a similar PDF-aware scraping strategy). - It sets `response_headers` to indicate "application/pdf" and `status_code` to 200. + - It also sets `placeholder_html=True` on the response. This tells `AsyncWebCrawler` that the `html` is a stand-in, so the anti-bot content heuristics skip it — otherwise the short placeholder would be misread as a blocked, near-empty page and the crawl would be marked failed. Any custom crawler strategy that defers content extraction to its scraping strategy should set the same flag. - **`async close(self)`**: - A method for cleaning up any resources used by the strategy. For `PDFCrawlerStrategy`, this is usually minimal. - **`async __aenter__(self)` / `async __aexit__(self, exc_type, exc_val, exc_tb)`**: @@ -37,8 +40,8 @@ async def main(): # Initialize the PDF crawler strategy pdf_crawler_strategy = PDFCrawlerStrategy() - # PDFCrawlerStrategy is typically used in conjunction with PDFContentScrapingStrategy - # The scraping strategy handles the actual PDF content extraction + # PDFCrawlerStrategy must be paired with PDFContentScrapingStrategy — + # the scraping strategy is what actually downloads and extracts the PDF pdf_scraping_strategy = PDFContentScrapingStrategy() run_config = CrawlerRunConfig(scraping_strategy=pdf_scraping_strategy) From 027b6f05b2d32124d8d9ec50826f1e762d408837 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 17 Aug 2026 10:17:49 +0200 Subject: [PATCH 14/38] fix(crawler): warn when the body-visibility wait times out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wait result is discarded when ignore_body_visibility is True (the default), so a page whose body never becomes visible — ng-cloak/v-cloak left behind by an app that failed to bootstrap — costs a flat body_visibility_timeout ms on every crawl while still reporting success=True with an empty error_message. Nothing was logged, so the delay could only be found by instrumenting the pipeline, and body_visibility_timeout (added in #2131) was undiscoverable by the users who most needed it. Warn once when the wait times out and its result is ignored, naming the option and the timeout that applied. The strict path (ignore_body_visibility=False) already raises with visibility details, so it stays quiet. Fixes #2144 --- crawl4ai/async_crawler_strategy.py | 15 +++++ tests/test_body_visibility_warning.py | 89 +++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 tests/test_body_visibility_warning.py diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index a65915c5d..d904ba81d 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -826,6 +826,21 @@ async def handle_request_failed_capture(request): timeout=config.body_visibility_timeout, ) + if not is_visible and config.ignore_body_visibility: + # The wait timed out and its result is about to be discarded, + # so the crawl still succeeds — just body_visibility_timeout ms + # slower. Say so, otherwise the delay is invisible (see #2144). + self.logger.warning( + message=( + "Body never became visible after {timeout}ms — the page may " + "use ng-cloak/v-cloak. This delay is added to every crawl of " + "this page; lower CrawlerRunConfig.body_visibility_timeout " + "to shorten it." + ), + tag="WARNING", + params={"timeout": config.body_visibility_timeout}, + ) + if not is_visible and not config.ignore_body_visibility: visibility_info = await self.check_visibility(page) raise Error(f"Body element is hidden: {visibility_info}") diff --git a/tests/test_body_visibility_warning.py b/tests/test_body_visibility_warning.py new file mode 100644 index 000000000..29e8a2893 --- /dev/null +++ b/tests/test_body_visibility_warning.py @@ -0,0 +1,89 @@ +"""The body-visibility wait must not time out silently (issue #2144). + +When the wait times out and `ignore_body_visibility` is True (the default), the +result is discarded and the crawl succeeds — just `body_visibility_timeout` ms +slower, with no signal at all. That silence is what made the delay in #2129 +impossible to attribute without instrumenting the pipeline, and what makes +`body_visibility_timeout` undiscoverable. A warning names the option. +""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from crawl4ai.async_configs import CrawlerRunConfig +from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy + + +def _strategy(is_visible: bool): + """A strategy whose body-visibility wait returns `is_visible`.""" + page = MagicMock() + page.evaluate = AsyncMock() + page.set_content = AsyncMock() + page.wait_for_selector = AsyncMock() + page.content = AsyncMock(return_value="hi") + + strategy = AsyncPlaywrightCrawlerStrategy.__new__(AsyncPlaywrightCrawlerStrategy) + strategy.browser_config = SimpleNamespace( + use_persistent_context=False, accept_downloads=False, text_mode=True, verbose=False + ) + strategy.browser_manager = SimpleNamespace( + get_page=AsyncMock(return_value=(page, MagicMock())) + ) + strategy.execute_hook = AsyncMock() + strategy.csp_compliant_wait = AsyncMock(return_value=is_visible) + strategy.check_visibility = AsyncMock(return_value={}) + strategy.logger = MagicMock() + return strategy + + +def _warnings(strategy): + return [ + call.kwargs.get("message", "") + for call in strategy.logger.warning.call_args_list + ] + + +@pytest.mark.asyncio +async def test_warns_when_wait_times_out_and_result_is_discarded(): + strategy = _strategy(is_visible=False) + config = CrawlerRunConfig( + session_id="body-vis-warn", body_visibility_timeout=1234 + ) + + await strategy._crawl_web("raw:hi", config) + + messages = _warnings(strategy) + assert any("never became visible" in m for m in messages), messages + assert any("body_visibility_timeout" in m for m in messages), messages + # The timeout that actually applied is reported, not the hardcoded default. + params = strategy.logger.warning.call_args.kwargs["params"] + assert params["timeout"] == 1234 + + +@pytest.mark.asyncio +async def test_no_warning_when_body_is_visible(): + strategy = _strategy(is_visible=True) + config = CrawlerRunConfig(session_id="body-vis-quiet") + + await strategy._crawl_web("raw:hi", config) + + assert not any("never became visible" in m for m in _warnings(strategy)) + + +@pytest.mark.asyncio +async def test_no_warning_when_hidden_body_is_treated_as_an_error(): + """With ignore_body_visibility=False the hidden body raises with details, + so the warning would be redundant noise.""" + from playwright.async_api import Error + + strategy = _strategy(is_visible=False) + config = CrawlerRunConfig( + session_id="body-vis-strict", ignore_body_visibility=False + ) + + with pytest.raises(Error, match="Body element is hidden"): + await strategy._crawl_web("raw:hi", config) + + assert not any("never became visible" in m for m in _warnings(strategy)) From 1bac351fa3894894795558c07ce1f3d7629de985 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 17 Aug 2026 10:44:06 +0200 Subject: [PATCH 15/38] fix(crawler): make the visibility warning survive verbose=False MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on three points: force_verbose — AsyncLogger drops a plain warning when verbose is off (async_logger.py:238), and CrawlerRunConfig.verbose overrides the logger at arun() time. Servers and batch jobs run with verbose off, so the warning was suppressed in exactly the deployments that hit this and can least afford a silent 30s. Verified end to end: with verbose=False the warning was invisible before, and prints now. False positives — csp_compliant_wait also returns False when the page evaluation itself fails (page closed, context destroyed by a redirect), which costs no time. Time the wait and only warn when it actually burned its budget, and report the elapsed time rather than the configured timeout so the number matches the delay being explained. Test — select the warning call by message instead of reading the last call, which only passed because text_mode=True happened to skip the wait_for_images warning. Added coverage for force_verbose and for the early-failure path. --- crawl4ai/async_crawler_strategy.py | 27 ++++++++--- tests/test_body_visibility_warning.py | 69 +++++++++++++++++++++------ 2 files changed, 75 insertions(+), 21 deletions(-) diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index d904ba81d..6d0fb4769 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -812,6 +812,7 @@ async def handle_request_failed_capture(request): await page.wait_for_selector("body", state="attached", timeout=30000) # Use the new check_visibility function with csp_compliant_wait + _visibility_wait_start = time.perf_counter() is_visible = await self.csp_compliant_wait( page, """() => { @@ -826,19 +827,33 @@ async def handle_request_failed_capture(request): timeout=config.body_visibility_timeout, ) - if not is_visible and config.ignore_body_visibility: - # The wait timed out and its result is about to be discarded, - # so the crawl still succeeds — just body_visibility_timeout ms - # slower. Say so, otherwise the delay is invisible (see #2144). + _visibility_wait_ms = ( + time.perf_counter() - _visibility_wait_start + ) * 1000 + + # csp_compliant_wait also returns False when the evaluation itself + # fails (page closed, context destroyed by a redirect), which costs + # no time — only warn about a wait that actually burned its budget. + if ( + not is_visible + and config.ignore_body_visibility + and _visibility_wait_ms >= config.body_visibility_timeout * 0.9 + ): + # The wait timed out and its result is about to be discarded, so + # the crawl still succeeds — just this much slower, on every crawl + # of this page. force_verbose because the whole point is that the + # delay is otherwise invisible, and the servers and batch jobs that + # most need to see it run with verbose off (see #2144). self.logger.warning( message=( - "Body never became visible after {timeout}ms — the page may " + "Body never became visible after {elapsed}ms — the page may " "use ng-cloak/v-cloak. This delay is added to every crawl of " "this page; lower CrawlerRunConfig.body_visibility_timeout " "to shorten it." ), tag="WARNING", - params={"timeout": config.body_visibility_timeout}, + params={"elapsed": round(_visibility_wait_ms)}, + force_verbose=True, ) if not is_visible and not config.ignore_body_visibility: diff --git a/tests/test_body_visibility_warning.py b/tests/test_body_visibility_warning.py index 29e8a2893..7f967665d 100644 --- a/tests/test_body_visibility_warning.py +++ b/tests/test_body_visibility_warning.py @@ -7,6 +7,7 @@ `body_visibility_timeout` undiscoverable. A warning names the option. """ +import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -16,8 +17,9 @@ from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy -def _strategy(is_visible: bool): - """A strategy whose body-visibility wait returns `is_visible`.""" +def _strategy(is_visible: bool, wait_seconds: float = 0.0): + """A strategy whose body-visibility wait returns `is_visible` after + `wait_seconds` of (real) waiting.""" page = MagicMock() page.evaluate = AsyncMock() page.set_content = AsyncMock() @@ -32,34 +34,71 @@ def _strategy(is_visible: bool): get_page=AsyncMock(return_value=(page, MagicMock())) ) strategy.execute_hook = AsyncMock() - strategy.csp_compliant_wait = AsyncMock(return_value=is_visible) + + async def wait(*args, **kwargs): + await asyncio.sleep(wait_seconds) + return is_visible + + strategy.csp_compliant_wait = AsyncMock(side_effect=wait) strategy.check_visibility = AsyncMock(return_value={}) strategy.logger = MagicMock() return strategy -def _warnings(strategy): +def _visibility_warnings(strategy): + """The body-visibility warning calls, selected by message rather than by + position — other warnings may fire on the same crawl.""" return [ - call.kwargs.get("message", "") + call for call in strategy.logger.warning.call_args_list + if "never became visible" in call.kwargs.get("message", "") ] @pytest.mark.asyncio async def test_warns_when_wait_times_out_and_result_is_discarded(): - strategy = _strategy(is_visible=False) + strategy = _strategy(is_visible=False, wait_seconds=0.1) + config = CrawlerRunConfig(session_id="body-vis-warn", body_visibility_timeout=100) + + await strategy._crawl_web("raw:hi", config) + + calls = _visibility_warnings(strategy) + assert len(calls) == 1, strategy.logger.warning.call_args_list + assert "body_visibility_timeout" in calls[0].kwargs["message"] + # The time actually spent is reported, so the number matches the delay the + # user is trying to explain. + assert calls[0].kwargs["params"]["elapsed"] >= 90 + + +@pytest.mark.asyncio +async def test_warning_is_not_suppressed_by_verbose_off(): + """Servers and batch jobs run with verbose off — the case that most needs + this warning. Without force_verbose the logger drops it (#2144).""" + strategy = _strategy(is_visible=False, wait_seconds=0.1) + config = CrawlerRunConfig( + session_id="body-vis-quiet-logger", body_visibility_timeout=100, verbose=False + ) + + await strategy._crawl_web("raw:hi", config) + + calls = _visibility_warnings(strategy) + assert len(calls) == 1 + assert calls[0].kwargs.get("force_verbose") is True + + +@pytest.mark.asyncio +async def test_no_warning_when_wait_fails_early(): + """csp_compliant_wait also returns False when the evaluation errors out + (page closed, context destroyed) — that costs no time, so blaming + body_visibility_timeout for it would send the user after the wrong knob.""" + strategy = _strategy(is_visible=False, wait_seconds=0.0) config = CrawlerRunConfig( - session_id="body-vis-warn", body_visibility_timeout=1234 + session_id="body-vis-early-fail", body_visibility_timeout=30000 ) await strategy._crawl_web("raw:hi", config) - messages = _warnings(strategy) - assert any("never became visible" in m for m in messages), messages - assert any("body_visibility_timeout" in m for m in messages), messages - # The timeout that actually applied is reported, not the hardcoded default. - params = strategy.logger.warning.call_args.kwargs["params"] - assert params["timeout"] == 1234 + assert _visibility_warnings(strategy) == [] @pytest.mark.asyncio @@ -69,7 +108,7 @@ async def test_no_warning_when_body_is_visible(): await strategy._crawl_web("raw:hi", config) - assert not any("never became visible" in m for m in _warnings(strategy)) + assert _visibility_warnings(strategy) == [] @pytest.mark.asyncio @@ -86,4 +125,4 @@ async def test_no_warning_when_hidden_body_is_treated_as_an_error(): with pytest.raises(Error, match="Body element is hidden"): await strategy._crawl_web("raw:hi", config) - assert not any("never became visible" in m for m in _warnings(strategy)) + assert _visibility_warnings(strategy) == [] From 95f29529844130d33c13de587a023471801f240c Mon Sep 17 00:00:00 2001 From: Weike Zhang <1390881871@qq.com> Date: Tue, 18 Aug 2026 15:10:06 +0800 Subject: [PATCH 16/38] fix(docker): cap mcp below 2 to keep the v1 low-level API used by mcp_bridge --- deploy/docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/requirements.txt b/deploy/docker/requirements.txt index 212fcf036..b8df98ef2 100644 --- a/deploy/docker/requirements.txt +++ b/deploy/docker/requirements.txt @@ -11,6 +11,6 @@ pydantic>=2.11 rank-bm25==0.2.2 anyio==4.9.0 PyJWT==2.10.1 -mcp>=1.18.0 +mcp>=1.18.0,<2 websockets>=15.0.1 httpx[http2]>=0.27.2 From 4ccd143026e0507af556a80d7d5ce772bd62adc5 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Tue, 18 Aug 2026 16:39:52 +0530 Subject: [PATCH 17/38] fix(docker): route PDFContentScrapingStrategy requests to PDFCrawlerStrategy (#2127) Headless Chromium can't render PDFs inline, so pair the client's PDF scraping strategy with a per-request PDFCrawlerStrategy crawler instead of a pooled browser, as the library documents. --- deploy/docker/api.py | 40 +++++++- tests/test_docker_pdf_crawler_pairing.py | 123 +++++++++++++++++++++++ 2 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 tests/test_docker_pdf_crawler_pairing.py diff --git a/deploy/docker/api.py b/deploy/docker/api.py index f11494b12..72928ea17 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -85,6 +85,7 @@ def _attach_declarative_hooks(crawler, hooks_config: dict) -> dict: get_llm_base_url, get_redis_task_ttl, validate_url_destination, + datetime_handler, ) from webhook import WebhookDeliveryService @@ -607,6 +608,7 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) import json from utils import datetime_handler from crawler_pool import release_crawler + from crawl4ai.processors.pdf import PDFCrawlerStrategy try: async for result in results_gen: @@ -634,7 +636,10 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) logger.warning("Client disconnected during streaming") finally: if crawler: - await release_crawler(crawler) + if isinstance(crawler.crawler_strategy, PDFCrawlerStrategy): + await crawler.close() # not pooled; release_crawler would be a no-op + else: + await release_crawler(crawler) def _normalize_and_validate_seeds(urls: List[str]) -> List[str]: @@ -659,6 +664,7 @@ async def handle_crawl_request( # Track request start request_id = f"req_{uuid4().hex[:8]}" crawler = None + is_pdf_crawl = False try: from monitor import get_monitor await get_monitor().track_request_start( @@ -689,7 +695,17 @@ async def handle_crawl_request( ) from crawler_pool import get_crawler, release_crawler - crawler = await get_crawler(browser_config) + from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy + is_pdf_crawl = isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy) + if is_pdf_crawl: + if hooks_config: + # PDFCrawlerStrategy has no browser page, so hooks can't attach to it + raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy") + # Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline + crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()) + await crawler.start() + else: + crawler = await get_crawler(browser_config) # Attach declarative hooks if provided hooks_status = {} @@ -771,6 +787,10 @@ async def handle_crawl_request( if result_dict.get('pdf') is not None and isinstance(result_dict.get('pdf'), bytes): result_dict['pdf'] = b64encode(result_dict['pdf']).decode('utf-8') + if is_pdf_crawl: + # PDF metadata contains datetimes that JSONResponse can't serialize + result_dict = json.loads(json.dumps(result_dict, default=datetime_handler)) + processed_results.append(result_dict) except Exception as e: logger.error(f"Error processing result: {e}") @@ -851,7 +871,10 @@ async def handle_crawl_request( ) finally: if crawler: - await release_crawler(crawler) + if is_pdf_crawl: + await crawler.close() # not pooled; release_crawler would be a no-op + else: + await release_crawler(crawler) async def handle_stream_crawl_request( urls: List[str], @@ -895,7 +918,16 @@ async def handle_stream_crawl_request( ) from crawler_pool import get_crawler, release_crawler - crawler = await get_crawler(browser_config) + from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy + if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy): + if hooks_config: + # PDFCrawlerStrategy has no browser page, so hooks can't attach to it + raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy") + # Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline + crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()) + await crawler.start() + else: + crawler = await get_crawler(browser_config) # Attach declarative hooks if provided if hooks_config: diff --git a/tests/test_docker_pdf_crawler_pairing.py b/tests/test_docker_pdf_crawler_pairing.py new file mode 100644 index 000000000..cdce7f533 --- /dev/null +++ b/tests/test_docker_pdf_crawler_pairing.py @@ -0,0 +1,123 @@ +"""Tests for the Docker API's PDF crawler pairing. + +When a client requests PDFContentScrapingStrategy, the crawl handlers must +pair it with PDFCrawlerStrategy (which downloads the PDF itself) instead of a +pooled Playwright crawler — headless Chromium cannot render PDFs inline, so +browser navigation fails with "Page.goto: Download is starting" before the +scraping strategy ever runs. +""" + +import importlib +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from crawl4ai.processors.pdf import PDFCrawlerStrategy + +ROOT = Path(__file__).resolve().parent.parent + +CONFIG = { + "crawler": { + "memory_threshold_percent": 90, + "rate_limiter": {"enabled": False, "base_delay": [0.1, 0.3]}, + "base_config": {}, + } +} + + +def _crawler_config_payload(with_pdf_strategy): + params = {"cache_mode": "bypass"} + if with_pdf_strategy: + params["scraping_strategy"] = { + "type": "PDFContentScrapingStrategy", + "params": {}, + } + return {"type": "CrawlerRunConfig", "params": params} + + +@pytest.fixture +def api(monkeypatch): + monkeypatch.syspath_prepend(str(ROOT / "deploy" / "docker")) + return importlib.import_module("api") + + +@pytest.fixture +def pool_mock(api, monkeypatch): + crawler_pool = importlib.import_module("crawler_pool") + egress_broker = importlib.import_module("egress_broker") + governor = importlib.import_module("governor") + + pooled = MagicMock() + pooled.arun = AsyncMock(return_value=[]) + pooled.active_requests = 1 # release_crawler decrements this int + mock = AsyncMock(return_value=pooled) + monkeypatch.setattr(crawler_pool, "get_crawler", mock) + monkeypatch.setattr(api, "_normalize_and_validate_seeds", lambda urls: urls) + monkeypatch.setattr(egress_broker, "enforce_egress", lambda _: None) + monkeypatch.setattr(governor, "clamp_deep_crawl", lambda _: None) + return mock + + +@pytest.mark.asyncio +async def test_pdf_scraping_strategy_gets_pdf_crawler(api, pool_mock, monkeypatch): + used = {} + real_crawler_cls = api.AsyncWebCrawler + + def spy_crawler(*args, **kwargs): + crawler = real_crawler_cls(*args, **kwargs) + used["crawler"] = crawler + used["crawler_strategy"] = crawler.crawler_strategy + crawler.arun = AsyncMock(return_value=[]) + crawler.close = AsyncMock(wraps=crawler.close) + return crawler + + monkeypatch.setattr(api, "AsyncWebCrawler", spy_crawler) + + response = await api.handle_crawl_request( + urls=["https://example.com/document.pdf"], + browser_config={"type": "BrowserConfig", "params": {}}, + crawler_config=_crawler_config_payload(with_pdf_strategy=True), + config=CONFIG, + ) + + assert response["success"] is True + assert isinstance(used["crawler_strategy"], PDFCrawlerStrategy) + pool_mock.assert_not_awaited() + # The dedicated PDF crawler is not pooled, so the handler must close it. + used["crawler"].close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_pdf_strategy_with_hooks_rejected(api, pool_mock): + from fastapi import HTTPException + + # A VALID hook action: without the guard this reaches set_hook and blows up + # with AttributeError (PDFCrawlerStrategy has no set_hook) -> 500. An invalid + # action would 400 via HookValidationError even without the guard, proving nothing. + hooks = {"hooks": [{"action": "block_resources", "params": {"resource_types": ["image"]}}]} + with pytest.raises(HTTPException) as exc_info: + await api.handle_crawl_request( + urls=["https://example.com/document.pdf"], + browser_config={"type": "BrowserConfig", "params": {}}, + crawler_config=_crawler_config_payload(with_pdf_strategy=True), + config=CONFIG, + hooks_config=hooks, + ) + + assert exc_info.value.status_code == 400 + assert "PDFContentScrapingStrategy" in exc_info.value.detail + pool_mock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_default_strategy_still_uses_pool(api, pool_mock): + response = await api.handle_crawl_request( + urls=["https://example.com/"], + browser_config={"type": "BrowserConfig", "params": {}}, + crawler_config=_crawler_config_payload(with_pdf_strategy=False), + config=CONFIG, + ) + + assert response["success"] is True + pool_mock.assert_awaited_once() From 4138ada0bbb653fbf9a42469a1024c765ca7f801 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Tue, 18 Aug 2026 19:16:43 +0530 Subject: [PATCH 18/38] fix(docker): validate PDF download redirects against SSRF (#2127) Follow redirects manually in PDFContentScrapingStrategy so the Docker server's validate_url_destination vets the download URL and every hop before fetch, blocking redirects to internal addresses. --- crawl4ai/processors/pdf/__init__.py | 26 +++++++- deploy/docker/api.py | 4 ++ tests/test_docker_pdf_crawler_pairing.py | 85 +++++++++++++++++++++++- 3 files changed, 112 insertions(+), 3 deletions(-) diff --git a/crawl4ai/processors/pdf/__init__.py b/crawl4ai/processors/pdf/__init__.py index ba61e946e..7722fe7fd 100644 --- a/crawl4ai/processors/pdf/__init__.py +++ b/crawl4ai/processors/pdf/__init__.py @@ -7,6 +7,8 @@ from crawl4ai.content_scraping_strategy import ContentScrapingStrategy from .processor import NaivePDFProcessorStrategy # Assuming your current PDF code is in pdf_processor.py +MAX_PDF_DOWNLOAD_REDIRECTS = 10 # Max redirect hops to follow when downloading a PDF + class PDFCrawlerStrategy(AsyncCrawlerStrategy): """Crawler strategy for PDF documents. @@ -68,8 +70,10 @@ def __init__(self, extract_images : bool = False, image_save_dir : str = None, batch_size: int = 4, - logger: AsyncLogger = None): + logger: AsyncLogger = None, + url_validator=None): self.logger = logger + self.url_validator = url_validator # vets the download URL before fetch self.pdf_processor = NaivePDFProcessorStrategy( save_images_locally=save_images_locally, extract_images=extract_images, @@ -164,7 +168,25 @@ def _get_pdf_path(self, url: str) -> str: # Download PDF with streaming and timeout # Connection timeout: 10s, Read timeout: 300s (5 minutes for large PDFs) - response = requests.get(url, stream=True, timeout=(20, 60 * 10)) + # Redirects are followed manually so url_validator (when set) can vet every hop BEFORE it is fetched + from urllib.parse import urljoin + current_url = url + for _ in range(MAX_PDF_DOWNLOAD_REDIRECTS): + if self.url_validator: + self.url_validator(current_url) + response = requests.get(current_url, stream=True, timeout=(20, 60 * 10), + allow_redirects=False) + if response.is_redirect: + location = response.headers.get("location") + response.close() # hop response holds its connection open (stream=True) + if not location: + raise RuntimeError(f"Redirect without Location header from {current_url}") + current_url = urljoin(current_url, location) + continue + break + else: + raise RuntimeError( + f"Too many redirects (>{MAX_PDF_DOWNLOAD_REDIRECTS}) downloading PDF from {url}") response.raise_for_status() # Get file size if available diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 72928ea17..404d05f1d 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -701,6 +701,8 @@ async def handle_crawl_request( if hooks_config: # PDFCrawlerStrategy has no browser page, so hooks can't attach to it raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy") + # SSRF protection: vet the PDF download URL and every redirect hop + crawler_config.scraping_strategy.url_validator = validate_url_destination # Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()) await crawler.start() @@ -923,6 +925,8 @@ async def handle_stream_crawl_request( if hooks_config: # PDFCrawlerStrategy has no browser page, so hooks can't attach to it raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy") + # SSRF protection: vet the PDF download URL and every redirect hop + crawler_config.scraping_strategy.url_validator = validate_url_destination # Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()) await crawler.start() diff --git a/tests/test_docker_pdf_crawler_pairing.py b/tests/test_docker_pdf_crawler_pairing.py index cdce7f533..345c6c420 100644 --- a/tests/test_docker_pdf_crawler_pairing.py +++ b/tests/test_docker_pdf_crawler_pairing.py @@ -13,7 +13,7 @@ import pytest -from crawl4ai.processors.pdf import PDFCrawlerStrategy +from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy ROOT = Path(__file__).resolve().parent.parent @@ -86,6 +86,10 @@ def spy_crawler(*args, **kwargs): pool_mock.assert_not_awaited() # The dedicated PDF crawler is not pooled, so the handler must close it. used["crawler"].close.assert_awaited_once() + # SSRF protection: the handler must wire the server's URL validator into + # the scraping strategy so every download/redirect hop is vetted. + effective_config = used["crawler"].arun.await_args.kwargs["config"] + assert effective_config.scraping_strategy.url_validator is api.validate_url_destination @pytest.mark.asyncio @@ -121,3 +125,82 @@ async def test_default_strategy_still_uses_pool(api, pool_mock): assert response["success"] is True pool_mock.assert_awaited_once() + + +# --------------------------------------------------------------------------- +# PDF download redirect handling (url_validator SSRF guard) +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def redirect_server(): + """Real HTTP server: /hop redirects to /doc.pdf, which serves a tiny PDF.""" + import http.server + import socket + import threading + + pdf_bytes = (b"%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n" + b"2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n" + b"xref\n0 3\n0000000000 65535 f \n0000000009 00000 n \n" + b"0000000058 00000 n \ntrailer\n<< /Size 3 /Root 1 0 R >>\n" + b"startxref\n110\n%%EOF\n") + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/hop": + self.send_response(302) + self.send_header("Location", "/doc.pdf") + self.end_headers() + elif self.path == "/loop": + self.send_response(302) + self.send_header("Location", "/loop") + self.end_headers() + else: + self.send_response(200) + self.send_header("Content-Type", "application/pdf") + self.end_headers() + self.wfile.write(pdf_bytes) + + def log_message(self, *args): + pass + + with socket.socket() as s: + s.bind(("localhost", 0)) + port = s.getsockname()[1] + httpd = http.server.ThreadingHTTPServer(("localhost", port), Handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + yield f"http://localhost:{port}" + httpd.shutdown() + + +def test_download_validator_vets_every_redirect_hop(redirect_server): + """The validator must see BOTH the original URL and the redirect target, + and a raising validator must abort the download before the hop is fetched.""" + seen = [] + + def validator(u): + seen.append(u) + if u.endswith("/doc.pdf"): + raise ValueError("blocked hop") + + strategy = PDFContentScrapingStrategy(url_validator=validator) + with pytest.raises(RuntimeError, match="Failed to download"): + strategy._get_pdf_path(f"{redirect_server}/hop") + + assert seen == [f"{redirect_server}/hop", f"{redirect_server}/doc.pdf"] + + +def test_download_redirects_still_followed_without_validator(redirect_server): + """Back-compat: with no validator, redirects are followed as before.""" + strategy = PDFContentScrapingStrategy() + path = strategy._get_pdf_path(f"{redirect_server}/hop") + try: + assert Path(path).read_bytes().startswith(b"%PDF") + finally: + Path(path).unlink(missing_ok=True) + + +def test_download_redirect_loop_aborts(redirect_server): + """An endless redirect chain must abort after the cap, not hang.""" + strategy = PDFContentScrapingStrategy() + with pytest.raises(RuntimeError, match="[Tt]oo many redirects"): + strategy._get_pdf_path(f"{redirect_server}/loop") From 2341507d505dca7449a35a49dbcffd2a636c8e49 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Wed, 19 Aug 2026 17:47:35 +0530 Subject: [PATCH 19/38] fix(docker): close SSRF gap on per-URL configs and dispose PDF crawlers (#2127) Vet PDF strategies sent via crawler_configs, close PDF crawlers on stream error paths, and fix the stream test to spy on AsyncWebCrawler instead of the pool. --- deploy/docker/api.py | 31 ++++++++++++++++++----------- tests/test_issue_2127_docker_pdf.py | 4 ++-- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/deploy/docker/api.py b/deploy/docker/api.py index 404d05f1d..09a783939 100644 --- a/deploy/docker/api.py +++ b/deploy/docker/api.py @@ -603,12 +603,19 @@ def create_task_response(task: dict, task_id: str, base_url: str) -> dict: return response +async def _dispose_crawler(crawler): + """Close a dedicated PDF crawler (not pooled) or release a pooled one.""" + from crawl4ai.processors.pdf import PDFCrawlerStrategy + from crawler_pool import release_crawler + if isinstance(crawler.crawler_strategy, PDFCrawlerStrategy): + await crawler.close() + else: + await release_crawler(crawler) + async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) -> AsyncGenerator[bytes, None]: """Stream results with heartbeats and completion markers.""" import json from utils import datetime_handler - from crawler_pool import release_crawler - from crawl4ai.processors.pdf import PDFCrawlerStrategy try: async for result in results_gen: @@ -636,10 +643,7 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) logger.warning("Client disconnected during streaming") finally: if crawler: - if isinstance(crawler.crawler_strategy, PDFCrawlerStrategy): - await crawler.close() # not pooled; release_crawler would be a no-op - else: - await release_crawler(crawler) + await _dispose_crawler(crawler) def _normalize_and_validate_seeds(urls: List[str]) -> List[str]: @@ -727,6 +731,9 @@ async def handle_crawl_request( current_value = getattr(cfg, key) if current_value is None or current_value == "": setattr(cfg, key, value) + # SSRF: per-URL PDF strategies need the validator wired too + if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy): + cfg.scraping_strategy.url_validator = validate_url_destination effective_config = config_list else: # Single config (original behavior) @@ -919,7 +926,7 @@ async def handle_stream_crawl_request( ), ) - from crawler_pool import get_crawler, release_crawler + from crawler_pool import get_crawler from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy): if hooks_config: @@ -964,21 +971,21 @@ async def handle_stream_crawl_request( except (UntrustedConfigError, HookValidationError) as e: if crawler: - await release_crawler(crawler) + await _dispose_crawler(crawler) raise HTTPException(status_code=400, detail=f"Rejected request: {e}") except HTTPException: # Deliberate status (e.g. 400 SSRF "URL blocked") must pass through # rather than be genericized to 500 by the handler below. if crawler: - await release_crawler(crawler) + await _dispose_crawler(crawler) raise except Exception as e: - # Release crawler on setup error (for successful streams, - # release happens in stream_results finally block) + # Dispose crawler on setup error (for successful streams, + # disposal happens in stream_results finally block) if crawler: - await release_crawler(crawler) + await _dispose_crawler(crawler) logger.error(f"Stream crawl error: {str(e)}", exc_info=True) # Raising HTTPException here will prevent streaming response raise HTTPException( diff --git a/tests/test_issue_2127_docker_pdf.py b/tests/test_issue_2127_docker_pdf.py index 1fe187622..952f1daa2 100644 --- a/tests/test_issue_2127_docker_pdf.py +++ b/tests/test_issue_2127_docker_pdf.py @@ -31,16 +31,16 @@ async def test_stream_handler_preserves_requested_scraping_strategy(monkeypatch) monkeypatch.syspath_prepend(str(docker_dir)) api = importlib.import_module("api") - crawler_pool = importlib.import_module("crawler_pool") egress_broker = importlib.import_module("egress_broker") governor = importlib.import_module("governor") crawler = MagicMock() crawler.arun_many = AsyncMock(return_value=MagicMock()) + crawler.start = AsyncMock() monkeypatch.setattr(api, "_normalize_and_validate_seeds", lambda urls: urls) monkeypatch.setattr(egress_broker, "enforce_egress", lambda _: None) monkeypatch.setattr(governor, "clamp_deep_crawl", lambda _: None) - monkeypatch.setattr(crawler_pool, "get_crawler", AsyncMock(return_value=crawler)) + monkeypatch.setattr(api, "AsyncWebCrawler", MagicMock(return_value=crawler)) crawler_config = { "type": "CrawlerRunConfig", From 01e6e96453ef1bca6f9d650022208f1e6422cc4b Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Wed, 19 Aug 2026 19:11:12 +0530 Subject: [PATCH 20/38] fix(docker): fall through junk proxy env values, refuse non-http proxy schemes Keep the first proxy env candidate that parses, warn on unparseable/unsupported values, and match NO_PROXY host:port. --- deploy/docker/egress_proxy.py | 43 +++++++++++-------- .../tests/test_security_egress_proxy.py | 12 +++++- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/deploy/docker/egress_proxy.py b/deploy/docker/egress_proxy.py index 1aa1bcc2f..807bb80a1 100644 --- a/deploy/docker/egress_proxy.py +++ b/deploy/docker/egress_proxy.py @@ -39,35 +39,39 @@ _MAX_HEADER_BYTES = 64 * 1024 -def _env(*names: str) -> str: - return next((os.environ[n] for n in names if os.environ.get(n)), "") - - def upstream_proxy(scheme: str = "https"): """(host, port, auth_header_bytes|None) of the upstream proxy, or None. Read per-call (not at import) so operators and tests see env changes. - The target scheme picks HTTP(S)_PROXY per convention, falling back to - the other pair when only one is set. + The target scheme picks HTTP(S)_PROXY per convention; the first + candidate that parses wins, so junk values fall through to a fallback. """ order = ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") if scheme == "http" \ else ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy") - raw = _env("CRAWL4AI_UPSTREAM_PROXY", *order).strip() - if not raw: - return None - sp = urlsplit(raw if "://" in raw else "http://" + raw) - if not sp.hostname: - return None - auth = None - if sp.username: - cred = f"{unquote(sp.username)}:{unquote(sp.password or '')}".encode("utf-8") - auth = b"Proxy-Authorization: Basic " + base64.b64encode(cred) + b"\r\n" - return sp.hostname, sp.port or 80, auth + for name in ("CRAWL4AI_UPSTREAM_PROXY", *order): + raw = (os.environ.get(name) or "").strip() + if not raw: + continue + sp = urlsplit(raw if "://" in raw else "http://" + raw) + if not sp.hostname: + logger.warning("ignoring %s: unparseable proxy URL", name) + continue + if sp.scheme != "http": + # We only speak plaintext HTTP to the upstream (no TLS/socks dial). + logger.warning("ignoring %s: unsupported proxy scheme %r", name, sp.scheme) + continue + auth = None + if sp.username: + cred = f"{unquote(sp.username)}:{unquote(sp.password or '')}".encode("utf-8") + auth = b"Proxy-Authorization: Basic " + base64.b64encode(cred) + b"\r\n" + return sp.hostname, sp.port or 80, auth + return None def _no_proxy_match(host: str, ip: str) -> bool: """True if NO_PROXY says this target must bypass the upstream proxy.""" - entries = [e.strip() for e in _env("NO_PROXY", "no_proxy").split(",") if e.strip()] + raw = os.environ.get("NO_PROXY") or os.environ.get("no_proxy") or "" + entries = [e.strip() for e in raw.split(",") if e.strip()] for entry in entries: if entry == "*": return True @@ -78,6 +82,9 @@ def _no_proxy_match(host: str, ip: str) -> bool: except ValueError: pass suffix = entry.lower().lstrip(".") + head, sep, port_part = suffix.rpartition(":") + if sep and port_part.isdigit(): + suffix = head low = host.lower() if low == suffix or low.endswith("." + suffix): return True diff --git a/deploy/docker/tests/test_security_egress_proxy.py b/deploy/docker/tests/test_security_egress_proxy.py index f53767f35..9dc377e25 100644 --- a/deploy/docker/tests/test_security_egress_proxy.py +++ b/deploy/docker/tests/test_security_egress_proxy.py @@ -264,8 +264,16 @@ def test_upstream_proxy_env_parsing(monkeypatch): monkeypatch.delenv("HTTP_PROXY") monkeypatch.delenv("HTTPS_PROXY") assert egress_proxy.upstream_proxy() is None - # non-latin-1 credentials must not raise (encoded as UTF-8) + # a junk/unsupported candidate falls through to a valid fallback + monkeypatch.setenv("HTTP_PROXY", "http://good:3128") + monkeypatch.setenv("HTTPS_PROXY", "http://") + assert egress_proxy.upstream_proxy() == ("good", 3128, None) + monkeypatch.setenv("HTTPS_PROXY", "https://tls-proxy.corp") # unsupported scheme + assert egress_proxy.upstream_proxy() == ("good", 3128, None) monkeypatch.delenv("CRAWL4AI_UPSTREAM_PROXY") + monkeypatch.delenv("HTTP_PROXY") + assert egress_proxy.upstream_proxy() is None # https:// alone -> refused, not mis-dialed + # non-latin-1 credentials must not raise (encoded as UTF-8) monkeypatch.setenv("HTTPS_PROXY", "http://u:%E5%AF%86%E7%A0%81@10.0.0.1:8080") assert egress_proxy.upstream_proxy()[2] is not None # NO_PROXY routing: suffix and CIDR entries force a direct dial @@ -275,6 +283,8 @@ def test_upstream_proxy_env_parsing(monkeypatch): assert egress_proxy._use_upstream(pin) is None monkeypatch.setenv("NO_PROXY", "203.0.113.0/24") assert egress_proxy._use_upstream(pin) is None + monkeypatch.setenv("NO_PROXY", "site.corp.example:443") # host:port form + assert egress_proxy._use_upstream(pin) is None class TestEnforceEgressWiring: From 8748b81bb64edfb77cebbca36c742565d95dd74e Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Wed, 22 Jul 2026 16:41:50 +0530 Subject: [PATCH 21/38] fix(docker): compose v5 compatibility, legacy-field warnings, playground error handling - docker-compose.yml: move the PID cap to deploy.resources.limits.pids (Compose v5 rejects pids_limit alongside a limits block; same behavior on v2.x). - entrypoint.sh: explain the loopback-only bind when no CRAWL4AI_API_TOKEN is set and how to fix it. - server.py: warn when the removed output_path (screenshot/pdf) or legacy hooks.code is sent - fields are ignored, never executed; hooks status reads "ignored" and /crawl/stream sends X-Hooks-Warning. Make "/" public so the /playground redirect works; data routes stay gated. - schemas.py: capture legacy hooks.code so it can be reported (never run). - playground: check response.ok on the streaming branch, surface server error details, hint at the token bar on 401. - tests: add deploy/docker/tests/test_legacy_compat.py (13 tests). --- deploy/docker/entrypoint.sh | 2 + deploy/docker/schemas.py | 8 + deploy/docker/server.py | 40 +++- deploy/docker/static/playground/index.html | 21 ++- deploy/docker/tests/test_legacy_compat.py | 206 +++++++++++++++++++++ docker-compose.yml | 4 +- 6 files changed, 272 insertions(+), 9 deletions(-) create mode 100644 deploy/docker/tests/test_legacy_compat.py diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh index b624a3115..19d6766f6 100644 --- a/deploy/docker/entrypoint.sh +++ b/deploy/docker/entrypoint.sh @@ -31,6 +31,8 @@ else # No credential -> refuse to expose; serve loopback only. GUNICORN_BIND="127.0.0.1:${PORT}" echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2 + echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2 + echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose) and restart." >&2 fi export GUNICORN_BIND diff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.py index f066e5c88..b1065b5a8 100644 --- a/deploy/docker/schemas.py +++ b/deploy/docker/schemas.py @@ -42,6 +42,14 @@ class HookConfig(BaseModel): le=120, description="Timeout in seconds for each hook execution", ) + # Legacy 0.8.x field: inline-Python hook code, removed in 0.9.0 (it was an + # exec()-based RCE surface). Captured here (instead of being dropped by + # pydantic) solely so the server can tell the caller it was NOT executed; + # it is never run. + code: Optional[Dict[str, str]] = Field( + default=None, + description="REMOVED in 0.9.0: inline hook code is accepted for compatibility but never executed", + ) class Config: json_schema_extra = { diff --git a/deploy/docker/server.py b/deploy/docker/server.py index ce1403b87..86919ef0c 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -392,7 +392,7 @@ def _current_api_token() -> str: app.add_middleware( AuthGateMiddleware, token_provider=_current_api_token, - public_paths={HEALTH_PATH, "/token"}, + public_paths={HEALTH_PATH, "/token", "/"}, public_prefixes=_UI_PREFIXES, ) @@ -660,6 +660,17 @@ async def get_artifact(artifact_id: str, _td: Dict = Depends(token_dep)): # Screenshot endpoint +_OUTPUT_PATH_WARNING = ( + "output_path was removed in 0.9.0 and is ignored - no file was written. " + "The result is stored server-side; fetch it with an authenticated " + "GET /artifacts/{artifact_id}." +) + +_HOOKS_CODE_WARNING = ( + "Inline hook code (hooks.code) was removed in 0.9.0 and was NOT executed. " + "Use declarative hook actions instead (GET /hooks/info for the schema)." +) + @app.post("/screenshot") @limiter.limit(config["rate_limiting"]["default_limit"]) @@ -684,7 +695,12 @@ async def generate_screenshot( raise HTTPException(500, detail=results[0].error_message or "Crawl failed") screenshot_data = results[0].screenshot art = _store_artifact("png", base64.b64decode(screenshot_data)) - return {"success": True, "screenshot": screenshot_data, **art} + response = {"success": True, "screenshot": screenshot_data, **art} + # Legacy 0.8.x key, no longer a schema field: peek at the raw body + # (already parsed and cached by FastAPI) to warn that it was ignored. + if (await request.json()).get("output_path"): + response["warning"] = _OUTPUT_PATH_WARNING + return response except HTTPException: raise except Exception as e: @@ -719,7 +735,12 @@ async def generate_pdf( raise HTTPException(500, detail=results[0].error_message or "Crawl failed") pdf_data = results[0].pdf art = _store_artifact("pdf", pdf_data) - return {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art} + response = {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art} + # Legacy 0.8.x key, no longer a schema field: peek at the raw body + # (already parsed and cached by FastAPI) to warn that it was ignored. + if (await request.json()).get("output_path"): + response["warning"] = _OUTPUT_PATH_WARNING + return response except HTTPException: raise except Exception as e: @@ -889,7 +910,7 @@ async def crawl( raise HTTPException(400, f"Rejected config: {e}") if crawler_config.stream: return await stream_process(crawl_request=crawl_request) - + # Prepare hooks config if provided hooks_config = None if crawl_request.hooks: @@ -897,7 +918,7 @@ async def crawl( 'hooks': crawl_request.hooks.hooks, 'timeout': crawl_request.hooks.timeout } - + results = await handle_crawl_request( urls=crawl_request.urls, browser_config=crawl_request.browser_config, @@ -906,6 +927,11 @@ async def crawl( hooks_config=hooks_config, crawler_configs=crawl_request.crawler_configs, ) + if crawl_request.hooks and crawl_request.hooks.code: + hooks_resp = results.setdefault("hooks", {"attached": []}) + if not crawl_request.hooks.hooks: + hooks_resp["status"] = "ignored" + hooks_resp["warning"] = _HOOKS_CODE_WARNING return JSONResponse(results) @@ -924,7 +950,7 @@ async def crawl_stream( return await stream_process(crawl_request=crawl_request) async def stream_process(crawl_request: CrawlRequestWithHooks): - + # Prepare hooks config if provided# Prepare hooks config if provided hooks_config = None if crawl_request.hooks: @@ -950,6 +976,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks): if hooks_info: import json headers["X-Hooks-Status"] = json.dumps(hooks_info['status']['status']) + if crawl_request.hooks and crawl_request.hooks.code: + headers["X-Hooks-Warning"] = _HOOKS_CODE_WARNING return StreamingResponse( stream_results(crawler, gen), diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html index e88c281a0..cf72a303b 100644 --- a/deploy/docker/static/playground/index.html +++ b/deploy/docker/static/playground/index.html @@ -617,6 +617,17 @@

🔥 Stress Test

} } + // Build a useful error message from a failed HTTP response + function httpErrorMessage(response, data) { + let msg = (data && (data.detail || data.error)) || `HTTP ${response.status}`; + if (response.status === 401) { + msg += getToken() + ? ' — token rejected; check the API token in the token bar (top right)' + : ' — set your API token in the token bar (top right)'; + } + return msg; + } + // Generate code snippets function generateSnippets(api, payload, method = 'POST') { // Python snippet @@ -751,7 +762,7 @@

🔥 Stress Test

const time = Math.round(performance.now() - startTime); if (!response.ok) { updateStatus('error', time); - throw new Error(responseData.error || 'Request failed'); + throw new Error(httpErrorMessage(response, responseData)); } updateStatus('success', time); document.querySelector('#response-content code').textContent = JSON.stringify(responseData, null, 2); @@ -765,6 +776,12 @@

🔥 Stress Test

body: JSON.stringify(payload) }); + if (!response.ok) { + const errData = await response.json().catch(() => ({})); + updateStatus('error', Math.round(performance.now() - startTime)); + throw new Error(httpErrorMessage(response, errData)); + } + const reader = response.body.getReader(); let text = ''; let maxMemory = 0; @@ -809,7 +826,7 @@

🔥 Stress Test

if (!response.ok) { updateStatus('error', time); - throw new Error(responseData.error || 'Request failed'); + throw new Error(httpErrorMessage(response, responseData)); } updateStatus( diff --git a/deploy/docker/tests/test_legacy_compat.py b/deploy/docker/tests/test_legacy_compat.py new file mode 100644 index 000000000..b8b8402c5 --- /dev/null +++ b/deploy/docker/tests/test_legacy_compat.py @@ -0,0 +1,206 @@ +""" +Behavioral tests for 0.9.x legacy-compatibility handling: + + * root redirect - "/" is public and redirects to /playground instead of + dying in the auth gate with a bare 401; /monitor and the + data routes stay gated. + * output_path - /screenshot and /pdf still accept the 0.8.x output_path + field but return a warning saying no file was written, + instead of silently dropping it. + * legacy hooks - hooks.code (removed 0.8.x inline Python) is captured, + never executed, and reported as status "ignored" with a + warning when hooks are enabled; any hooks payload is + still refused (403) while hooks are disabled. + * compose file - the PID cap lives under deploy.resources.limits (not + pids_limit), which Compose v5 rejects alongside a + limits block. + +These exercise the running app via TestClient (no browser / Redis needed); +crawl internals are stubbed where a handler would otherwise need a browser. +""" + +import base64 +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from auth import create_access_token # noqa: E402 + + +def _bearer() -> dict: + return {"Authorization": f"Bearer {create_access_token({'sub': 'user@x.com'}, scope='data')}"} + + +# ───────────────────────── root redirect ───────────────────────── + + +class TestRootRedirect: + def test_root_is_public_and_redirects_to_playground(self, stock_client): + r = stock_client.get("/", follow_redirects=False) + assert r.status_code in (302, 307), ( + f"GET / returned {r.status_code}; expected a redirect. The auth " + f"gate must allow the exact path '/' so the redirect route runs." + ) + assert r.headers["location"] == "/playground" + + def test_monitor_and_data_routes_stay_gated(self, stock_client): + assert stock_client.get("/monitor").status_code == 401 + assert stock_client.get("/monitor/health").status_code == 401 + assert stock_client.post("/crawl", json={"urls": ["https://x"]}).status_code == 401 + + +# ───────────────────────── output_path warning ───────────────────────── + + +@pytest.fixture +def stub_crawler(server_module, monkeypatch): + """Stub the crawler pool + artifact store so /screenshot and /pdf run + without a browser. Returns the fake artifact dict for assertions.""" + art = {"artifact_id": "a1", "url": "/artifacts/a1", "mime": "x", "size": 1} + png_b64 = base64.b64encode(b"fake-png").decode() + + fake_result = SimpleNamespace(success=True, screenshot=png_b64, pdf=b"fake-pdf") + + class _FakeCrawler: + async def arun(self, url, config): + return [fake_result] + + async def fake_get_crawler(cfg): + return _FakeCrawler() + + async def fake_release_crawler(crawler): + pass + + monkeypatch.setattr(server_module, "get_crawler", fake_get_crawler) + monkeypatch.setattr(server_module, "release_crawler", fake_release_crawler) + monkeypatch.setattr(server_module, "_store_artifact", lambda kind, data: dict(art)) + return art + + +class TestOutputPathWarning: + @pytest.mark.parametrize("endpoint,payload_key", [("/screenshot", "screenshot"), ("/pdf", "pdf")]) + def test_output_path_accepted_with_warning(self, stock_client, stub_crawler, endpoint, payload_key): + r = stock_client.post( + endpoint, + json={"url": "https://example.com", "output_path": "/tmp/x.bin"}, + headers=_bearer(), + ) + assert r.status_code == 200 + body = r.json() + assert body["success"] is True + assert "warning" in body, "output_path must not be silently dropped" + assert "no file was written" in body["warning"] + assert body["artifact_id"] == stub_crawler["artifact_id"] + + @pytest.mark.parametrize("endpoint", ["/screenshot", "/pdf"]) + def test_no_warning_without_output_path(self, stock_client, stub_crawler, endpoint): + r = stock_client.post(endpoint, json={"url": "https://example.com"}, headers=_bearer()) + assert r.status_code == 200 + assert "warning" not in r.json() + + +# ───────────────────────── legacy hooks.code ───────────────────────── + +LEGACY_HOOKS = {"code": {"before_goto": "async def hook(p, c, u, **kw): return p"}} +DECLARATIVE_HOOKS = {"hooks": [{"action": "scroll_to_bottom", "params": {"max_steps": 2}}]} + + +class TestLegacyHookCode: + def test_hook_config_captures_code_field(self): + """The legacy field must be parsed (not dropped) so it can be reported.""" + from schemas import CrawlRequestWithHooks + + req = CrawlRequestWithHooks(urls=["https://x"], hooks=LEGACY_HOOKS) + assert req.hooks.code == LEGACY_HOOKS["code"] + assert req.hooks.hooks == [] + + @pytest.mark.parametrize("hooks_payload", [LEGACY_HOOKS, DECLARATIVE_HOOKS]) + def test_any_hooks_payload_403_when_disabled(self, server_module, monkeypatch, stock_client, hooks_payload): + monkeypatch.setattr(server_module, "HOOKS_ENABLED", False) + r = stock_client.post( + "/crawl", + json={"urls": ["https://example.com"], "hooks": hooks_payload}, + headers=_bearer(), + ) + assert r.status_code == 403 + + def _stub_crawl(self, server_module, monkeypatch, results): + async def fake_handle_crawl_request(**kwargs): + return dict(results) + + monkeypatch.setattr(server_module, "handle_crawl_request", fake_handle_crawl_request) + + def test_legacy_code_warned_and_ignored_when_enabled(self, server_module, monkeypatch, stock_client): + monkeypatch.setattr(server_module, "HOOKS_ENABLED", True) + # Empty declarative specs -> api.py reports a vacuous success + self._stub_crawl( + server_module, monkeypatch, + {"success": True, "results": [{"success": True}], + "hooks": {"status": "success", "attached": []}}, + ) + r = stock_client.post( + "/crawl", + json={"urls": ["https://example.com"], "hooks": LEGACY_HOOKS}, + headers=_bearer(), + ) + assert r.status_code == 200 + hooks = r.json()["hooks"] + assert hooks["status"] == "ignored", "vacuous 'success' must be rewritten" + assert hooks["attached"] == [] + assert "NOT executed" in hooks["warning"] + + def test_mixed_request_keeps_declarative_status_and_warns(self, server_module, monkeypatch, stock_client): + monkeypatch.setattr(server_module, "HOOKS_ENABLED", True) + self._stub_crawl( + server_module, monkeypatch, + {"success": True, "results": [{"success": True}], + "hooks": {"status": "success", "attached": ["before_retrieve_html"]}}, + ) + r = stock_client.post( + "/crawl", + json={"urls": ["https://example.com"], + "hooks": {**DECLARATIVE_HOOKS, **LEGACY_HOOKS}}, + headers=_bearer(), + ) + assert r.status_code == 200 + hooks = r.json()["hooks"] + assert hooks["status"] == "success", "real declarative execution must not be relabeled" + assert hooks["attached"] == ["before_retrieve_html"] + assert "NOT executed" in hooks["warning"] + + def test_no_hooks_response_untouched(self, server_module, monkeypatch, stock_client): + self._stub_crawl( + server_module, monkeypatch, + {"success": True, "results": [{"success": True}]}, + ) + r = stock_client.post("/crawl", json={"urls": ["https://example.com"]}, headers=_bearer()) + assert r.status_code == 200 + assert "hooks" not in r.json() + + +# ───────────────────────── compose file ───────────────────────── + + +class TestComposeFile: + def test_pid_cap_lives_under_deploy_limits(self): + """Compose v5 rejects pids_limit next to deploy.resources.limits + ("can't set distinct values"); the cap must be expressed once, under + deploy.resources.limits.pids.""" + import os + + override = os.environ.get("CRAWL4AI_COMPOSE_FILE") + if override: + compose_path = Path(override) + else: + here = Path(__file__).resolve() + if len(here.parents) < 4: + pytest.skip("not running from a repo checkout") + compose_path = here.parents[3] / "docker-compose.yml" + if not compose_path.exists(): + pytest.skip(f"docker-compose.yml not found at {compose_path}") + doc = yaml.safe_load(compose_path.read_text()) + base = doc["x-base-config"] + assert "pids_limit" not in base + assert base["deploy"]["resources"]["limits"]["pids"] == 512 diff --git a/docker-compose.yml b/docker-compose.yml index 1cd87ad93..5939b3321 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,6 @@ x-base-config: &base-config - ALL security_opt: - no-new-privileges:true - pids_limit: 512 # Read-only root filesystem; only these paths are writable (tmpfs). read_only: true tmpfs: @@ -39,6 +38,9 @@ x-base-config: &base-config resources: limits: memory: 4G + # PID cap; expressed here (not as pids_limit) so the file stays valid + # on Compose v5+, which rejects pids_limit alongside a limits block. + pids: 512 reservations: memory: 1G restart: unless-stopped From cd78a4406fd6ce29ac9b6c616bbb3bb1a957065c Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Sat, 25 Jul 2026 12:10:14 +0530 Subject: [PATCH 22/38] fix(docker): distinguish removed hooks.code in the disabled-hooks 403 Per review: enabling CRAWL4AI_HOOKS_ENABLED cannot run inline hook code (removed in 0.9.0), so code-carrying payloads now get a removal message instead of the misleading generic hint. --- deploy/docker/server.py | 19 +++++++++++++++++-- deploy/docker/tests/test_legacy_compat.py | 15 +++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 86919ef0c..e6f63db23 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -672,6 +672,21 @@ async def get_artifact(artifact_id: str, _td: Dict = Depends(token_dep)): ) +def _reject_disabled_hooks(hooks) -> None: + """403 for any hooks payload while hooks are disabled. Legacy inline code + gets its own detail: enabling CRAWL4AI_HOOKS_ENABLED would not run it (the + feature was removed in 0.9.0), so the generic remedy would mislead.""" + if hooks.code: + raise HTTPException( + 403, + "Inline hook code (hooks.code) was removed in 0.9.0 and cannot be " + "enabled; it was not executed. Use declarative hook actions instead " + "(GET /hooks/info), which are additionally disabled on this server " + "(CRAWL4AI_HOOKS_ENABLED).", + ) + raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.") + + @app.post("/screenshot") @limiter.limit(config["rate_limiting"]["default_limit"]) @mcp_tool("screenshot") @@ -900,7 +915,7 @@ async def crawl( if not crawl_request.urls: raise HTTPException(400, "At least one URL required") if crawl_request.hooks and not HOOKS_ENABLED: - raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.") + _reject_disabled_hooks(crawl_request.hooks) # Check whether it is a redirection for a streaming request try: crawler_config = CrawlerRunConfig.load( @@ -945,7 +960,7 @@ async def crawl_stream( if not crawl_request.urls: raise HTTPException(400, "At least one URL required") if crawl_request.hooks and not HOOKS_ENABLED: - raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.") + _reject_disabled_hooks(crawl_request.hooks) return await stream_process(crawl_request=crawl_request) diff --git a/deploy/docker/tests/test_legacy_compat.py b/deploy/docker/tests/test_legacy_compat.py index b8b8402c5..63a4e22fc 100644 --- a/deploy/docker/tests/test_legacy_compat.py +++ b/deploy/docker/tests/test_legacy_compat.py @@ -116,8 +116,18 @@ def test_hook_config_captures_code_field(self): assert req.hooks.code == LEGACY_HOOKS["code"] assert req.hooks.hooks == [] - @pytest.mark.parametrize("hooks_payload", [LEGACY_HOOKS, DECLARATIVE_HOOKS]) - def test_any_hooks_payload_403_when_disabled(self, server_module, monkeypatch, stock_client, hooks_payload): + # Any hooks payload is refused while hooks are disabled, but the detail + # must not mislead legacy callers: enabling the flag would not run + # hooks.code (removed in 0.9.0), so payloads carrying it get a removal + # message instead of the generic 'set CRAWL4AI_HOOKS_ENABLED' hint. + @pytest.mark.parametrize("hooks_payload,expected_detail", [ + (LEGACY_HOOKS, "removed in 0.9.0"), # code-only + ({**DECLARATIVE_HOOKS, **LEGACY_HOOKS}, "removed in 0.9.0"), # mixed + (DECLARATIVE_HOOKS, "Set CRAWL4AI_HOOKS_ENABLED=true"), # declarative-only + ]) + def test_any_hooks_payload_403_when_disabled( + self, server_module, monkeypatch, stock_client, hooks_payload, expected_detail + ): monkeypatch.setattr(server_module, "HOOKS_ENABLED", False) r = stock_client.post( "/crawl", @@ -125,6 +135,7 @@ def test_any_hooks_payload_403_when_disabled(self, server_module, monkeypatch, s headers=_bearer(), ) assert r.status_code == 403 + assert expected_detail in r.json()["detail"] def _stub_crawl(self, server_module, monkeypatch, results): async def fake_handle_crawl_request(**kwargs): From a01442d3a5a663a85b476b67e22eb9c2d80d69fd Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Thu, 30 Jul 2026 19:21:01 +0530 Subject: [PATCH 23/38] fix(docker): declare output_path as deprecated no-op, harden legacy-compat tests - output_path back as a deprecated schema field (visible in OpenAPI); hooks.code widened to Optional[Any] - pids test parses compose YAML instead of grepping raw text - security pins replaced with a behavioral no-file-written test - playground: readable 422 errors, non-JSON error bodies handled - entrypoint: JWT flag tied to security.jwt_enabled in config.yml --- deploy/docker/entrypoint.sh | 2 +- deploy/docker/schemas.py | 25 +++++-- deploy/docker/server.py | 16 +++-- deploy/docker/static/playground/index.html | 14 +++- deploy/docker/tests/requirements.txt | 1 + deploy/docker/tests/test_legacy_compat.py | 65 ++++++++++--------- deploy/docker/tests/test_security_2026_04.py | 35 ++++++---- .../tests/test_security_container_posture.py | 9 ++- 8 files changed, 107 insertions(+), 60 deletions(-) diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh index 19d6766f6..3d0037916 100644 --- a/deploy/docker/entrypoint.sh +++ b/deploy/docker/entrypoint.sh @@ -32,7 +32,7 @@ else GUNICORN_BIND="127.0.0.1:${PORT}" echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2 echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2 - echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose) and restart." >&2 + echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose), then restart. (If you enabled security.jwt_enabled in a custom config.yml, set CRAWL4AI_JWT_ENABLED=true instead.)" >&2 fi export GUNICORN_BIND diff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.py index b1065b5a8..978f21999 100644 --- a/deploy/docker/schemas.py +++ b/deploy/docker/schemas.py @@ -46,7 +46,7 @@ class HookConfig(BaseModel): # exec()-based RCE surface). Captured here (instead of being dropped by # pydantic) solely so the server can tell the caller it was NOT executed; # it is never run. - code: Optional[Dict[str, str]] = Field( + code: Optional[Any] = Field( default=None, description="REMOVED in 0.9.0: inline hook code is accepted for compatibility but never executed", ) @@ -92,14 +92,29 @@ class ScreenshotRequest(BaseModel): url: str screenshot_wait_for: Optional[float] = 2 wait_for_images: Optional[bool] = False - # output_path removed: callers never name a filesystem path (it was an - # arbitrary-write -> RCE vector). The server writes to the sandboxed - # artifact store and returns an opaque artifact_id. + # Deprecated no-op: caller paths were an arbitrary-write -> RCE vector. + # Never written; the server stores results in the artifact store instead. + output_path: Optional[str] = Field( + default=None, + deprecated=True, + description=( + "REMOVED in 0.9.0 and ignored - no file is written. Results are " + "stored server-side; fetch via GET /artifacts/{artifact_id}." + ), + ) class PDFRequest(BaseModel): url: str - # output_path removed (see ScreenshotRequest). + # output_path deprecated no-op (see ScreenshotRequest). + output_path: Optional[str] = Field( + default=None, + deprecated=True, + description=( + "REMOVED in 0.9.0 and ignored - no file is written. Results are " + "stored server-side; fetch via GET /artifacts/{artifact_id}." + ), + ) class JSEndpointRequest(BaseModel): diff --git a/deploy/docker/server.py b/deploy/docker/server.py index e6f63db23..eb0fddadf 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -701,6 +701,9 @@ async def generate_screenshot( sandboxed artifact store; the response includes an `artifact_id` and a `url` to fetch it. """ validate_url_scheme(body.url) + # model_dump, not attribute access: the field is marked deprecated and + # reading the attribute emits DeprecationWarning on every request. + legacy_output_path = body.model_dump(include={"output_path"}).get("output_path") crawler = None try: cfg = CrawlerRunConfig(screenshot=True, screenshot_wait_for=body.screenshot_wait_for, wait_for_images=body.wait_for_images) @@ -711,9 +714,7 @@ async def generate_screenshot( screenshot_data = results[0].screenshot art = _store_artifact("png", base64.b64decode(screenshot_data)) response = {"success": True, "screenshot": screenshot_data, **art} - # Legacy 0.8.x key, no longer a schema field: peek at the raw body - # (already parsed and cached by FastAPI) to warn that it was ignored. - if (await request.json()).get("output_path"): + if legacy_output_path: response["warning"] = _OUTPUT_PATH_WARNING return response except HTTPException: @@ -741,6 +742,9 @@ async def generate_pdf( sandboxed artifact store; the response includes an `artifact_id` and a `url` to fetch it. """ validate_url_scheme(body.url) + # model_dump, not attribute access: the field is marked deprecated and + # reading the attribute emits DeprecationWarning on every request. + legacy_output_path = body.model_dump(include={"output_path"}).get("output_path") crawler = None try: cfg = CrawlerRunConfig(pdf=True) @@ -751,9 +755,7 @@ async def generate_pdf( pdf_data = results[0].pdf art = _store_artifact("pdf", pdf_data) response = {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art} - # Legacy 0.8.x key, no longer a schema field: peek at the raw body - # (already parsed and cached by FastAPI) to warn that it was ignored. - if (await request.json()).get("output_path"): + if legacy_output_path: response["warning"] = _OUTPUT_PATH_WARNING return response except HTTPException: @@ -943,7 +945,7 @@ async def crawl( crawler_configs=crawl_request.crawler_configs, ) if crawl_request.hooks and crawl_request.hooks.code: - hooks_resp = results.setdefault("hooks", {"attached": []}) + hooks_resp = results.setdefault("hooks", {"status": "ignored", "attached": []}) if not crawl_request.hooks.hooks: hooks_resp["status"] = "ignored" hooks_resp["warning"] = _HOOKS_CODE_WARNING diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html index cf72a303b..ba368b768 100644 --- a/deploy/docker/static/playground/index.html +++ b/deploy/docker/static/playground/index.html @@ -619,7 +619,15 @@

🔥 Stress Test

// Build a useful error message from a failed HTTP response function httpErrorMessage(response, data) { - let msg = (data && (data.detail || data.error)) || `HTTP ${response.status}`; + let detail = data && (data.detail || data.error); + // FastAPI 422s send detail as an array of error objects; other + // non-strings would render as [object Object]. + if (Array.isArray(detail)) { + detail = detail.map(e => e && e.msg ? `${(e.loc || []).join('.')}: ${e.msg}` : JSON.stringify(e)).join('; '); + } else if (detail && typeof detail !== 'string') { + detail = JSON.stringify(detail); + } + let msg = detail || `HTTP ${response.status}`; if (response.status === 401) { msg += getToken() ? ' — token rejected; check the API token in the token bar (top right)' @@ -758,7 +766,7 @@

🔥 Stress Test

method: 'GET', headers: { 'Accept': 'application/json' } }); - responseData = await response.json(); + responseData = await response.json().catch(() => ({})); const time = Math.round(performance.now() - startTime); if (!response.ok) { updateStatus('error', time); @@ -821,7 +829,7 @@

🔥 Stress Test

body: JSON.stringify(payload) }); - responseData = await response.json(); + responseData = await response.json().catch(() => ({})); const time = Math.round(performance.now() - startTime); if (!response.ok) { diff --git a/deploy/docker/tests/requirements.txt b/deploy/docker/tests/requirements.txt index 5f7a842fe..b0206020f 100644 --- a/deploy/docker/tests/requirements.txt +++ b/deploy/docker/tests/requirements.txt @@ -1,2 +1,3 @@ httpx>=0.25.0 docker>=7.0.0 +pyyaml>=6.0 diff --git a/deploy/docker/tests/test_legacy_compat.py b/deploy/docker/tests/test_legacy_compat.py index 63a4e22fc..2a76d6e27 100644 --- a/deploy/docker/tests/test_legacy_compat.py +++ b/deploy/docker/tests/test_legacy_compat.py @@ -11,20 +11,17 @@ never executed, and reported as status "ignored" with a warning when hooks are enabled; any hooks payload is still refused (403) while hooks are disabled. - * compose file - the PID cap lives under deploy.resources.limits (not - pids_limit), which Compose v5 rejects alongside a - limits block. + +(The compose PID-cap check lives in test_security_container_posture.py.) These exercise the running app via TestClient (no browser / Redis needed); crawl internals are stubbed where a handler would otherwise need a browser. """ import base64 -from pathlib import Path from types import SimpleNamespace import pytest -import yaml from auth import create_access_token # noqa: E402 @@ -46,7 +43,11 @@ def test_root_is_public_and_redirects_to_playground(self, stock_client): assert r.headers["location"] == "/playground" def test_monitor_and_data_routes_stay_gated(self, stock_client): - assert stock_client.get("/monitor").status_code == 401 + # /monitor must not serve content without a token; a future + # /monitor -> /dashboard redirect is fine (the target is UI-public), + # so accept 401 or a redirect, never 200. + r = stock_client.get("/monitor", follow_redirects=False) + assert r.status_code in (401, 302, 307, 308) assert stock_client.get("/monitor/health").status_code == 401 assert stock_client.post("/crawl", json={"urls": ["https://x"]}).status_code == 401 @@ -100,6 +101,19 @@ def test_no_warning_without_output_path(self, stock_client, stub_crawler, endpoi assert r.status_code == 200 assert "warning" not in r.json() + @pytest.mark.parametrize("endpoint", ["/screenshot", "/pdf"]) + def test_output_path_is_never_written(self, stock_client, stub_crawler, endpoint, tmp_path): + """Security tripwire for the 0.8.x arbitrary-write vuln: the handler + runs for real here (only crawler/artifact store are stubbed), so any + reintroduced write of body.output_path creates this file and fails.""" + target = tmp_path / "out.bin" + r = stock_client.post( + endpoint, json={"url": "https://example.com", "output_path": str(target)}, + headers=_bearer(), + ) + assert r.status_code == 200 + assert not target.exists(), "output_path must never be written to disk" + # ───────────────────────── legacy hooks.code ───────────────────────── @@ -116,6 +130,19 @@ def test_hook_config_captures_code_field(self): assert req.hooks.code == LEGACY_HOOKS["code"] assert req.hooks.hooks == [] + @pytest.mark.parametrize("code", [ + "def hook(): ...", # bare string + {"before_goto": {"nested": "dict"}}, # non-string values + ["a", "list"], # wrong container entirely + ]) + def test_hook_code_accepts_any_legacy_shape(self, code): + """The 0.8.x wire shape was never pinned; a 422 on a field we only + report about would be worse than the silent drop it replaces.""" + from schemas import CrawlRequestWithHooks + + req = CrawlRequestWithHooks(urls=["https://x"], hooks={"code": code}) + assert req.hooks.code == code + # Any hooks payload is refused while hooks are disabled, but the detail # must not mislead legacy callers: enabling the flag would not run # hooks.code (removed in 0.9.0), so payloads carrying it get a removal @@ -191,27 +218,5 @@ def test_no_hooks_response_untouched(self, server_module, monkeypatch, stock_cli assert "hooks" not in r.json() -# ───────────────────────── compose file ───────────────────────── - - -class TestComposeFile: - def test_pid_cap_lives_under_deploy_limits(self): - """Compose v5 rejects pids_limit next to deploy.resources.limits - ("can't set distinct values"); the cap must be expressed once, under - deploy.resources.limits.pids.""" - import os - - override = os.environ.get("CRAWL4AI_COMPOSE_FILE") - if override: - compose_path = Path(override) - else: - here = Path(__file__).resolve() - if len(here.parents) < 4: - pytest.skip("not running from a repo checkout") - compose_path = here.parents[3] / "docker-compose.yml" - if not compose_path.exists(): - pytest.skip(f"docker-compose.yml not found at {compose_path}") - doc = yaml.safe_load(compose_path.read_text()) - base = doc["x-base-config"] - assert "pids_limit" not in base - assert base["deploy"]["resources"]["limits"]["pids"] == 512 +# The compose PID-cap check lives in +# test_security_container_posture.py::test_pids_limit (YAML-parsed). diff --git a/deploy/docker/tests/test_security_2026_04.py b/deploy/docker/tests/test_security_2026_04.py index 4109de046..e015274b6 100644 --- a/deploy/docker/tests/test_security_2026_04.py +++ b/deploy/docker/tests/test_security_2026_04.py @@ -82,23 +82,29 @@ def validate_webhook_url(url): # ============================================================================ class TestOutputPathRemoved(unittest.TestCase): - """output_path is gone; the server owns paths via the artifact store. + """output_path never reaches the filesystem; the server owns paths via the + artifact store. The old string-only validate_output_path was bypassable (symlink/TOCTOU, sibling-prefix '...-evil') -> arbitrary write -> RCE. The fix is to never - accept a caller path at all. Behavioral artifact-store coverage (O_NOFOLLOW, - O_EXCL, hex id, TTL, quota) lives in test_security_artifact_store.py. + use a caller path: output_path exists only as a deprecated no-op for 0.8.x + compatibility and must stay that way. Artifact-store coverage lives in + test_security_artifact_store.py; warning behavior in test_legacy_compat.py. """ - def test_screenshot_request_has_no_output_path(self): + def test_screenshot_output_path_is_deprecated_noop(self): sys.path.insert(0, DEPLOY_DIR) from schemas import ScreenshotRequest - self.assertNotIn("output_path", ScreenshotRequest.model_fields) + field = ScreenshotRequest.model_fields["output_path"] + self.assertTrue(field.deprecated, "output_path must be marked deprecated") + self.assertIsNone(field.default) - def test_pdf_request_has_no_output_path(self): + def test_pdf_output_path_is_deprecated_noop(self): sys.path.insert(0, DEPLOY_DIR) from schemas import PDFRequest - self.assertNotIn("output_path", PDFRequest.model_fields) + field = PDFRequest.model_fields["output_path"] + self.assertTrue(field.deprecated, "output_path must be marked deprecated") + self.assertIsNone(field.default) def test_validate_output_path_deleted(self): sys.path.insert(0, DEPLOY_DIR) @@ -110,18 +116,21 @@ def test_validate_output_path_deleted(self): class TestPydanticPathValidator(unittest.TestCase): - """output_path (and its traversal validator) are gone entirely. + """The traversal validator is gone entirely. Traversal rejection used to be the mitigation; the real fix is that no - caller path is accepted at all, so there is nothing to traverse. The - sandboxed artifact store owns all paths now. + caller path is ever *used*, so there is nothing to traverse. output_path + survives only as a deprecated no-op field (see TestOutputPathRemoved); the + sandboxed artifact store owns all paths. """ - def test_no_output_path_field_on_request_models(self): + def test_output_path_is_inert_on_request_models(self): sys.path.insert(0, DEPLOY_DIR) from schemas import ScreenshotRequest, PDFRequest - self.assertNotIn("output_path", ScreenshotRequest.model_fields) - self.assertNotIn("output_path", PDFRequest.model_fields) + for model in (ScreenshotRequest, PDFRequest): + field = model.model_fields["output_path"] + self.assertTrue(field.deprecated) + self.assertIsNone(field.default) def test_traversal_validator_removed(self): # No reject_traversal validator should remain registered on the models. diff --git a/deploy/docker/tests/test_security_container_posture.py b/deploy/docker/tests/test_security_container_posture.py index 3399d6bd1..eb5df6153 100644 --- a/deploy/docker/tests/test_security_container_posture.py +++ b/deploy/docker/tests/test_security_container_posture.py @@ -104,7 +104,14 @@ def test_no_host_dev_shm_bind(self, compose): assert "shm_size" in compose def test_pids_limit(self, compose): - assert "pids_limit" in compose + # Parse, don't grep: a raw-text search matched the word "pids_limit" + # inside a comment and guarded nothing. The cap lives under + # deploy.resources.limits (not pids_limit) for Compose v5 compatibility. + import yaml + + base = yaml.safe_load(compose)["x-base-config"] + assert "pids_limit" not in base + assert base["deploy"]["resources"]["limits"]["pids"] == 512 def test_read_only_runtime_tmpfs_are_appuser_owned(self, compose): assert "/var/lib/redis:uid=999,gid=999,mode=0700" in compose From e07ac4fd3201d3c7582a27865392fdc08b4f3268 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Thu, 20 Aug 2026 12:06:21 +0530 Subject: [PATCH 24/38] fix(docker): forward CRAWL4AI_API_TOKEN through compose, make .llm.env optional Pass the token from the shell (or project .env) via environment, make .llm.env optional with required: false, drop the obsolete version key, and align entrypoint hint and docs --- deploy/docker/.llm.env.example | 8 ------- deploy/docker/MIGRATION.md | 13 ++++++----- deploy/docker/entrypoint.sh | 2 +- docker-compose.yml | 10 +++++---- docs/md_v2/core/self-hosting.md | 39 +++++++++++++++++++-------------- 5 files changed, 37 insertions(+), 35 deletions(-) diff --git a/deploy/docker/.llm.env.example b/deploy/docker/.llm.env.example index 2d7a2ea25..6f84f1ff1 100644 --- a/deploy/docker/.llm.env.example +++ b/deploy/docker/.llm.env.example @@ -1,11 +1,3 @@ -# REQUIRED for a reachable server: API token for the Docker server (0.9.0+). -# Without it the server binds loopback inside the container and the published -# port answers with "connection reset". Any non-empty value works, but treat it -# as a password — use a long random string (e.g. from: openssl rand -hex 32). -# Note: with docker compose, the token MUST be set here — exporting it in your -# shell does not reach the container. -CRAWL4AI_API_TOKEN= - # Optional: enable declarative hooks support (disabled by default) # CRAWL4AI_HOOKS_ENABLED=true diff --git a/deploy/docker/MIGRATION.md b/deploy/docker/MIGRATION.md index b895ae588..bcd2097c7 100644 --- a/deploy/docker/MIGRATION.md +++ b/deploy/docker/MIGRATION.md @@ -25,12 +25,13 @@ loopback by default and will not expose itself without a credential. export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)" ``` -> ⚠️ **Docker Compose users:** `export` alone does **not** work — the shipped -> `docker-compose.yml` does not forward host environment variables. Set the -> token in the `.llm.env` file at the project root instead (the example file -> ships an empty `CRAWL4AI_API_TOKEN=` line — fill it in). -> -> For plain `docker run`, pass it explicitly: +With `docker compose`, run the export in the same shell before +`docker compose up`; the compose file passes the token into the container. +For a persistent setup, put the `CRAWL4AI_API_TOKEN=...` line in a `.env` +file in the project root instead — compose reads it automatically (a shell +export still takes precedence). + +> ⚠️ For plain `docker run`, pass it explicitly: > `-e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN"` (the value-less shorthand > `-e CRAWL4AI_API_TOKEN` silently passes empty from a shell where the variable > isn't set). diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh index 3d0037916..22c7483c5 100644 --- a/deploy/docker/entrypoint.sh +++ b/deploy/docker/entrypoint.sh @@ -32,7 +32,7 @@ else GUNICORN_BIND="127.0.0.1:${PORT}" echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2 echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2 - echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose), then restart. (If you enabled security.jwt_enabled in a custom config.yml, set CRAWL4AI_JWT_ENABLED=true instead.)" >&2 + echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or 'export CRAWL4AI_API_TOKEN=...' before 'docker compose up'), then restart. (If you enabled security.jwt_enabled in a custom config.yml, set CRAWL4AI_JWT_ENABLED=true instead.)" >&2 fi export GUNICORN_BIND diff --git a/docker-compose.yml b/docker-compose.yml index 5939b3321..6beb00169 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,15 @@ -version: '3.8' - # Shared configuration for all environments x-base-config: &base-config ports: - "11235:11235" # Gunicorn port env_file: - - .llm.env # API keys (create from .llm.env.example) + # API keys (create from .llm.env.example); optional so a fresh clone runs. + - path: .llm.env + required: false + environment: + # Auth token passthrough from the host shell (overwrites .llm.env). + - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-} # Uncomment to set default environment variables (will overwrite .llm.env) - # environment: # - OPENAI_API_KEY=${OPENAI_API_KEY:-} # - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-} # - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.md index 966672f04..09e5e37a5 100644 --- a/docs/md_v2/core/self-hosting.md +++ b/docs/md_v2/core/self-hosting.md @@ -62,7 +62,7 @@ When you self-host, you can scale from a single container to a full browser infr ## Prerequisites Before we dive in, make sure you have: -- Docker installed and running (version 20.10.0 or higher), including `docker compose` (usually bundled with Docker Desktop). +- Docker installed and running (version 20.10.0 or higher), including `docker compose` v2.24+ (usually bundled with Docker Desktop). - `git` for cloning the repository. - At least 4GB of RAM available for the container (more recommended for heavy use). - Python 3.10+ (if using the Python SDK). @@ -212,29 +212,36 @@ cd crawl4ai #### 2. Environment Setup (Required) -The compose file loads `.llm.env` from the **project root directory** — the -file must exist even if you don't use LLMs, or compose will fail with -"env file .llm.env not found". Create it from the example and add an API token: +Export an API token in your shell — the compose file passes it into the +container. **Required**, or the server will be unreachable (loopback-only, +published port → connection reset): ```bash -# Make sure you are in the 'crawl4ai' root directory -cp deploy/docker/.llm.env.example .llm.env +export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)" ``` -Then open `.llm.env` and fill in the `CRAWL4AI_API_TOKEN=` line at the top — -**required**, or the server will be unreachable (loopback-only). Any long -random string works, e.g. from `openssl rand -hex 32`. One-liner: +Prefer a file? Put the same line in a `.env` file in the project root — +compose reads it automatically on every run, no export needed (if both are +set, the shell export wins): ```bash -sed -i.bak "s|^CRAWL4AI_API_TOKEN=.*|CRAWL4AI_API_TOKEN=$(openssl rand -hex 32)|" .llm.env && rm .llm.env.bak +echo "CRAWL4AI_API_TOKEN=$(openssl rand -hex 32)" > .env ``` -Optionally add your LLM API keys in the same file. +If you use LLMs, also create `.llm.env` in the **project root directory** with +your API keys (optional — compose starts fine without it): + +```bash +# Make sure you are in the 'crawl4ai' root directory +cp deploy/docker/.llm.env.example .llm.env + +# Now edit .llm.env and add your LLM API keys +``` -> ⚠️ **The token must go inside `.llm.env`.** `export CRAWL4AI_API_TOKEN=...` -> in your shell does **not** work with compose — the compose file does not -> forward host environment variables, and the server silently starts in -> loopback-only mode (published port → connection reset). +> ⚠️ Run the export in the **same shell** you run `docker compose up` from. +> With compose, only the shell export or a `.env` line carries the token — a +> `CRAWL4AI_API_TOKEN=` line in `.llm.env` is overridden by the compose +> passthrough. **Flexible LLM Provider Configuration:** @@ -305,7 +312,7 @@ The `docker-compose.yml` file in the project root provides a simplified approach > The server will be available at `http://localhost:11235` (allow ~10 seconds > for startup). All endpoints except `GET /health` require -> `Authorization: Bearer `. +> `Authorization: Bearer `. #### 4. Stopping the Service From fe4d4b493b04bcdf6b27c5ecef05eaf3152c7260 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Thu, 20 Aug 2026 09:46:01 +0200 Subject: [PATCH 25/38] fix(docker): align commented env var lines with the environment list The optional LLM-key lines sat at 5-space indent once uncommented, while the live entry uses 4. Removing the '#' produced 'did not find expected - indicator' and broke every compose command until re-indented by hand. Co-Authored-By: Claude Opus 5 (1M context) --- docker-compose.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6beb00169..02cbe0584 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,15 +9,15 @@ x-base-config: &base-config environment: # Auth token passthrough from the host shell (overwrites .llm.env). - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-} - # Uncomment to set default environment variables (will overwrite .llm.env) - # - OPENAI_API_KEY=${OPENAI_API_KEY:-} - # - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-} - # - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} - # - GROQ_API_KEY=${GROQ_API_KEY:-} - # - TOGETHER_API_KEY=${TOGETHER_API_KEY:-} - # - MISTRAL_API_KEY=${MISTRAL_API_KEY:-} - # - GEMINI_API_KEY=${GEMINI_API_KEY:-} - # - LLM_PROVIDER=${LLM_PROVIDER:-} # Optional: Override default provider (e.g., "anthropic/claude-3-opus") + # Uncomment to set default environment variables (will overwrite .llm.env) + # - OPENAI_API_KEY=${OPENAI_API_KEY:-} + # - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-} + # - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} + # - GROQ_API_KEY=${GROQ_API_KEY:-} + # - TOGETHER_API_KEY=${TOGETHER_API_KEY:-} + # - MISTRAL_API_KEY=${MISTRAL_API_KEY:-} + # - GEMINI_API_KEY=${GEMINI_API_KEY:-} + # - LLM_PROVIDER=${LLM_PROVIDER:-} # Optional: Override default provider (e.g., "anthropic/claude-3-opus") # Chromium needs shared memory but the host /dev/shm bind was a shared, # writable mount; use a private sized tmpfs instead. shm_size: "1gb" From ffd9a36f23fd51f79b8be24acda89714d7e377fa Mon Sep 17 00:00:00 2001 From: ntohidi Date: Thu, 20 Aug 2026 09:56:26 +0200 Subject: [PATCH 26/38] fix(docker): redirect /monitor to the dashboard UI (issue #2091, point 7) Pre-0.9 docs pointed at /monitor for the dashboard, which now lives at /dashboard. /monitor is the monitoring API prefix with no page of its own, so a browser landed on a bare 401 with no pointer to the real UI. Redirect the exact path /monitor -> /dashboard and allow it in the auth gate's exact-match public_paths. The /monitor/* API routes - including /monitor/ws and the destructive admin actions - stay gated; making the prefix public would expose them unauthenticated. Tests pin both halves and were mutation-checked: widening the gate to the /monitor prefix fails the gated-route and websocket tests, and dropping /monitor from public_paths fails the redirect test. Co-Authored-By: Claude Opus 5 (1M context) --- deploy/docker/server.py | 13 +++++- deploy/docker/tests/test_legacy_compat.py | 51 ++++++++++++++++++----- docs/md_v2/core/self-hosting.md | 7 ++-- 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/deploy/docker/server.py b/deploy/docker/server.py index eb0fddadf..70352913a 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -292,6 +292,14 @@ async def _timeline_updater(): async def root(): return RedirectResponse("/playground") + +# Pre-0.9 docs pointed at /monitor for the dashboard UI, which now lives at +# /dashboard; /monitor is the monitoring API prefix and has no page of its own. +# Only this exact path redirects - /monitor/* stays gated (see public_paths). +@app.get("/monitor", include_in_schema=False) +async def monitor_ui_redirect(): + return RedirectResponse("/dashboard") + # ─────────────────── infra / middleware ───────────────────── redis = aioredis.from_url(_build_redis_url(config)) @@ -392,7 +400,10 @@ def _current_api_token() -> str: app.add_middleware( AuthGateMiddleware, token_provider=_current_api_token, - public_paths={HEALTH_PATH, "/token", "/"}, + # Exact paths only: "/monitor" reaches the redirect above, while every + # "/monitor/*" API route (incl. /monitor/ws and the admin actions) keeps + # requiring a credential. + public_paths={HEALTH_PATH, "/token", "/", "/monitor"}, public_prefixes=_UI_PREFIXES, ) diff --git a/deploy/docker/tests/test_legacy_compat.py b/deploy/docker/tests/test_legacy_compat.py index 2a76d6e27..a8a4039e3 100644 --- a/deploy/docker/tests/test_legacy_compat.py +++ b/deploy/docker/tests/test_legacy_compat.py @@ -1,8 +1,9 @@ """ Behavioral tests for 0.9.x legacy-compatibility handling: - * root redirect - "/" is public and redirects to /playground instead of - dying in the auth gate with a bare 401; /monitor and the + * UI redirects - "/" and "/monitor" are public and redirect to + /playground and /dashboard instead of dying in the auth + gate with a bare 401; the /monitor/* API routes and the data routes stay gated. * output_path - /screenshot and /pdf still accept the 0.8.x output_path field but return a warning saying no file was written, @@ -30,10 +31,10 @@ def _bearer() -> dict: return {"Authorization": f"Bearer {create_access_token({'sub': 'user@x.com'}, scope='data')}"} -# ───────────────────────── root redirect ───────────────────────── +# ───────────────────────── UI redirects ───────────────────────── -class TestRootRedirect: +class TestUiRedirects: def test_root_is_public_and_redirects_to_playground(self, stock_client): r = stock_client.get("/", follow_redirects=False) assert r.status_code in (302, 307), ( @@ -42,13 +43,43 @@ def test_root_is_public_and_redirects_to_playground(self, stock_client): ) assert r.headers["location"] == "/playground" - def test_monitor_and_data_routes_stay_gated(self, stock_client): - # /monitor must not serve content without a token; a future - # /monitor -> /dashboard redirect is fine (the target is UI-public), - # so accept 401 or a redirect, never 200. + def test_monitor_redirects_to_dashboard(self, stock_client): + # Pre-0.9 docs sent people to /monitor for the dashboard UI. That exact + # path redirects to /dashboard (UI-public) instead of dead-ending in the + # auth gate with a bare 401. r = stock_client.get("/monitor", follow_redirects=False) - assert r.status_code in (401, 302, 307, 308) - assert stock_client.get("/monitor/health").status_code == 401 + assert r.status_code in (302, 307), ( + f"GET /monitor returned {r.status_code}; expected a redirect. The " + f"auth gate must allow the exact path '/monitor' so the route runs." + ) + assert r.headers["location"] == "/dashboard" + + @pytest.mark.parametrize( + "method,path", + [ + ("get", "/monitor/health"), + ("get", "/monitor/requests"), + ("get", "/monitor/browsers"), + ("get", "/monitor/timeline"), + ("get", "/monitor/logs/errors"), + ("post", "/monitor/actions/cleanup"), + ("post", "/monitor/actions/kill_browser"), + ("post", "/monitor/stats/reset"), + ], + ) + def test_monitor_api_routes_stay_gated(self, stock_client, method, path): + # The redirect above is exact-path only. Making the /monitor *prefix* + # public would expose request logs, browser state and the destructive + # admin actions without a credential. + assert getattr(stock_client, method)(path).status_code == 401 + + def test_monitor_websocket_stays_gated(self, stock_client): + # /monitor/ws is where live stats stream; it must not open unauthenticated. + with pytest.raises(Exception): + with stock_client.websocket_connect("/monitor/ws"): + pass + + def test_data_routes_stay_gated(self, stock_client): assert stock_client.post("/crawl", json={"urls": ["https://x"]}).status_code == 401 diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.md index 09e5e37a5..68a8a1065 100644 --- a/docs/md_v2/core/self-hosting.md +++ b/docs/md_v2/core/self-hosting.md @@ -1495,9 +1495,10 @@ Access the **built-in real-time monitoring dashboard** for complete operational http://localhost:11235/dashboard ``` -> ⚠️ The dashboard UI lives at `/dashboard` — **not** `/monitor`, which is the -> API namespace (`/monitor/health`, `/monitor/ws`, …) and returns -> `{"detail": "Authentication required"}` in a browser. On the dashboard, paste +> ⚠️ The dashboard UI lives at `/dashboard`. `/monitor` is the API namespace +> (`/monitor/health`, `/monitor/ws`, …); older docs pointed there, so that exact +> URL now redirects to `/dashboard` for convenience — the `/monitor/*` routes +> themselves still require a token. On the dashboard, paste > your API token into the **API token** bar (top right) and click **Set**; the > WebSocket then connects and live stats appear. From d57a31c8d2e75007680a939d651498d4f7a13780 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Thu, 20 Aug 2026 10:02:50 +0200 Subject: [PATCH 27/38] test(docker): cover the per-URL crawler_configs PDF SSRF guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handle_crawl_request builds the per-URL config list on a separate branch from the top-level config, so the url_validator wiring has to be repeated there. #2150 fixed that, but nothing exercised it — the existing pairing tests only go through the top-level path, leaving the guard free to be dropped by a future refactor without a red test. Verified by mutation: removing the guard from api.py fails this test and nothing else. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_docker_pdf_crawler_pairing.py | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_docker_pdf_crawler_pairing.py b/tests/test_docker_pdf_crawler_pairing.py index 345c6c420..20e94ed1f 100644 --- a/tests/test_docker_pdf_crawler_pairing.py +++ b/tests/test_docker_pdf_crawler_pairing.py @@ -50,6 +50,7 @@ def pool_mock(api, monkeypatch): pooled = MagicMock() pooled.arun = AsyncMock(return_value=[]) + pooled.arun_many = AsyncMock(return_value=[]) # per-URL config list path pooled.active_requests = 1 # release_crawler decrements this int mock = AsyncMock(return_value=pooled) monkeypatch.setattr(crawler_pool, "get_crawler", mock) @@ -127,6 +128,40 @@ async def test_default_strategy_still_uses_pool(api, pool_mock): pool_mock.assert_awaited_once() +@pytest.mark.asyncio +async def test_per_url_pdf_strategy_gets_validator(api, pool_mock): + """SSRF: a PDF strategy sent per-URL via crawler_configs must be vetted too. + + crawler_configs is a public per-URL field on /crawl, and the handler builds + that config list on a separate branch from the top-level config. Wiring + url_validator only on the top-level branch leaves the per-URL one doing an + unvalidated download of whatever the request names. + """ + pdf_config = _crawler_config_payload(with_pdf_strategy=True) + plain_config = _crawler_config_payload(with_pdf_strategy=False) + + response = await api.handle_crawl_request( + # The config list branch only engages with more than one URL. + urls=["https://example.com/document.pdf", "https://example.com/page.html"], + browser_config={"type": "BrowserConfig", "params": {}}, + crawler_config=plain_config, # top-level is clean; the PDF rides per-URL + crawler_configs=[pdf_config, plain_config], + config=CONFIG, + ) + + assert response["success"] is True + configs = pool_mock.return_value.arun_many.await_args.kwargs["config"] + pdf_strategies = [ + cfg.scraping_strategy for cfg in configs + if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) + ] + # Guard the guard: if deserialization ever drops the strategy, the loop + # below would pass vacuously and the test would protect nothing. + assert len(pdf_strategies) == 1 + for strategy in pdf_strategies: + assert strategy.url_validator is api.validate_url_destination + + # --------------------------------------------------------------------------- # PDF download redirect handling (url_validator SSRF guard) # --------------------------------------------------------------------------- From a86fe353b08e82c26a5cea8db90fe8e7e4b8c39a Mon Sep 17 00:00:00 2001 From: ntohidi Date: Thu, 20 Aug 2026 10:26:03 +0200 Subject: [PATCH 28/38] fix(pdf): carry cookies across manual redirect hops #2150 switched the PDF download to allow_redirects=False and followed hops by hand so url_validator can vet each one before it is fetched. Each hop used a bare requests.get(), which starts with an empty cookie jar, so a host that sets a cookie and then redirects never gets its own cookie back and answers 403. That is the normal shape for gated and CDN-signed PDFs, and it worked before #2150 because allow_redirects=True carried cookies implicitly. Use one requests.Session for the chain. The SSRF guarantee is unchanged: hops are still validated before the fetch and still not auto-followed. Verified by mutation: reverting session.get to requests.get fails the new test and nothing else. Co-Authored-By: Claude Opus 5 (1M context) --- crawl4ai/processors/pdf/__init__.py | 10 ++++++-- tests/test_docker_pdf_crawler_pairing.py | 32 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/crawl4ai/processors/pdf/__init__.py b/crawl4ai/processors/pdf/__init__.py index 7722fe7fd..b41677c6b 100644 --- a/crawl4ai/processors/pdf/__init__.py +++ b/crawl4ai/processors/pdf/__init__.py @@ -171,11 +171,17 @@ def _get_pdf_path(self, url: str) -> str: # Redirects are followed manually so url_validator (when set) can vet every hop BEFORE it is fetched from urllib.parse import urljoin current_url = url + # One Session for the whole chain: a bare requests.get() per hop + # starts with an empty cookie jar, so a host that sets a cookie + # and then redirects (common for gated/CDN-signed PDFs) would + # get its own cookie back. allow_redirects=True used to carry + # them for us; following hops by hand means we carry them here. + session = requests.Session() for _ in range(MAX_PDF_DOWNLOAD_REDIRECTS): if self.url_validator: self.url_validator(current_url) - response = requests.get(current_url, stream=True, timeout=(20, 60 * 10), - allow_redirects=False) + response = session.get(current_url, stream=True, timeout=(20, 60 * 10), + allow_redirects=False) if response.is_redirect: location = response.headers.get("location") response.close() # hop response holds its connection open (stream=True) diff --git a/tests/test_docker_pdf_crawler_pairing.py b/tests/test_docker_pdf_crawler_pairing.py index 345c6c420..c6c5f1f1a 100644 --- a/tests/test_docker_pdf_crawler_pairing.py +++ b/tests/test_docker_pdf_crawler_pairing.py @@ -154,6 +154,21 @@ def do_GET(self): self.send_response(302) self.send_header("Location", "/loop") self.end_headers() + elif self.path == "/gated": + # Sets a cookie, then redirects to a target that demands it. + self.send_response(302) + self.send_header("Set-Cookie", "sess=abc123; Path=/") + self.send_header("Location", "/gated-doc.pdf") + self.end_headers() + elif self.path == "/gated-doc.pdf": + if "sess=abc123" not in self.headers.get("Cookie", ""): + self.send_response(403) + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Type", "application/pdf") + self.end_headers() + self.wfile.write(pdf_bytes) else: self.send_response(200) self.send_header("Content-Type", "application/pdf") @@ -199,6 +214,23 @@ def test_download_redirects_still_followed_without_validator(redirect_server): Path(path).unlink(missing_ok=True) +def test_download_carries_cookies_across_redirect_hops(redirect_server): + """Cookies set by a redirecting host must reach the next hop. + + Following redirects by hand (needed so url_validator can vet each hop) + means a bare requests.get() per hop starts with an empty cookie jar, so + a host that sets a cookie and then redirects never gets it back. That is + the normal shape for gated or CDN-signed PDFs, and allow_redirects=True + used to handle it implicitly. + """ + strategy = PDFContentScrapingStrategy() + path = strategy._get_pdf_path(f"{redirect_server}/gated") + try: + assert Path(path).read_bytes().startswith(b"%PDF") + finally: + Path(path).unlink(missing_ok=True) + + def test_download_redirect_loop_aborts(redirect_server): """An endless redirect chain must abort after the cap, not hang.""" strategy = PDFContentScrapingStrategy() From a574fbc5ae585c3d91dabcd9862be7a87bbd6677 Mon Sep 17 00:00:00 2001 From: Krzysztof Olipra Date: Thu, 20 Aug 2026 12:59:38 +0200 Subject: [PATCH 29/38] fix: remove_overlay_elements can remove body if it has a global popup class. --- .../js_snippet/remove_overlay_elements.js | 21 +++-- tests/test_issue_2161_overlay_html_body.py | 81 +++++++++++++++++++ 2 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 tests/test_issue_2161_overlay_html_body.py diff --git a/crawl4ai/js_snippet/remove_overlay_elements.js b/crawl4ai/js_snippet/remove_overlay_elements.js index a50d94274..2b274ac66 100644 --- a/crawl4ai/js_snippet/remove_overlay_elements.js +++ b/crawl4ai/js_snippet/remove_overlay_elements.js @@ -5,6 +5,12 @@ async () => { return style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0"; }; + // Never strip document structure , , or ; removing those tags empties the page. + const isDocumentStructure = (elem) => { + const tag = elem && elem.tagName; + return tag === "HTML" || tag === "HEAD" || tag === "BODY"; + }; + // Common selectors for popups and overlays const commonSelectors = [ // Close buttons first @@ -54,6 +60,7 @@ async () => { // Find elements with high z-index const allElements = document.querySelectorAll("*"); for (const elem of allElements) { + if (isDocumentStructure(elem)) continue; const style = window.getComputedStyle(elem); const zIndex = parseInt(style.zIndex); const position = style.position; @@ -74,6 +81,7 @@ async () => { for (const selector of commonSelectors) { const elements = document.querySelectorAll(selector); elements.forEach((elem) => { + if (isDocumentStructure(elem)) return; if (isVisible(elem)) { elem.remove(); } @@ -88,6 +96,7 @@ async () => { const removeFixedElements = () => { const elements = document.querySelectorAll("*"); elements.forEach((elem) => { + if (isDocumentStructure(elem)) return; const style = window.getComputedStyle(elem); if ((style.position === "fixed" || style.position === "sticky") && isVisible(elem)) { elem.remove(); @@ -110,11 +119,13 @@ async () => { }; // Remove margin-right and padding-right from body (often added by modal scripts) - document.body.style.marginRight = "0px"; - document.body.style.paddingRight = "0px"; - document.body.style.overflow = "auto"; + if (document.body) { + document.body.style.marginRight = "0px"; + document.body.style.paddingRight = "0px"; + document.body.style.overflow = "auto"; - // Wait a bit for any animations to complete - document.body.scrollIntoView(false); + // Wait a bit for any animations to complete + document.body.scrollIntoView(false); + } await new Promise((resolve) => setTimeout(resolve, 50)); }; diff --git a/tests/test_issue_2161_overlay_html_body.py b/tests/test_issue_2161_overlay_html_body.py new file mode 100644 index 000000000..7855a8e87 --- /dev/null +++ b/tests/test_issue_2161_overlay_html_body.py @@ -0,0 +1,81 @@ +"""Tests for issue #2161: remove_overlay_elements can remove body if it has a global popup class. + +https://github.com/unclecode/crawl4ai/issues/2161 + +WordPress themes such as Qode/Bridge put classes like +`.qode_popup_menu_push_text_right` on . The overlay snippet matches +`[class*="popup"]` and used to delete the entire document, leaving an empty page. +""" + +import pytest + +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig + + +QODE_BODY_HTML = """\ + + +Qode popup body class + +
+

Tunnel Girona

+

Portfolio page content that must survive overlay removal.

+
+ + + +""" + +FIXED_BODY_HTML = """\ + + +Fixed body scroll lock + +
+

Locked body

+

Body is position:fixed while a modal is open; it must not be removed.

+
+
+ Newsletter popup +
+ + +""" + + +@pytest.mark.asyncio +async def test_overlay_removal_keeps_body_with_popup_in_class(): + """#2161: Body/html with a class substring 'popup' must remain after overlay removal.""" + async with AsyncWebCrawler() as crawler: + result = await crawler.arun( + f"raw:{QODE_BODY_HTML}", + config=CrawlerRunConfig(remove_overlay_elements=True, verbose=False), + ) + + assert result.success, result.error_message + html = result.html.lower() + assert " Date: Thu, 20 Aug 2026 13:07:42 +0200 Subject: [PATCH 30/38] fix: changes. Removed unnecessary wait for animations before scrolling. --- crawl4ai/js_snippet/remove_overlay_elements.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/crawl4ai/js_snippet/remove_overlay_elements.js b/crawl4ai/js_snippet/remove_overlay_elements.js index c7ccf386a..c134cc6b1 100644 --- a/crawl4ai/js_snippet/remove_overlay_elements.js +++ b/crawl4ai/js_snippet/remove_overlay_elements.js @@ -123,7 +123,5 @@ async () => { document.body.style.paddingRight = "0px"; document.body.style.overflow = "auto"; - // Wait a bit for any animations to complete document.body.scrollIntoView(false); - await new Promise((resolve) => setTimeout(resolve, 50)); }; From 701b0b197092d5d7981f3fdb655aaae8b40a69e0 Mon Sep 17 00:00:00 2001 From: Soham Kukreti Date: Thu, 20 Aug 2026 13:51:46 +0530 Subject: [PATCH 31/38] fix(browser): clean up leaked Playwright driver when launch fails in __aenter__ Roll back partial startup in BrowserManager.start() so a failed browser launch stops the already-started driver process before re-raising. Fixes #2155 --- crawl4ai/browser_manager.py | 29 +++++++++- tests/regression/test_reg_browser.py | 82 ++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/crawl4ai/browser_manager.py b/crawl4ai/browser_manager.py index d7b8009c8..f4ab0aa32 100644 --- a/crawl4ai/browser_manager.py +++ b/crawl4ai/browser_manager.py @@ -626,7 +626,15 @@ async def acquire(cls, cdp_url: str, use_undetected: bool = False): else: from playwright.async_api import async_playwright pw = await async_playwright().start() - browser = await pw.chromium.connect_over_cdp(cdp_url) + try: + browser = await pw.chromium.connect_over_cdp(cdp_url) + except BaseException: + # Stop the driver we just started so a failed connect doesn't leak it + try: + await pw.stop() + except BaseException: + pass + raise cls._cache[cdp_url] = (pw, browser, 1) return pw, browser @@ -788,6 +796,25 @@ async def start(self): Note: This method should be called in a separate task to avoid blocking the main event loop. """ + try: + await self._start_impl() + except BaseException: + # Roll back partial startup (e.g. Playwright driver) since a failed __aenter__ never triggers __aexit__ + try: + await self.close() + except BaseException: + pass + if self.playwright is not None: + # close() intentionally skips the driver for external-CDP configs; on failed startup we still own it + try: + await self.playwright.stop() + except BaseException: + pass + self.playwright = None + self.browser = None + raise + + async def _start_impl(self): if self.playwright is not None: await self.close() diff --git a/tests/regression/test_reg_browser.py b/tests/regression/test_reg_browser.py index 2d9f87fb4..dac55a841 100644 --- a/tests/regression/test_reg_browser.py +++ b/tests/regression/test_reg_browser.py @@ -7,6 +7,7 @@ real browser crawling with no mocking. """ +import asyncio import base64 import time @@ -37,6 +38,87 @@ async def test_browser_lifecycle(local_server): await crawler.close() +@pytest.mark.asyncio +async def test_failed_browser_launch_leaves_no_driver(monkeypatch): + """Issue #2155: a browser-launch failure inside __aenter__ must roll back + partial startup — no leaked Playwright driver process, no half-built state. + + Launch failure is forced by pointing PLAYWRIGHT_BROWSERS_PATH at an empty + directory, so the driver starts but the browser executable is missing. + """ + import tempfile + + import psutil + + from crawl4ai.browser_manager import BrowserManager + + monkeypatch.setenv( + "PLAYWRIGHT_BROWSERS_PATH", tempfile.mkdtemp(prefix="c4ai-no-browsers-") + ) + before = {p.pid for p in psutil.Process().children(recursive=True)} + + manager = BrowserManager(browser_config=BrowserConfig(headless=True)) + with pytest.raises(Exception, match="Executable doesn't exist"): + await manager.start() + + # Half-built state must be rolled back (this also proves the driver was + # stopped, since only close()/stop() reset it). + assert manager.playwright is None, "playwright driver not rolled back" + assert manager.browser is None, "browser reference not rolled back" + + # No Playwright driver child process may survive the failed start. + await asyncio.sleep(0.5) # allow the stopped subprocess to be reaped + leaked = [] + for child in psutil.Process().children(recursive=True): + if child.pid in before: + continue + try: + cmd = " ".join(child.cmdline()) + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + if "playwright" in cmd and "run-driver" in cmd: + leaked.append(child.pid) + assert leaked == [], f"leaked Playwright driver process(es): {leaked}" + + +@pytest.mark.asyncio +async def test_failed_cached_cdp_connect_leaves_no_driver(): + """Issue #2155: a failed connect inside _CDPConnectionCache.acquire() must + stop the driver it started (the manager never sees it, so start()'s + rollback alone cannot clean it up).""" + import psutil + + from crawl4ai.browser_manager import BrowserManager + + before = {p.pid for p in psutil.Process().children(recursive=True)} + config = BrowserConfig( + headless=True, + cdp_url="http://127.0.0.1:9", # unreachable endpoint + cache_cdp_connection=True, + ) + manager = BrowserManager(browser_config=config) + # Matching on connect_over_cdp proves the driver was already started + # when the failure happened (the scenario under test). + with pytest.raises(Exception, match="connect_over_cdp"): + await manager.start() + + assert manager.playwright is None + assert manager.browser is None + + await asyncio.sleep(0.5) + leaked = [] + for child in psutil.Process().children(recursive=True): + if child.pid in before: + continue + try: + cmd = " ".join(child.cmdline()) + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + if "playwright" in cmd and "run-driver" in cmd: + leaked.append(child.pid) + assert leaked == [], f"leaked Playwright driver process(es): {leaked}" + + @pytest.mark.asyncio async def test_browser_context_manager(local_server): """Verify async with pattern works and cleanup happens without error.""" From 3273f756bbd0f962e803a11ff520ea07e91b18aa Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 24 Aug 2026 11:41:00 +0200 Subject: [PATCH 32/38] fix(security): block PDF image-write fields from untrusted config bodies PDFContentScrapingStrategy is an UNTRUSTED_ALLOWED_TYPE but had no field allowlist, so _filter_untrusted_fields fell open and kept its filesystem-write knobs (image_save_dir / save_images_locally). An untrusted request body could steer the PDF image writer to an attacker-chosen directory and write attacker-controlled image-stream bytes there. - Forbid image_save_dir / save_images_locally on PDFContentScrapingStrategy. - Add UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS: a fail-closed backstop of write- and code-bearing arg names checked on every untrusted type, so a future strategy cannot reopen the same class of hole. - extract_images stays allowed (base64-inline, no disk write). - Trusted (SDK/in-process) construction is unchanged. Add regression tests exercising the real from_serializable_dict / CrawlerRunConfig.load(provenance=UNTRUSTED) path. Reported by sec-reex. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 215bb4c7d30daaf7e044bdc5ebd810754d2d0e82) --- crawl4ai/async_configs.py | 25 +++++- .../tests/test_security_pdf_image_write.py | 83 +++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 deploy/docker/tests/test_security_pdf_image_write.py diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 3efb0d380..87500a90e 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -214,6 +214,29 @@ class UntrustedConfigError(ValueError): "override_navigator", "magic", "process_in_browser", "shared_data", "session_id", }, + # PDFContentScrapingStrategy has no scalar allowlist (it is kept for its + # value fields), so its filesystem-write knobs must be blocked explicitly: + # image_save_dir is an unconfined write destination and save_images_locally + # turns the write on. extract_images stays allowed - it only returns the + # image bytes base64-inline in the response, no disk write. + "PDFContentScrapingStrategy": { + "image_save_dir", "save_images_locally", + }, +} + +# Field names that must NEVER be set from an untrusted body on ANY allowed +# type, checked regardless of per-type allowlist. This is the fail-closed +# backstop: the per-type maps above only cover BrowserConfig/CrawlerRunConfig, +# so a strategy that (now or in future) exposes a filesystem-write, code, or +# routing constructor arg is caught here even with no explicit allowlist. +# Presence => 400 (loud), matching js_code/extra_args behavior. +UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS = { + # filesystem write / read sinks + "image_save_dir", "save_images_locally", "downloads_path", "user_data_dir", + "output_path", "save_path", "file_path", "local_path", "storage_state", + # code / command execution + "js_code", "js_code_before_wait", "c4a_script", "init_scripts", + "code", "command", "hook", "hooks", } # Scalar knobs an untrusted body MAY set, per class. A field not listed here is @@ -281,7 +304,7 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict: allowlist = UNTRUSTED_FIELD_ALLOWLIST.get(type_name) # None => keep all non-forbidden out = {} for key, value in params.items(): - if key in forbidden: + if key in UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS or key in forbidden: raise UntrustedConfigError( f"field '{key}' is not permitted on {type_name} from an untrusted request" ) diff --git a/deploy/docker/tests/test_security_pdf_image_write.py b/deploy/docker/tests/test_security_pdf_image_write.py new file mode 100644 index 000000000..4f171ce4a --- /dev/null +++ b/deploy/docker/tests/test_security_pdf_image_write.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +""" +Regression test for the PDF-image arbitrary-write via untrusted config body +(reported by sec-reex). + +Root cause: PDFContentScrapingStrategy is an UNTRUSTED_ALLOWED_TYPE but had no +field allowlist, so _filter_untrusted_fields fell open and kept its +filesystem-write knobs (image_save_dir / save_images_locally). A request body +the API loads as UNTRUSTED could then steer the PDF image writer to an +attacker-chosen directory. + +These tests hit the real deserialization path (from_serializable_dict / +CrawlerRunConfig.load with provenance=UNTRUSTED), not a copy. +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))) + +from crawl4ai.async_configs import ( + CrawlerRunConfig, + Provenance, + UntrustedConfigError, + from_serializable_dict, +) + + +def _pdf_strategy(params: dict) -> dict: + return {"type": "PDFContentScrapingStrategy", "params": params} + + +class TestPdfImageWriteGate(unittest.TestCase): + def test_untrusted_image_save_dir_is_rejected(self): + """image_save_dir on an untrusted body must raise (loud 400), not pass.""" + body = _pdf_strategy({"save_images_locally": True, "image_save_dir": "/home/appuser/.ssh"}) + with self.assertRaises(UntrustedConfigError): + from_serializable_dict(body, provenance=Provenance.UNTRUSTED) + + def test_untrusted_save_images_locally_is_rejected(self): + """The boolean that turns the write on is forbidden on untrusted bodies too.""" + body = _pdf_strategy({"save_images_locally": True}) + with self.assertRaises(UntrustedConfigError): + from_serializable_dict(body, provenance=Provenance.UNTRUSTED) + + def test_untrusted_rejected_when_nested_in_crawlerrunconfig(self): + """The real server path: PDF strategy nested under CrawlerRunConfig.scraping_strategy.""" + body = { + "type": "CrawlerRunConfig", + "params": { + "scraping_strategy": _pdf_strategy( + {"save_images_locally": True, "image_save_dir": "/tmp/c4_oob"} + ) + }, + } + with self.assertRaises(UntrustedConfigError): + CrawlerRunConfig.load(body, provenance=Provenance.UNTRUSTED) + + def test_untrusted_extract_images_still_allowed(self): + """Negative control: extract_images returns bytes base64-inline, no disk + write, so it must NOT be blocked (feature preserved over the API).""" + body = _pdf_strategy({"extract_images": True}) + obj = from_serializable_dict(body, provenance=Provenance.UNTRUSTED) + self.assertEqual(type(obj).__name__, "PDFContentScrapingStrategy") + + def test_trusted_path_unchanged(self): + """No regression: the in-process SDK (TRUSTED) may still set image_save_dir.""" + body = _pdf_strategy({"save_images_locally": True, "image_save_dir": "/tmp/mine"}) + obj = from_serializable_dict(body, provenance=Provenance.TRUSTED) + self.assertEqual(type(obj).__name__, "PDFContentScrapingStrategy") + + def test_global_backstop_covers_future_types(self): + """The global forbidden set catches a write-arg regardless of per-type map. + LXMLWebScrapingStrategy has no path arg today, but a smuggled downloads_path + must still be rejected by the fail-closed backstop.""" + body = {"type": "LXMLWebScrapingStrategy", "params": {"downloads_path": "/etc"}} + with self.assertRaises(UntrustedConfigError): + from_serializable_dict(body, provenance=Provenance.UNTRUSTED) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 4772236b9ed0b585d8bb1673ff300021541f02c9 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 27 Jul 2026 16:44:34 +0200 Subject: [PATCH 33/38] fix(pdf): validate egress destinations and bound resource use in PDF fetch Addresses two advisories in the PDFContentScrapingStrategy download path, which fetches with `requests` outside the browser and so bypassed the server's egress and resource controls. GHSA-q5rj-45vw-vp2g (SSRF, high): - Stop following redirects blindly. Redirects are now resolved by hand with a per-hop destination check (max 5 hops), and the peer IP of the response we read back is validated to close DNS rebinding. - The library exposes injectable validators (set_url_validator / set_peer_ip_validator), defaulting to no-op so plain library use is unchanged. The Docker server wires in egress_broker at boot, giving the PDF path the same non-global-IP policy the browser path already has. GHSA-v2rm-hvrj-2x9q (DoS, medium): - Cap the streamed download at max_pdf_bytes (100 MiB default), enforced on the running total rather than the attacker-controlled content-length. - Cap parsed pages at max_pdf_pages (2000 default) in both process paths. - Ship a non-zero wall_clock_s (300s) default in the Docker config. - Clamp these caps and force extract_images off for untrusted request bodies so a client cannot raise its own limits back to unbounded. Adds tests/unit/test_pdf_download_limits.py (22 tests) covering per-hop validation, rebinding, redirect bounds, byte/page caps, and untrusted clamping. Reported by Nguyen Tran Thanh Lam (https://github.com/c240030). (cherry picked from commit 0cd4fae0d717f9b78694682fcb0abd05e2fa996b) --- crawl4ai/async_configs.py | 13 + crawl4ai/processors/pdf/__init__.py | 244 +++++++++++---- crawl4ai/processors/pdf/processor.py | 26 +- deploy/docker/config.yml | 2 +- deploy/docker/server.py | 36 +++ tests/unit/test_pdf_download_limits.py | 409 +++++++++++++++++++++++++ 6 files changed, 667 insertions(+), 63 deletions(-) create mode 100644 tests/unit/test_pdf_download_limits.py diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 87500a90e..383079fef 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -296,6 +296,8 @@ class UntrustedConfigError(ValueError): _MAX_TIMEOUT_MS = 60_000 _MAX_SCROLL_STEPS = 1000 _MAX_VIEWPORT = 4000 +_MAX_PDF_BYTES = 100 * 1024 * 1024 +_MAX_PDF_PAGES = 2000 def _filter_untrusted_fields(type_name: str, params: dict) -> dict: @@ -332,6 +334,17 @@ def _cap_timeout(v): for f in ("viewport_width", "viewport_height"): if isinstance(params.get(f), int): params[f] = max(1, min(params[f], _MAX_VIEWPORT)) + elif type_name == "PDFContentScrapingStrategy": + # This class has no field allowlist, so without clamping a body could + # simply raise its own caps back to unbounded and re-open the DoS. + for f, cap in (("max_pdf_bytes", _MAX_PDF_BYTES), ("max_pdf_pages", _MAX_PDF_PAGES)): + if f in params: + v = params[f] + # <=0 or non-int would read as "no limit"; pin to the cap. + params[f] = cap if not isinstance(v, int) or v <= 0 else min(v, cap) + # Rasterizing every page is the most expensive thing this strategy can + # do, and nothing about untrusted crawling needs it. + params["extract_images"] = False return params diff --git a/crawl4ai/processors/pdf/__init__.py b/crawl4ai/processors/pdf/__init__.py index b41677c6b..9fc5ad781 100644 --- a/crawl4ai/processors/pdf/__init__.py +++ b/crawl4ai/processors/pdf/__init__.py @@ -7,7 +7,44 @@ from crawl4ai.content_scraping_strategy import ContentScrapingStrategy from .processor import NaivePDFProcessorStrategy # Assuming your current PDF code is in pdf_processor.py -MAX_PDF_DOWNLOAD_REDIRECTS = 10 # Max redirect hops to follow when downloading a PDF +# Default resource bounds for remote PDF fetches. These cap what an untrusted +# caller can make the process download and parse; see set_url_validator() for +# the destination side of the same problem. +DEFAULT_MAX_PDF_BYTES = 100 * 1024 * 1024 # 100 MiB +DEFAULT_MAX_PDF_PAGES = 2000 +DEFAULT_MAX_REDIRECTS = 5 + +_REDIRECT_STATUSES = (301, 302, 303, 307, 308) + +# Destination-policy hooks, injected by whoever embeds the library. +# +# The library deliberately has no egress policy of its own: as a plain library +# the caller already chooses the URL, so there is nothing to defend against. +# It matters when the URL comes from an untrusted API client, which is the +# Docker server's situation -- deploy/docker/server.py installs the egress +# broker here at boot so the PDF path enforces the same non-global-IP policy +# the browser path already gets via enforce_egress(). +# +# _url_validator runs on every hop (initial URL and each redirect Location). +# _peer_ip_validator runs on the IP actually connected to; without it a name +# that passed validation could still resolve to an internal address on the +# second lookup requests performs (DNS rebinding). +# Both must raise to block; returning anything is treated as "allowed". +_url_validator = None +_peer_ip_validator = None + + +def set_url_validator(fn): + """Install a per-hop destination validator: fn(url) -> None, raises to block.""" + global _url_validator + _url_validator = fn + + +def set_peer_ip_validator(fn): + """Install a connected-peer check: fn(ip: str) -> None, raises to block.""" + global _peer_ip_validator + _peer_ip_validator = fn + class PDFCrawlerStrategy(AsyncCrawlerStrategy): """Crawler strategy for PDF documents. @@ -18,7 +55,6 @@ class PDFCrawlerStrategy(AsyncCrawlerStrategy): which performs the actual PDF download and content extraction. With any other scraping strategy the result will contain only the placeholder text. """ - def __init__(self, logger: AsyncLogger = None): self.logger = logger @@ -65,20 +101,30 @@ class PDFContentScrapingStrategy(ContentScrapingStrategy): ) """ - def __init__(self, + def __init__(self, save_images_locally : bool = False, extract_images : bool = False, image_save_dir : str = None, batch_size: int = 4, + max_pdf_bytes: int = DEFAULT_MAX_PDF_BYTES, + max_pdf_pages: int = DEFAULT_MAX_PDF_PAGES, + max_redirects: int = DEFAULT_MAX_REDIRECTS, logger: AsyncLogger = None, url_validator=None): self.logger = logger - self.url_validator = url_validator # vets the download URL before fetch + # Per-instance validator, set by deploy/docker/api.py on the strategy it + # builds for a request. It runs in addition to the module-level policy + # installed at server boot, not instead of it. + self.url_validator = url_validator + self.max_pdf_bytes = max_pdf_bytes + self.max_pdf_pages = max_pdf_pages + self.max_redirects = max_redirects self.pdf_processor = NaivePDFProcessorStrategy( save_images_locally=save_images_locally, extract_images=extract_images, image_save_dir=image_save_dir, - batch_size=batch_size + batch_size=batch_size, + max_pages=max_pdf_pages ) self._temp_files = [] # Track temp files for cleanup @@ -152,79 +198,159 @@ async def ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult: return await asyncio.to_thread(self.scrap, url, html, **kwargs) + def _discard_temp_file(self, path: str) -> None: + Path(path).unlink(missing_ok=True) + if path in self._temp_files: + self._temp_files.remove(path) + + @staticmethod + def _peer_ip(response): + """Best-effort peer address of a streaming response, or None. + + The socket lives in a different place across urllib3 majors, and on a + bodyless response (a redirect) it is already released -- hence None + rather than an exception, and hence `strict` in _check_peer_ip. + """ + for get_sock in ( + lambda: response.raw._fp.fp.raw._sock, # urllib3 2.x + lambda: response.raw._original_response.fp.raw._sock, # urllib3 1.x + lambda: response.raw._connection.sock, + ): + try: + sock = get_sock() + if sock is not None: + return sock.getpeername()[0] + except Exception: + continue + return None + + def _check_peer_ip(self, response, strict: bool = True) -> None: + """Validate the address actually connected to. + + Validating the URL only proves the *name* resolved somewhere allowed; + requests resolves it again when it dials, so a hostname with a short + TTL can point somewhere internal by then (DNS rebinding). Checking the + connected peer is what closes that window. + + `strict` is False for redirect hops, where urllib3 has already released + the socket and no peer is observable. Those hops are still covered by + the per-hop URL validation; what strict mode protects is the response + whose body we are about to read back to the caller, which is where a + rebound connection would actually exfiltrate something. + """ + if not _peer_ip_validator: + return + + peer = self._peer_ip(response) + if peer is None: + if not strict: + return + # Fail closed: a policy was installed, so an unverifiable peer on + # the response we are about to read is not something to wave through. + response.close() + raise RuntimeError("Could not determine peer address for PDF download") + + try: + _peer_ip_validator(peer) + except Exception: + response.close() + raise + + def _fetch_with_redirect_checks(self, requests, url: str): + """GET `url`, validating every hop against the injected destination policy. + + Redirects are followed by hand because requests would otherwise chase a + Location into a private address without consulting the validator at all + -- a public URL that 302s to 169.254.169.254 was the reported SSRF. + Returns an open streaming response; the caller closes it. + """ + from urllib.parse import urljoin + + current = url + # One Session for the whole chain: a bare requests.get() per hop starts + # with an empty cookie jar, so a host that sets a cookie and then + # redirects (common for gated or CDN-signed PDFs) would never get it + # back. allow_redirects=True used to carry cookies for us. + session = requests.Session() + for _ in range(self.max_redirects + 1): + if self.url_validator: + self.url_validator(current) + if _url_validator: + _url_validator(current) + + # Connection timeout: 20s, Read timeout: 600s (for large PDFs) + response = session.get( + current, stream=True, timeout=(20, 60 * 10), allow_redirects=False + ) + is_redirect = response.status_code in _REDIRECT_STATUSES + self._check_peer_ip(response, strict=not is_redirect) + + if is_redirect: + location = response.headers.get('location') + response.close() + if not location: + raise RuntimeError(f"Redirect without Location header from {current}") + # Relative Locations are legal, so resolve against the current + # URL before validating -- the validator needs an absolute URL. + current = urljoin(current, location) + continue + + response.raise_for_status() + return response + + raise RuntimeError( + f"Too many redirects (>{self.max_redirects}) downloading PDF from {url}" + ) + def _get_pdf_path(self, url: str) -> str: if url.startswith(("http://", "https://")): import tempfile import requests - + # Create temp file with .pdf extension temp_file = tempfile.NamedTemporaryFile(suffix='.pdf', delete=False) temp_file.close() # Close handle immediately; file persists due to delete=False self._temp_files.append(temp_file.name) - + try: if self.logger: self.logger.info(f"Downloading PDF from {url}...") - - # Download PDF with streaming and timeout - # Connection timeout: 10s, Read timeout: 300s (5 minutes for large PDFs) - # Redirects are followed manually so url_validator (when set) can vet every hop BEFORE it is fetched - from urllib.parse import urljoin - current_url = url - # One Session for the whole chain: a bare requests.get() per hop - # starts with an empty cookie jar, so a host that sets a cookie - # and then redirects (common for gated/CDN-signed PDFs) would - # get its own cookie back. allow_redirects=True used to carry - # them for us; following hops by hand means we carry them here. - session = requests.Session() - for _ in range(MAX_PDF_DOWNLOAD_REDIRECTS): - if self.url_validator: - self.url_validator(current_url) - response = session.get(current_url, stream=True, timeout=(20, 60 * 10), - allow_redirects=False) - if response.is_redirect: - location = response.headers.get("location") - response.close() # hop response holds its connection open (stream=True) - if not location: - raise RuntimeError(f"Redirect without Location header from {current_url}") - current_url = urljoin(current_url, location) - continue - break - else: - raise RuntimeError( - f"Too many redirects (>{MAX_PDF_DOWNLOAD_REDIRECTS}) downloading PDF from {url}") - response.raise_for_status() - - # Get file size if available - total_size = int(response.headers.get('content-length', 0)) + + response = self._fetch_with_redirect_checks(requests, url) + + # content-length is a hint only: it is attacker-controlled and + # may be absent or understated, so the running total is what + # enforces the cap. + total_size = int(response.headers.get('content-length', 0) or 0) downloaded = 0 - - # Write to temp file - with open(temp_file.name, 'wb') as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - downloaded += len(chunk) - if self.logger and total_size > 0: - progress = (downloaded / total_size) * 100 - if progress % 10 < 0.1: # Log every 10% - self.logger.debug(f"PDF download progress: {progress:.0f}%") - + + with response: + with open(temp_file.name, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + downloaded += len(chunk) + if downloaded > self.max_pdf_bytes: + raise RuntimeError( + f"PDF from {url} exceeds max_pdf_bytes " + f"({self.max_pdf_bytes} bytes)" + ) + f.write(chunk) + if self.logger and total_size > 0: + progress = (downloaded / total_size) * 100 + if progress % 10 < 0.1: # Log every 10% + self.logger.debug(f"PDF download progress: {progress:.0f}%") + if self.logger: self.logger.info(f"PDF downloaded successfully: {temp_file.name}") - + return temp_file.name - + except requests.exceptions.Timeout as e: - # Clean up temp file if download fails - Path(temp_file.name).unlink(missing_ok=True) - self._temp_files.remove(temp_file.name) + self._discard_temp_file(temp_file.name) raise RuntimeError(f"Timeout downloading PDF from {url}: {str(e)}") except Exception as e: - # Clean up temp file if download fails - Path(temp_file.name).unlink(missing_ok=True) - self._temp_files.remove(temp_file.name) + self._discard_temp_file(temp_file.name) raise RuntimeError(f"Failed to download PDF from {url}: {str(e)}") - + elif url.startswith("file://"): return url[7:] # Strip file:// prefix diff --git a/crawl4ai/processors/pdf/processor.py b/crawl4ai/processors/pdf/processor.py index 1af25c05c..96ff0d260 100644 --- a/crawl4ai/processors/pdf/processor.py +++ b/crawl4ai/processors/pdf/processor.py @@ -55,8 +55,9 @@ def process(self, pdf_path: Path) -> PDFProcessResult: pass class NaivePDFProcessorStrategy(PDFProcessorStrategy): - def __init__(self, image_dpi: int = 144, image_quality: int = 85, extract_images: bool = True, - save_images_locally: bool = False, image_save_dir: Optional[Path] = None, batch_size: int = 4): + def __init__(self, image_dpi: int = 144, image_quality: int = 85, extract_images: bool = True, + save_images_locally: bool = False, image_save_dir: Optional[Path] = None, batch_size: int = 4, + max_pages: Optional[int] = None): # Import check at initialization time try: import pypdf @@ -70,8 +71,24 @@ def __init__(self, image_dpi: int = 144, image_quality: int = 85, extract_images self.save_images_locally = save_images_locally self.image_save_dir = image_save_dir self.batch_size = batch_size + # None means unbounded, which is the right default for a library caller + # opening a file they chose. Callers accepting untrusted PDFs (the + # Docker server) pass a cap so page count can't be used to burn CPU. + self.max_pages = max_pages self._temp_dir = None + def _page_limit(self, total_pages: int) -> int: + """Number of pages to actually process, honouring max_pages.""" + if self.max_pages is None: + return total_pages + capped = min(total_pages, self.max_pages) + if capped < total_pages: + logger.warning( + f"PDF has {total_pages} pages; processing first {capped} " + f"(max_pages={self.max_pages})" + ) + return capped + def process(self, pdf_path: Path) -> PDFProcessResult: # Import inside method to allow dependency to be optional try: @@ -101,7 +118,10 @@ def process(self, pdf_path: Path) -> PDFProcessResult: self._temp_dir = tempfile.mkdtemp(prefix='pdf_images_') image_dir = Path(self._temp_dir) + page_limit = self._page_limit(len(reader.pages)) for page_num, page in enumerate(reader.pages): + if page_num >= page_limit: + break self.current_page_number = page_num + 1 pdf_page = self._process_page(page, image_dir) result.pages.append(pdf_page) @@ -149,7 +169,7 @@ def process_batch(self, pdf_path: Path) -> PDFProcessResult: with pdf_path.open('rb') as file: reader = PdfReader(file) result.metadata = self._extract_metadata(pdf_path, reader) - total_pages = len(reader.pages) + total_pages = self._page_limit(len(reader.pages)) # Handle image directory setup image_dir = None diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml index 614ac24d1..7aabc6814 100644 --- a/deploy/docker/config.yml +++ b/deploy/docker/config.yml @@ -39,7 +39,7 @@ limits: max_body_bytes: 10485760 # 10 MiB request body cap (413 if exceeded); 0 = unbounded max_pages: 100 # deep-crawl page budget clamp (defense in depth) max_depth: 5 # deep-crawl depth clamp - wall_clock_s: 0 # per-crawl deadline in seconds (504 on timeout); 0 = no deadline + wall_clock_s: 300 # per-crawl deadline in seconds (504 on timeout); 0 = no deadline queue: # background job queue for /crawl/job and /llm/job maxsize: 1000 # max queued jobs (503 when full); 0 = unbounded workers: 4 # concurrent background workers diff --git a/deploy/docker/server.py b/deploy/docker/server.py index 70352913a..3636efb93 100644 --- a/deploy/docker/server.py +++ b/deploy/docker/server.py @@ -159,6 +159,37 @@ async def capped_arun(self, *a, **kw): return await orig_arun(self, *a, **kw) AsyncWebCrawler.arun = capped_arun + +def _install_pdf_egress_policy() -> None: + """Hand the PDF scraping strategy this server's egress policy. + + PDFContentScrapingStrategy is in UNTRUSTED_ALLOWED_TYPES, so any API client + can select it and name the URL. It downloads with requests rather than the + browser, so none of the Chromium-side pinning applies to it. Injecting the + broker here makes the PDF path enforce the same non-global-IP rule on the + initial URL, on every redirect hop, and on the peer actually connected to. + """ + from crawl4ai.processors.pdf import set_peer_ip_validator, set_url_validator + from egress_broker import ( + ALLOW_INTERNAL, + EgressBlocked, + is_forbidden_ip, + resolve_and_pin, + ) + + def _validate_url(url: str) -> None: + resolve_and_pin(url) # raises EgressBlocked on a non-global destination + + def _validate_peer(ip: str) -> None: + if ALLOW_INTERNAL: + return + if is_forbidden_ip(ip): + raise EgressBlocked() + + set_url_validator(_validate_url) + set_peer_ip_validator(_validate_peer) + + # ───────────────────── FastAPI lifespan ────────────────────── @@ -182,6 +213,11 @@ async def lifespan(_: FastAPI): app.state.egress_proxy = PinningProxy() set_egress_proxy(await app.state.egress_proxy.start()) + # The pinning proxy only covers Chromium. PDFContentScrapingStrategy fetches + # with requests on its own, so hand the library the same destination policy + # or that path stays an unguarded SSRF hole. + _install_pdf_egress_policy() + # Bounded background-job queue (per-principal quotas optional). from work_queue import WorkQueue, set_job_queue from governor import job_queue_caps diff --git a/tests/unit/test_pdf_download_limits.py b/tests/unit/test_pdf_download_limits.py new file mode 100644 index 000000000..993107354 --- /dev/null +++ b/tests/unit/test_pdf_download_limits.py @@ -0,0 +1,409 @@ +"""Regression tests for the PDF fetch path: destination validation and resource caps. + +Covers two reported issues: + - a public URL that redirects to an internal address must fail closed (SSRF) + - an oversized remote PDF must abort instead of filling the disk (DoS) + +The fetch helper takes the `requests` module as an argument, so these drive it +with a stub and need no network or live server. +""" + +import os + +import pytest + +from crawl4ai.processors.pdf import ( + PDFContentScrapingStrategy, + set_peer_ip_validator, + set_url_validator, +) +import crawl4ai.processors.pdf as pdf_module + + +class Blocked(Exception): + """Stand-in for egress_broker.EgressBlocked.""" + + +class FakeResponse: + def __init__(self, status_code=200, headers=None, chunks=(), peer_ip="93.184.216.34"): + self.status_code = status_code + self.headers = headers or {} + self._chunks = list(chunks) + self.closed = False + self.raw = self._make_raw(peer_ip) + + def _make_raw(self, peer_ip): + """Mirror urllib3 2.x layout: raw._fp.fp.raw._sock. + + Worth keeping faithful -- an earlier version of this stub exposed the + socket somewhere urllib3 does not, and hid a real bug. + """ + if peer_ip is None: + return object() # bodyless response: socket already released + + class _Sock: + def getpeername(self_inner): + return (peer_ip, 443) + + class _SocketIO: + _sock = _Sock() + + class _Buffered: + raw = _SocketIO() + + class _HTTPResponse: + fp = _Buffered() + + class _Raw: + _fp = _HTTPResponse() + + return _Raw() + + def iter_content(self, chunk_size=8192): + for chunk in self._chunks: + yield chunk + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + def close(self): + self.closed = True + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + return False + + +class FakeRequests: + """Stub of the `requests` module surface the downloader uses.""" + + class exceptions: + class Timeout(Exception): + pass + + def __init__(self, responses): + self._responses = list(responses) + self.calls = [] + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + return self._responses.pop(0) + + def Session(self): + # The downloader opens one Session for the whole redirect chain, so a + # host that sets a cookie and then redirects gets it back on the next + # hop. Returning self keeps every call recorded in one place. + return self + + +@pytest.fixture(autouse=True) +def _reset_validators(): + """Validators are module-level globals; leaking one breaks later tests.""" + yield + set_url_validator(None) + set_peer_ip_validator(None) + + +def _strategy(**kwargs): + return PDFContentScrapingStrategy(**kwargs) + + +# ── destination validation (SSRF) ────────────────────────────────────── + + +def test_redirect_target_is_validated_and_blocked(): + seen = [] + + def validator(url): + seen.append(url) + if "169.254.169.254" in url or "127.0.0.1" in url: + raise Blocked(url) + + set_url_validator(validator) + strategy = _strategy() + requests = FakeRequests([ + FakeResponse(302, {"location": "http://169.254.169.254/latest/meta-data/"}), + ]) + + with pytest.raises(Blocked): + strategy._fetch_with_redirect_checks(requests, "https://public.example/doc.pdf") + + assert seen == [ + "https://public.example/doc.pdf", + "http://169.254.169.254/latest/meta-data/", + ], "validator must run on the redirect target, not just the seed" + + +def test_redirects_are_never_followed_by_requests_itself(): + set_url_validator(lambda url: None) + strategy = _strategy() + requests = FakeRequests([ + FakeResponse(302, {"location": "https://public.example/real.pdf"}), + FakeResponse(200, {"content-length": "4"}, [b"%PDF"]), + ]) + + strategy._fetch_with_redirect_checks(requests, "https://public.example/doc.pdf") + + assert [kw["allow_redirects"] for _, kw in requests.calls] == [False, False] + + +def test_relative_location_is_resolved_before_validation(): + seen = [] + set_url_validator(seen.append) + strategy = _strategy() + requests = FakeRequests([ + FakeResponse(302, {"location": "/elsewhere.pdf"}), + FakeResponse(200, {}, [b"%PDF"]), + ]) + + strategy._fetch_with_redirect_checks(requests, "https://public.example/a/doc.pdf") + + assert seen[1] == "https://public.example/elsewhere.pdf" + + +def test_redirect_chain_is_bounded(): + set_url_validator(lambda url: None) + strategy = _strategy(max_redirects=2) + requests = FakeRequests([ + FakeResponse(302, {"location": f"https://public.example/{i}.pdf"}) + for i in range(5) + ]) + + with pytest.raises(RuntimeError, match="Too many redirects"): + strategy._fetch_with_redirect_checks(requests, "https://public.example/doc.pdf") + + +def test_redirect_without_location_is_rejected(): + set_url_validator(lambda url: None) + strategy = _strategy() + requests = FakeRequests([FakeResponse(302, {})]) + + with pytest.raises(RuntimeError, match="Redirect without Location"): + strategy._fetch_with_redirect_checks(requests, "https://public.example/doc.pdf") + + +def test_peer_ip_is_checked_closing_dns_rebinding(): + """The name may pass validation and still resolve inward when requests dials.""" + set_url_validator(lambda url: None) + + def peer_validator(ip): + if ip.startswith("127.") or ip.startswith("169.254."): + raise Blocked(ip) + + set_peer_ip_validator(peer_validator) + strategy = _strategy() + response = FakeResponse(200, {}, [b"%PDF"], peer_ip="169.254.169.254") + requests = FakeRequests([response]) + + with pytest.raises(Blocked): + strategy._fetch_with_redirect_checks(requests, "https://public.example/doc.pdf") + assert response.closed, "blocked response must be closed, not leaked" + + +def test_unverifiable_peer_fails_closed_on_the_body_response(): + set_url_validator(lambda url: None) + set_peer_ip_validator(lambda ip: None) + strategy = _strategy() + response = FakeResponse(200, {}, [b"%PDF"], peer_ip=None) # no reachable socket + requests = FakeRequests([response]) + + with pytest.raises(RuntimeError, match="Could not determine peer address"): + strategy._fetch_with_redirect_checks(requests, "https://public.example/doc.pdf") + + +def test_redirect_hop_without_observable_socket_is_not_fatal(): + """urllib3 releases the socket on a bodyless 302; that must not break redirects.""" + set_url_validator(lambda url: None) + set_peer_ip_validator(lambda ip: None) + strategy = _strategy() + requests = FakeRequests([ + FakeResponse(302, {"location": "https://public.example/real.pdf"}, peer_ip=None), + FakeResponse(200, {}, [b"%PDF"]), + ]) + + response = strategy._fetch_with_redirect_checks(requests, "https://public.example/doc.pdf") + + assert response.status_code == 200 + + +def test_no_validator_installed_leaves_plain_library_use_working(): + strategy = _strategy() + requests = FakeRequests([FakeResponse(200, {}, [b"%PDF"])]) + + response = strategy._fetch_with_redirect_checks(requests, "https://public.example/doc.pdf") + + assert response.status_code == 200 + + +# ── resource caps (DoS) ──────────────────────────────────────────────── + + +def test_download_aborts_past_max_bytes_and_cleans_up(monkeypatch): + strategy = _strategy(max_pdf_bytes=1024) + # Understated content-length must not be trusted: the running total decides. + requests = FakeRequests([ + FakeResponse(200, {"content-length": "10"}, [b"A" * 512] * 8), + ]) + monkeypatch.setattr(pdf_module, "requests", requests, raising=False) + + import sys + monkeypatch.setitem(sys.modules, "requests", requests) + + with pytest.raises(RuntimeError, match="exceeds max_pdf_bytes"): + strategy._get_pdf_path("https://public.example/huge.pdf") + + assert strategy._temp_files == [], "temp file must be dropped from tracking" + + +def test_download_under_cap_succeeds(monkeypatch): + import sys + + strategy = _strategy(max_pdf_bytes=1024) + requests = FakeRequests([FakeResponse(200, {"content-length": "4"}, [b"%PDF"])]) + monkeypatch.setitem(sys.modules, "requests", requests) + + path = strategy._get_pdf_path("https://public.example/small.pdf") + + try: + assert os.path.getsize(path) == 4 + finally: + strategy._discard_temp_file(path) + + +def test_page_limit_caps_page_count(): + from crawl4ai.processors.pdf.processor import NaivePDFProcessorStrategy + + processor = NaivePDFProcessorStrategy(max_pages=10) + assert processor._page_limit(5000) == 10 + assert processor._page_limit(3) == 3 + + +def test_page_limit_unbounded_by_default(): + from crawl4ai.processors.pdf.processor import NaivePDFProcessorStrategy + + processor = NaivePDFProcessorStrategy() + assert processor._page_limit(5000) == 5000 + + +# ── untrusted-body clamping ──────────────────────────────────────────── + + +@pytest.mark.parametrize( + "sent, expected_bytes, expected_pages", + [ + ({"max_pdf_bytes": 10**12, "max_pdf_pages": 10**6}, 100 * 1024 * 1024, 2000), + ({"max_pdf_bytes": 0, "max_pdf_pages": 0}, 100 * 1024 * 1024, 2000), + ({"max_pdf_bytes": -1, "max_pdf_pages": -1}, 100 * 1024 * 1024, 2000), + ({"max_pdf_bytes": "huge", "max_pdf_pages": None}, 100 * 1024 * 1024, 2000), + ({"max_pdf_bytes": 1024, "max_pdf_pages": 5}, 1024, 5), + ], +) +def test_untrusted_body_cannot_raise_its_own_caps(sent, expected_bytes, expected_pages): + """An untrusted client must not be able to restore unbounded behavior.""" + from crawl4ai.async_configs import _clamp_untrusted + + out = _clamp_untrusted("PDFContentScrapingStrategy", dict(sent)) + + assert out["max_pdf_bytes"] == expected_bytes + assert out["max_pdf_pages"] == expected_pages + + +# ── against real requests/urllib3 ─────────────────────────────────────── + + +@pytest.fixture +def live_server(): + """Loopback HTTP server serving a redirect and a small PDF body.""" + import http.server + import socketserver + import threading + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/redirect": + self.send_response(302) + self.send_header("Location", "/doc.pdf") + self.end_headers() + return + body = b"%PDF-1.4 " + b"x" * 4096 + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = socketserver.TCPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + +def test_real_redirect_chain_validates_every_hop_and_reads_peer(live_server): + """Exercises the actual urllib3 object graph, not a stub of it.""" + import requests as real_requests + + seen_urls, seen_ips = [], [] + set_url_validator(seen_urls.append) + set_peer_ip_validator(seen_ips.append) + strategy = _strategy() + + response = strategy._fetch_with_redirect_checks(real_requests, f"{live_server}/redirect") + response.close() + + assert seen_urls == [f"{live_server}/redirect", f"{live_server}/doc.pdf"] + assert seen_ips == ["127.0.0.1"], "peer must be read from the body response" + + +def test_real_internal_redirect_is_blocked(live_server): + """The reported attack shape: public seed -> 302 -> internal address.""" + import requests as real_requests + + def validator(url): + if "169.254.169.254" in url: + raise Blocked(url) + + set_url_validator(validator) + strategy = _strategy() + + # Point the seed at a redirect whose target is the metadata service. + requests = FakeRequests([ + FakeResponse(302, {"location": "http://169.254.169.254/latest/meta-data/"}), + ]) + with pytest.raises(Blocked): + strategy._fetch_with_redirect_checks(requests, f"{live_server}/redirect") + + # And the real client never followed anything on its own. + response = real_requests.get(f"{live_server}/redirect", allow_redirects=False) + assert response.status_code == 302 + response.close() + + +def test_real_oversize_body_aborts(live_server, monkeypatch): + import sys + + import requests as real_requests + + monkeypatch.setitem(sys.modules, "requests", real_requests) + strategy = _strategy(max_pdf_bytes=1024) # server sends ~4 KiB + + with pytest.raises(RuntimeError, match="exceeds max_pdf_bytes"): + strategy._get_pdf_path(f"{live_server}/doc.pdf") + + assert strategy._temp_files == [] + + +def test_untrusted_body_cannot_enable_image_extraction(): + from crawl4ai.async_configs import _clamp_untrusted + + out = _clamp_untrusted("PDFContentScrapingStrategy", {"extract_images": True}) + + assert out["extract_images"] is False From ceaf6a3edec0124550951b3fe223a94d16b517c5 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 27 Jul 2026 16:55:06 +0200 Subject: [PATCH 34/38] fix(pdf): escape paragraph text in cleaned_html to prevent XSS GHSA-7g3g-vhm6-79f3 (medium). clean_pdf_text_to_html() escaped every sink except the paragraph body, where html.escape() had been commented out. PDF paragraph text is attacker-controlled and flows verbatim into cleaned_html and the crawl JSON, so injected markup such as survived and executed when the result was rendered. Re-enable html.escape() on paragraph text. This is the "reliable injection vector" half of the advisory; the primary DOM sink (the Playground innerHTML round-trip) is fixed separately under GHSA-m446-hp3q-qfxp. Adds tests/unit/test_pdf_html_escaping.py asserting < > " ' appear only as entities in PDF cleaned_html. Reported by Nguyen Tran Thanh Lam (https://github.com/c240030). (cherry picked from commit 9b312db8533bf4cc3e0a48ea7d48713e5ca30d36) --- crawl4ai/processors/pdf/utils.py | 12 +++-- tests/unit/test_pdf_html_escaping.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_pdf_html_escaping.py diff --git a/crawl4ai/processors/pdf/utils.py b/crawl4ai/processors/pdf/utils.py index 3dc0e739d..e7cc6328b 100644 --- a/crawl4ai/processors/pdf/utils.py +++ b/crawl4ai/processors/pdf/utils.py @@ -97,10 +97,14 @@ def flush_paragraph(): para = ' '.join(current_paragraph) para = re.sub(r'\s+', ' ', para).strip() if para: - # escaped_para = html.escape(para) - escaped_para = para - # escaped_para = re.sub(r'\.\n', '.\n\n', escaped_para) - # Split escaped_para by <|break|> to avoid HTML escaping + # Paragraph text comes verbatim from the PDF and is attacker- + # controlled. Escape it so markup like cannot + # survive into cleaned_html and execute when the result is + # rendered (DOM XSS). Every other sink in this function already + # escapes; this one was the gap. + escaped_para = html.escape(para) + # Whitespace was collapsed above, so this split yields a single + # element -- kept only to preserve the original structure. escaped_para = escaped_para.split('.\n\n') # Wrap each part in

tag escaped_para = [f'

{part}

' for part in escaped_para] diff --git a/tests/unit/test_pdf_html_escaping.py b/tests/unit/test_pdf_html_escaping.py new file mode 100644 index 000000000..3fba16a73 --- /dev/null +++ b/tests/unit/test_pdf_html_escaping.py @@ -0,0 +1,76 @@ +"""Regression test for GHSA-7g3g-vhm6-79f3. + +PDF paragraph text is attacker-controlled and flows verbatim into cleaned_html +(clean_pdf_text_to_html -> pdf_page.html -> cleaned_html -> crawl JSON). It must +be HTML-escaped so injected markup cannot execute when the result is rendered. + +The escaping was disabled at the paragraph sink; every other sink in the +function already escaped. These tests pin the paragraph sink specifically and +assert the advisory's acceptance criterion: < > " ' survive only as entities. +""" + +import pytest + +from crawl4ai.processors.pdf.utils import clean_pdf_text_to_html + + +# A leading short line becomes an

title (its own escaped path); the blank +# line then starts a fresh paragraph, which is the sink under test. +TITLE = "Quarterly Financial Report\n\n" + + +def _paragraph_html(body: str) -> str: + return clean_pdf_text_to_html(2, TITLE + body) + + +def test_event_handler_markup_does_not_survive_as_live_html(): + payload = "" + body = f"Attackers embed {payload} inside pdf paragraph body text that an operator later views in the playground here." + + html = _paragraph_html(body) + + assert "" + body = f"A crafted string {payload} placed in the paragraph flow of the document body for testing purposes here now." + + html = _paragraph_html(body) + + assert "", ">"), ('"', """), ("'", "'")], +) +def test_dangerous_chars_appear_only_as_entities(char, entity): + body = ( + f"Paragraph body text containing a raw {char} character that must be " + f"escaped before it reaches cleaned html output for safety reasons here." + ) + + html = _paragraph_html(body) + paragraph = html.split('
', 1)[1] + + assert entity in paragraph, f"{char!r} should be emitted as {entity}" + # The only literal angle brackets/quotes allowed in the paragraph region are + # the ones this code emits itself (

,

,
,
). The injected + # char must not appear raw inside the text. + text = paragraph.replace("

", "").replace("

", "") + text = text.replace('
', "").replace("
", "") + assert char not in text, f"raw {char!r} survived into paragraph text" + + +def test_benign_paragraph_text_is_unchanged(): + body = "An ordinary paragraph of report text with no special characters at all in it whatsoever today." + + html = _paragraph_html(body) + + assert "ordinary paragraph of report text" in html + assert "<" not in html and "&" not in html From 0213355db08a3d7ea8ae572bf89af9e351bfb085 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Fri, 24 Jul 2026 10:16:56 +0200 Subject: [PATCH 35/38] fix(playground): remove innerHTML round-trip to prevent DOM-based XSS forceHighlightElement() reset the response code block with `element.innerHTML = element.textContent`, which re-parsed attacker-controlled crawled content (e.g. a reflected page title in metadata.title) as live HTML and executed it in the operator's authenticated Playground session. Removing the innerHTML round-trip closes the sink; hljs.highlightElement() re-highlights safely from textContent and emits escaped markup. Fixes GHSA-m446-hp3q-qfxp (cherry picked from commit 82950b4a1af26190edfed11b2d779f51750780b8) --- deploy/docker/static/playground/index.html | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html index ba368b768..49d27d436 100644 --- a/deploy/docker/static/playground/index.html +++ b/deploy/docker/static/playground/index.html @@ -1000,9 +1000,11 @@

🔥 Stress Test

// Save current scroll position (important for large code blocks) const scrollTop = element.parentElement.scrollTop; - // Reset the element - const text = element.textContent; - element.innerHTML = text; + // Reset highlight.js state. NOTE: do NOT round-trip through + // innerHTML here — the response text contains attacker-controlled + // crawled content and `element.innerHTML = element.textContent` + // re-parses it as live HTML (DOM-based XSS). highlight.js renders + // safely from textContent and emits escaped markup. element.removeAttribute('data-highlighted'); // Reapply highlighting From 8cef8bd243728a49f722557d71f4086984c890c5 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 31 Aug 2026 12:13:05 +0200 Subject: [PATCH 36/38] chore: bump version to 0.9.3 --- Dockerfile | 2 +- crawl4ai/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b39e307d..7ef9c2fb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.12-slim-bookworm AS build # C4ai version -ARG C4AI_VER=0.9.2 +ARG C4AI_VER=0.9.3 ENV C4AI_VERSION=$C4AI_VER LABEL c4ai.version=$C4AI_VER diff --git a/crawl4ai/__version__.py b/crawl4ai/__version__.py index 505ca152c..f8f1eb5d0 100644 --- a/crawl4ai/__version__.py +++ b/crawl4ai/__version__.py @@ -1,7 +1,7 @@ # crawl4ai/__version__.py # This is the version that will be used for stable releases -__version__ = "0.9.2" +__version__ = "0.9.3" # For nightly builds, this gets set during build process __nightly_version__ = None From 4bcd5fa8a56000ce103dd499e8ecdff2439f3e9c Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 31 Aug 2026 12:51:20 +0200 Subject: [PATCH 37/38] docs: release notes, changelog, README, and security credits for v0.9.3 --- CHANGELOG.md | 26 +++++++++ README.md | 21 ++++++- SECURITY-CREDITS.md | 3 + docs/blog/release-v0.9.3.md | 112 ++++++++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 docs/blog/release-v0.9.3.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c09b7d2e1..cc663edfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,32 @@ All notable changes to Crawl4AI will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.9.3] - 2026-08-31 + +0.9.3 is a security release. It closes five coordinated-disclosure advisories in the PDF processing path and the Docker Playground UI. There are no new features and no breaking changes. Users who accept untrusted URLs on the Docker server, or who open PDFs from sources they do not control, should upgrade. + +### Security + +The PDF path was the common thread. `PDFContentScrapingStrategy` is selectable from an untrusted Docker API request body, and it fetches with `requests` outside the browser, so none of the Chromium-side egress or resource controls applied to it. + +- **Arbitrary file write via PDF image-write fields (CWE-22, high)**: `PDFContentScrapingStrategy` had no field allowlist, so an untrusted request body could set `save_images_locally` and `image_save_dir` and make the server write extracted images to a path of the caller's choosing. Those fields are now filtered at the trust boundary and `extract_images` is forced off for untrusted bodies. Credit: Zhixi "Jace" Sun. (GHSA-xpp7-j28w-2gvx) +- **SSRF via PDF download redirects (CWE-918, high)**: the PDF download followed redirects without consulting any destination policy, so a public URL that redirected to an internal address reached it. Redirects are now resolved by hand with a per-hop destination check, bounded at five hops, and the peer IP of the response actually read back is validated to close DNS rebinding. The Docker server installs its egress policy into this path at boot. Credit: Nguyen Tran Thanh Lam ([c240030](https://github.com/c240030)). (GHSA-q5rj-45vw-vp2g) +- **Denial of service via unbounded PDF size and page count (CWE-400, medium)**: a remote PDF was streamed to disk and parsed with no cap on bytes or pages. Downloads now stop at `max_pdf_bytes` (100 MiB default), enforced on the running total rather than the caller-supplied `content-length`, and parsing stops at `max_pdf_pages` (2000 default). Untrusted bodies cannot raise their own caps. The Docker config now ships a non-zero `limits.wall_clock_s` of 300 seconds. Credit: Nguyen Tran Thanh Lam ([c240030](https://github.com/c240030)). (GHSA-v2rm-hvrj-2x9q) +- **XSS via unescaped PDF text in `cleaned_html` (CWE-79, medium)**: paragraph text taken verbatim from a PDF was written into `cleaned_html` without escaping, so markup embedded in a PDF survived into the result and executed when rendered. Paragraph text is now escaped like every other sink in that function. Credit: Nguyen Tran Thanh Lam ([c240030](https://github.com/c240030)). (GHSA-7g3g-vhm6-79f3) +- **DOM-based XSS in the Docker Playground leading to API token theft (CWE-79, high)**: the result viewer reset syntax highlighting with `element.innerHTML = element.textContent`, which re-parsed attacker-controlled crawled content as live HTML in the operator's session. The round trip is removed; highlight.js renders safely from `textContent`. Credit: [e1codes](https://github.com/e1codes). (GHSA-m446-hp3q-qfxp) + +All reporters are credited in `SECURITY-CREDITS.md`. GitHub Security Advisories accompany this release. + +### Tests + +- `tests/unit/test_pdf_download_limits.py`: 22 tests covering per-hop destination validation, DNS rebinding, redirect bounds, byte and page caps, and untrusted-body clamping. +- `tests/unit/test_pdf_html_escaping.py`: escaping of PDF paragraph text in `cleaned_html`. +- `deploy/docker/tests/test_security_pdf_image_write.py`: rejection of image-write fields from untrusted bodies. + +### Breaking Changes + +None. + ## [0.9.0] - 2026-06-18 0.9.0 is a major, secure-by-default release of the Crawl4AI Docker API server. The out-of-the-box deployment is now hardened with defense in depth: authentication is on by default, the server binds loopback unless you give it a token, and the network request body is treated as an untrusted trust boundary. This release contains breaking changes for the self-hosted HTTP server only. The core pip library (SDK / in-process use) is unchanged. diff --git a/README.md b/README.md index 838553762..1cd0ba157 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,11 @@ Limited slots._ Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community. -[✨ Check out latest update v0.9.2](#-recent-updates) +[✨ Check out latest update v0.9.3](#-recent-updates) -✨ **New in v0.9.2**: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md) +✨ **New in v0.9.3**: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md) + +✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md) ✨ Recent v0.9.0: Major secure-by-default release of the Docker API server. Auth is on by default, the server binds loopback unless given a token, and the request body is now an untrusted trust boundary. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.0.md) @@ -565,6 +567,21 @@ async def test_news_crawl(): ## ✨ Recent Updates
+Version 0.9.3 Release Highlights - Security Release + +A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token. + +No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0. + +```bash +pip install -U crawl4ai +``` + +[Full v0.9.3 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md) + +
+ +
Version 0.9.2 Release Highlights - Maintenance Bug Fixes A maintenance patch release with bug fixes across the dispatcher, Docker, and GPU builds. `MemoryAdaptiveDispatcher` no longer leaks crawl tasks and browser pages when a streaming crawl is closed. Docker fixes cover the Playground "Advanced Config" 400, the Monitor WebSocket 500 under JWT auth, and Playwright headless-shell packaging. `ENABLE_GPU=true` Docker builds no longer fail on the CUDA toolkit. diff --git a/SECURITY-CREDITS.md b/SECURITY-CREDITS.md index e2ae2bc66..a1c12441c 100644 --- a/SECURITY-CREDITS.md +++ b/SECURITY-CREDITS.md @@ -18,3 +18,6 @@ We thank the following security researchers for their responsible disclosure: | UDU_RisePho | GitHub: [hoanggxyuuki](https://github.com/hoanggxyuuki) | Chromium launch-flag RCE class via extra_args (0.9.0) | 2026-06-18 | | Y4tacker | GitHub: [Y4tacker](https://github.com/Y4tacker) | Hook system exec() sandbox escape (MRO chain RCE), Chromium launch-arg injection (--utility-cmd-prefix RCE), HTTP crawler path traversal arbitrary file write | 2026-07-09 | | Rafael | GitHub: [rafaelfiguereod-stack](https://github.com/rafaelfiguereod-stack) | Reported SSRF, LLM key exfiltration, and auth gaps (already fixed / not exploitable in current code) | 2026-07-09 | +| Zhixi "Jace" Sun | Independent security researcher | Arbitrary file write via unconfined PDFContentScrapingStrategy image-write fields in untrusted config bodies (0.9.3) | 2026-08-24 | +| Nguyen Tran Thanh Lam | GitHub: [c240030](https://github.com/c240030) | SSRF via PDF download redirects, DoS via unbounded PDF size and page count, XSS via unescaped PDF text in cleaned_html (0.9.3) | 2026-07-27 | +| e1codes | GitHub: [e1codes](https://github.com/e1codes) | DOM-based XSS in the Docker Playground leading to operator API-token theft (0.9.3) | 2026-07-24 | diff --git a/docs/blog/release-v0.9.3.md b/docs/blog/release-v0.9.3.md new file mode 100644 index 000000000..4d2d7260e --- /dev/null +++ b/docs/blog/release-v0.9.3.md @@ -0,0 +1,112 @@ +# Crawl4AI v0.9.3: Security Release + +*August 2026 - 4 min read* + +--- + +I'm releasing Crawl4AI v0.9.3, a security release that closes five coordinated-disclosure advisories. No new features, no breaking changes. + +Four of the five are in the PDF path, and they share one root cause. `PDFContentScrapingStrategy` can be selected from an untrusted Docker API request body, and it downloads with `requests` rather than through the browser. Every control the server applies on the Chromium side (destination pinning, resource caps) simply did not reach it. The fifth is a DOM-based XSS in the Playground UI that could hand an operator's API token to an attacker. + +If you self-host the Docker server and accept crawl requests from clients you do not fully trust, upgrade. If you use the pip library and open PDFs from sources you do not control, upgrade. + +## What's fixed at a glance + +- **Arbitrary file write**: an untrusted request body could choose where extracted PDF images were written +- **SSRF**: the PDF download followed redirects into internal addresses +- **Denial of service**: remote PDFs were downloaded and parsed with no size or page cap +- **XSS**: PDF text was written into `cleaned_html` without escaping +- **DOM XSS**: the Playground result viewer re-parsed crawled content as live HTML + +## Security fixes + +### Arbitrary file write via PDF image-write fields + +**GHSA-xpp7-j28w-2gvx, CWE-22, high.** Credit: Zhixi "Jace" Sun, independent security researcher. + +`PDFContentScrapingStrategy` was in the untrusted-allowed type list but had no field allowlist of its own. A request body could therefore set `save_images_locally: true` and `image_save_dir` to any path, and the server would write extracted PDF images there. + +Both fields are now filtered out of untrusted bodies at the trust boundary, and `extract_images` is forced off for them. Rasterizing every page of a caller-supplied PDF is the most expensive thing this strategy can do, and nothing about untrusted crawling needs it. + +### SSRF via PDF download redirects + +**GHSA-q5rj-45vw-vp2g, CWE-918, high.** Credit: Nguyen Tran Thanh Lam ([c240030](https://github.com/c240030)). + +The PDF download let `requests` follow redirects on its own. A public, allowed URL that returned a 302 to `169.254.169.254` or any other internal address was fetched without the destination ever being checked. + +Redirects are now resolved by hand, one hop at a time, with the destination policy consulted before each hop is fetched and a cap of five hops. The peer IP of the response whose body is actually read back is validated too, which closes DNS rebinding: passing the URL check only proves the *name* resolved somewhere allowed, and `requests` resolves it again when it dials. + +The library ships with no egress policy of its own, because a plain library caller already chose the URL. The Docker server installs its existing `egress_broker` policy into the PDF path at boot, so that path now gets the same non-global-IP rule the browser path already had. + +### Denial of service via unbounded PDF size and page count + +**GHSA-v2rm-hvrj-2x9q, CWE-400, medium.** Credit: Nguyen Tran Thanh Lam ([c240030](https://github.com/c240030)). + +A remote PDF was streamed to a temp file with no cap and then parsed with no page limit. One request pointing at a large or many-page PDF could exhaust disk, CPU, and bandwidth on a shared worker. + +Downloads now stop at `max_pdf_bytes`, 100 MiB by default. The cap is enforced on the running byte total, not on the `content-length` header, because that header is attacker-controlled and may be absent or understated. Parsing stops at `max_pdf_pages`, 2000 by default, in both the single and batch paths. Untrusted request bodies have both caps clamped, so a client cannot raise its own limits back to unbounded. + +The Docker config also ships a non-zero `limits.wall_clock_s` of 300 seconds. It was `0`, meaning no per-crawl deadline at all. + +### XSS via unescaped PDF text in cleaned_html + +**GHSA-7g3g-vhm6-79f3, CWE-79, medium.** Credit: Nguyen Tran Thanh Lam ([c240030](https://github.com/c240030)). + +`clean_pdf_text_to_html()` escapes every sink it writes to except one: paragraph text. The `html.escape()` call on that path had been commented out. Markup embedded in a PDF therefore survived verbatim into `cleaned_html` and executed wherever the result was rendered. + +The escape is restored. + +### DOM-based XSS in the Playground leading to API token theft + +**GHSA-m446-hp3q-qfxp, CWE-79, high.** Credit: [e1codes](https://github.com/e1codes). + +The Playground result viewer reset syntax highlighting with: + +```js +const text = element.textContent; +element.innerHTML = text; +``` + +That round trip re-parses the text as live HTML. The text is crawl output, so it is whatever the crawled page contained. Script in a crawled page could therefore run in the operator's Playground session and read the API token held there. + +The round trip is removed. highlight.js renders from `textContent` and emits escaped markup on its own, so it was never needed. + +## Tests + +- `tests/unit/test_pdf_download_limits.py`, 22 tests: per-hop destination validation, DNS rebinding, redirect bounds, byte and page caps, untrusted-body clamping, and temp-file cleanup on abort. +- `tests/unit/test_pdf_html_escaping.py`, 7 tests: escaping of PDF paragraph text in `cleaned_html`. +- `deploy/docker/tests/test_security_pdf_image_write.py`, 6 tests: rejection of image-write fields from untrusted bodies. + +## Breaking changes + +None. + +Two defaults changed, which is worth knowing if you process large PDFs in a self-hosted deployment: + +- `PDFContentScrapingStrategy` now caps downloads at 100 MiB and parsing at 2000 pages. Raise them with `max_pdf_bytes` and `max_pdf_pages` when you call it from the SDK. Requests arriving over the Docker API cannot raise them past those caps. +- `limits.wall_clock_s` in `deploy/docker/config.yml` is now `300` instead of `0`. Set it back to `0` if you deliberately want no per-crawl deadline. + +## Upgrade + +```bash +pip install -U crawl4ai +crawl4ai-doctor # verify installation +``` + +Docker users: pull the latest image once the Docker release workflow finishes. + +```bash +docker pull unclecode/crawl4ai:0.9.3 +``` + +## Acknowledgments + +Thank you to Zhixi "Jace" Sun, Nguyen Tran Thanh Lam ([c240030](https://github.com/c240030)), and [e1codes](https://github.com/e1codes) for reporting these issues privately and giving us time to fix them before disclosure. All reporters are listed in [SECURITY-CREDITS.md](https://github.com/unclecode/crawl4ai/blob/main/SECURITY-CREDITS.md). + +If you find a security issue in Crawl4AI, please report it privately. See [SECURITY.md](https://github.com/unclecode/crawl4ai/blob/main/SECURITY.md). + +## Support & Resources + +- [Documentation](https://docs.crawl4ai.com) +- [GitHub Issues](https://github.com/unclecode/crawl4ai/issues) +- [Discord Community](https://discord.gg/crawl4ai) From 7245f984282d28b85946ce8d96f8e48d66da6c24 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Mon, 31 Aug 2026 13:47:40 +0200 Subject: [PATCH 38/38] docs: add the non-security bug fixes to the v0.9.3 release notes The release carries 33 commits that landed on develop since 0.9.2 in addition to the security fixes. The notes covered only the security work, which under-reported what shipped. --- CHANGELOG.md | 36 +++++++++++++++++++++++++++++++++- README.md | 4 +++- docs/blog/release-v0.9.3.md | 39 ++++++++++++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc663edfe..6121b1cd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.9.3] - 2026-08-31 -0.9.3 is a security release. It closes five coordinated-disclosure advisories in the PDF processing path and the Docker Playground UI. There are no new features and no breaking changes. Users who accept untrusted URLs on the Docker server, or who open PDFs from sources they do not control, should upgrade. +0.9.3 is a security release. It closes five coordinated-disclosure advisories in the PDF processing path and the Docker Playground UI, and ships the 33 bug fixes that accumulated on `develop` since 0.9.2, most of them in the Docker server. There are no new features and no breaking changes. Users who accept untrusted URLs on the Docker server, or who open PDFs from sources they do not control, should upgrade. ### Security @@ -21,11 +21,45 @@ The PDF path was the common thread. `PDFContentScrapingStrategy` is selectable f All reporters are credited in `SECURITY-CREDITS.md`. GitHub Security Advisories accompany this release. +### Fixed + +This release also carries the bug fixes that accumulated on `develop` since 0.9.2. + +**Docker server** + +- PDF scraping is supported by default, and requests selecting `PDFContentScrapingStrategy` are routed to `PDFCrawlerStrategy` so the pairing works without extra configuration. (#2094, #2150) +- The egress proxy chains through an upstream `HTTP_PROXY` / `HTTPS_PROXY` instead of ignoring it. (#2142) +- Junk proxy environment values fall through to the next candidate, and non-http proxy schemes are refused. (#2094) +- Compose v5 compatibility, clearer warnings on legacy fields, and better playground error handling. (#2094) +- `CRAWL4AI_API_TOKEN` is forwarded through compose, and `.llm.env` is optional rather than required. (#2094) +- The disabled-hooks 403 distinguishes the removed `hooks.code` field from other rejections. (#2094) +- `output_path` is declared a deprecated no-op rather than silently ignored. (#2094) +- `GET /monitor` redirects to the dashboard UI. (#2157, issue #2091) +- Failed crawl results are preserved instead of dropped, for both batch and single-URL requests. (#2094, #2134, issue #2133) +- An unavailable IPv6 loopback no longer breaks startup. (#2081, issue #2078) +- `mcp` is capped below 2 so the v1 low-level API used by `mcp_bridge` keeps working. (#2148, thanks @weike-zhang) +- Commented environment variable lines in compose are aligned with the environment list. (#2156) + +**Crawler and core** + +- `ManagedBrowser` no longer leaks a Playwright driver process when the browser fails to launch inside `__aenter__`. (#2160) +- `PDFCrawlerStrategy` placeholder responses are no longer vetoed as anti-bot blocks, which previously failed every PDF crawl and burned the retry budget. (#2138, issue #2135) +- Cookies are carried across manual PDF redirect hops, so gated and CDN-signed PDFs download correctly. (#2159) +- Unconditional `setTimeout` waits are removed from the overlay and consent removal scripts, which could hang a crawl under a restrictive CSP. (#2139) +- `remove_overlay_elements` no longer removes `` when the body carries a global popup class. (#2163, thanks @Nalhin) +- The body-visibility timeout is configurable and validated, and the timeout warning is emitted even with `verbose=False`. (#2117, #2131, #2145, issues #2116, #2129, #2144) + +**Documentation** + +- Self-hosting and migration guides updated for 0.9.x. (#2093) +- The `PDFCrawlerStrategy` plus `PDFContentScrapingStrategy` pairing requirement is documented. + ### Tests - `tests/unit/test_pdf_download_limits.py`: 22 tests covering per-hop destination validation, DNS rebinding, redirect bounds, byte and page caps, and untrusted-body clamping. - `tests/unit/test_pdf_html_escaping.py`: escaping of PDF paragraph text in `cleaned_html`. - `deploy/docker/tests/test_security_pdf_image_write.py`: rejection of image-write fields from untrusted bodies. +- Docker endpoint coverage for crawl failures and for the per-URL `crawler_configs` PDF guard. ### Breaking Changes diff --git a/README.md b/README.md index 1cd0ba157..310ac0240 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data [✨ Check out latest update v0.9.3](#-recent-updates) -✨ **New in v0.9.3**: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md) +✨ **New in v0.9.3**: Security release. Closes five coordinated-disclosure advisories: arbitrary file write, SSRF, and denial of service in the PDF processing path, plus two XSS issues in the Docker Playground. Also ships 33 bug fixes across the Docker server, crawler, and PDF handling. No new features, no breaking changes. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.3.md) ✨ Recent v0.9.2: Maintenance patch release. Fixes a `MemoryAdaptiveDispatcher` task/page leak when a streaming crawl is closed, Docker Playground "Advanced Config" and Monitor WebSocket auth, Playwright headless-shell packaging, and GPU (`ENABLE_GPU=true`) Docker builds. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.9.2.md) @@ -571,6 +571,8 @@ async def test_news_crawl(): A security release closing five coordinated-disclosure advisories. Four are in the PDF processing path: an arbitrary file write through `PDFContentScrapingStrategy` image-write fields, an SSRF where the PDF download followed redirects into internal addresses, a denial of service from unbounded PDF size and page count, and an XSS from unescaped PDF text in `cleaned_html`. The fifth is a DOM-based XSS in the Docker Playground that could expose the operator's API token. +It also carries 33 bug fixes that accumulated since 0.9.2: PDF scraping now works out of the box on the Docker server, the egress proxy chains through an upstream proxy, failed crawl results are reported instead of dropped, a Playwright driver leak on failed browser launch is fixed, and PDF crawls are no longer wrongly flagged as anti-bot blocks. + No new features, no breaking changes. Two defaults changed: PDF downloads now cap at 100 MiB and 2000 pages, and the Docker `limits.wall_clock_s` is now 300 seconds instead of 0. ```bash diff --git a/docs/blog/release-v0.9.3.md b/docs/blog/release-v0.9.3.md index 4d2d7260e..256c4b3fc 100644 --- a/docs/blog/release-v0.9.3.md +++ b/docs/blog/release-v0.9.3.md @@ -4,7 +4,7 @@ --- -I'm releasing Crawl4AI v0.9.3, a security release that closes five coordinated-disclosure advisories. No new features, no breaking changes. +I'm releasing Crawl4AI v0.9.3, a security release that closes five coordinated-disclosure advisories. It also carries the 33 bug fixes that piled up on `develop` since 0.9.2, most of them in the Docker server. No new features, no breaking changes. Four of the five are in the PDF path, and they share one root cause. `PDFContentScrapingStrategy` can be selected from an untrusted Docker API request body, and it downloads with `requests` rather than through the browser. Every control the server applies on the Chromium side (destination pinning, resource caps) simply did not reach it. The fifth is a DOM-based XSS in the Playground UI that could hand an operator's API token to an attacker. @@ -17,6 +17,7 @@ If you self-host the Docker server and accept crawl requests from clients you do - **Denial of service**: remote PDFs were downloaded and parsed with no size or page cap - **XSS**: PDF text was written into `cleaned_html` without escaping - **DOM XSS**: the Playground result viewer re-parsed crawled content as live HTML +- **33 other bug fixes**: Docker server, crawler, and PDF handling, listed further down ## Security fixes @@ -71,11 +72,45 @@ That round trip re-parses the text as live HTML. The text is crawl output, so it The round trip is removed. highlight.js renders from `textContent` and emits escaped markup on its own, so it was never needed. +## Bug fixes + +Everything below landed on `develop` between 0.9.2 and this release. None of it is security related. + +### Docker server + +- **PDF works out of the box**: PDF scraping is supported by default, and a request that selects `PDFContentScrapingStrategy` is now routed to `PDFCrawlerStrategy` automatically. Before this you had to pair them by hand or get a placeholder back. (#2094, #2150) +- **Upstream proxy chaining**: the egress proxy now chains through `HTTP_PROXY` / `HTTPS_PROXY` instead of ignoring them, so the server works behind a corporate proxy. (#2142) +- **Proxy env handling**: junk values in the proxy environment fall through to the next candidate rather than failing the crawl, and non-http proxy schemes are refused up front. (#2094) +- **Compose v5**: compatibility fixes, clearer warnings when a legacy field is used, and better error handling in the playground. (#2094) +- **Token and LLM env**: `CRAWL4AI_API_TOKEN` is forwarded through compose, and `.llm.env` is optional instead of required. (#2094) +- **Clearer 403 on hooks**: the disabled-hooks response now says when the request used the removed `hooks.code` field, rather than giving one generic refusal. (#2094) +- **`output_path`**: declared a deprecated no-op instead of being silently ignored. (#2094) +- **`GET /monitor`**: redirects to the dashboard UI instead of returning nothing useful. (#2157, issue #2091) +- **Failed crawls are reported**: failure details are preserved for both batch and single-URL requests instead of being dropped. (#2094, #2134, issue #2133) +- **IPv6 loopback**: an unavailable IPv6 loopback no longer breaks startup. (#2081, issue #2078) +- **MCP pinning**: `mcp` is capped below 2, keeping the v1 low-level API that `mcp_bridge` depends on. (#2148, thanks @weike-zhang) +- **Compose formatting**: commented environment variable lines are aligned with the environment list. (#2156) + +### Crawler and core + +- **Playwright driver leak**: when a browser failed to launch inside `__aenter__`, the Playwright driver subprocess was left running. Repeated failed launches leaked one process each time. It is now cleaned up. (#2160) +- **PDF crawls wrongly flagged as blocked**: `PDFCrawlerStrategy` returns a deliberately near-empty placeholder response, and the anti-bot check read that as a block. Every PDF crawl failed, burned the whole retry budget, and then called the fallback fetch. The placeholder is now recognised. (#2138, issue #2135) +- **Cookies across PDF redirects**: redirects are followed by hand, which meant each hop started with an empty cookie jar. A host that set a cookie and then redirected never got it back, so gated and CDN-signed PDFs failed. One session now covers the whole chain. (#2159) +- **Overlay removal hang**: the overlay and consent removal scripts waited on unconditional `setTimeout` calls, which could hang a crawl under a restrictive CSP. The waits are gone. (#2139) +- **`remove_overlay_elements` deleting the page**: a page whose `` carried a global popup class had its entire body removed, leaving an empty result. (#2163, thanks @Nalhin) +- **Body-visibility timeout**: now configurable and validated, and the timeout warning is emitted even when `verbose=False`. (#2117, #2131, #2145, issues #2116, #2129, #2144) + +### Documentation + +- Self-hosting and migration guides updated for 0.9.x. (#2093) +- The `PDFCrawlerStrategy` plus `PDFContentScrapingStrategy` pairing requirement is now documented. + ## Tests - `tests/unit/test_pdf_download_limits.py`, 22 tests: per-hop destination validation, DNS rebinding, redirect bounds, byte and page caps, untrusted-body clamping, and temp-file cleanup on abort. - `tests/unit/test_pdf_html_escaping.py`, 7 tests: escaping of PDF paragraph text in `cleaned_html`. - `deploy/docker/tests/test_security_pdf_image_write.py`, 6 tests: rejection of image-write fields from untrusted bodies. +- Docker endpoint coverage for crawl failures and for the per-URL `crawler_configs` PDF guard. ## Breaking changes @@ -103,6 +138,8 @@ docker pull unclecode/crawl4ai:0.9.3 Thank you to Zhixi "Jace" Sun, Nguyen Tran Thanh Lam ([c240030](https://github.com/c240030)), and [e1codes](https://github.com/e1codes) for reporting these issues privately and giving us time to fix them before disclosure. All reporters are listed in [SECURITY-CREDITS.md](https://github.com/unclecode/crawl4ai/blob/main/SECURITY-CREDITS.md). +Thanks to the community contributors behind the bug fixes in this release: @nightcityblade (#2130, #2131, #2117, #2134, #2081), @weike-zhang (#2148), and @Nalhin (#2163). + If you find a security issue in Crawl4AI, please report it privately. See [SECURITY.md](https://github.com/unclecode/crawl4ai/blob/main/SECURITY.md). ## Support & Resources