From 5adcfbdba5a0a3b3094ddc7b2e2fd1d11c059ef6 Mon Sep 17 00:00:00 2001 From: Battleplus <3559424769@qq.com> Date: Sat, 22 Aug 2026 00:29:59 +0800 Subject: [PATCH] fix: skip anti-bot detection for non-HTML Content-Type responses PDFCrawlerStrategy returns a minimal placeholder HTML with Content-Type: application/pdf. The anti-bot detector's near-empty content check (threshold: 100 bytes) misreads this as a blocked page. Instead of padding the placeholder string to exceed the threshold, skip the anti-bot near-empty check when the response Content-Type is non-HTML (e.g. application/pdf). This is the correct semantic: PDF responses are expected to have minimal HTML by design. Fixes #2135 Signed-off-by: Battleplus <3559424769@qq.com> --- crawl4ai/async_webcrawler.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index 8216d19bc..e002904e3 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -504,8 +504,12 @@ async def arun( crawl_result.cache_status = "miss" # Check if blocked (skip for raw: URLs — - # caller-provided content, anti-bot N/A) - if _is_raw_url: + # caller-provided content, anti-bot N/A; + # skip for non-HTML Content-Type like application/pdf + # where minimal HTML is expected by design) + _ct = async_response.response_headers.get("Content-Type", "") + _non_html = _ct and not _ct.startswith("text/html") + if _is_raw_url or _non_html: _blocked = False _block_reason = "" else: @@ -552,9 +556,11 @@ async def arun( # Skip for raw: URLs — fallback expects a real URL, not raw HTML content. _fallback_fn = getattr(config, "fallback_fetch_function", None) if _fallback_fn and not _done and not _is_raw_url: + _ct2 = (getattr(crawl_result, "response_headers", None) or {}).get("Content-Type", "") + _non_html2 = _ct2 and not _ct2.startswith("text/html") _needs_fallback = ( crawl_result is None # All proxies threw exceptions - or is_blocked(crawl_result.status_code, crawl_result.html or "")[0] + or (not _non_html2 and is_blocked(crawl_result.status_code, crawl_result.html or "")[0]) ) if _needs_fallback: self.logger.warning( @@ -625,7 +631,9 @@ async def arun( # empty by design, and is_blocked() would misread "0 bytes # html" as a block. _has_download = bool(getattr(crawl_result, "downloaded_files", None)) - if not _fallback_succeeded and not _is_raw_url and not _has_download: + _ct = (crawl_result.response_headers or {}).get("Content-Type", "") + _non_html = _ct and not _ct.startswith("text/html") + if not _fallback_succeeded and not _is_raw_url and not _has_download and not _non_html: _blocked, _block_reason = is_blocked( crawl_result.status_code, crawl_result.html or "") if _blocked: