fix: allow single-label hostnames in deep crawling

bug_triage/repro_1278.py

@@ -0,0 +1,19 @@
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        # Target a page explicitly known for HTML tables
+        result = await crawler.arun(url="https://www.w3schools.com/html/html_tables.asp")
+
+        if result.success:
+            if "<table" not in result.cleaned_html.lower():
+                print("Bug reproduced: <table> tags are stripped from cleaned_html.")
+            else:
+                print("Fixed: Tables are preserved in cleaned_html.")
+        else:
+            print(f"Crawl failed: {result.error_message}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+

bug_triage/repro_1367.py

@@ -0,0 +1,25 @@
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def main():
+    # Bug often triggers with persistent contexts and multiple concurrent navigations
+    browser_cfg = BrowserConfig(headless=True, use_persistent_context=True)
+    crawl_config = CrawlerRunConfig(wait_until="networkidle")
+
+    urls = ["https://example.com", "https://example.org", "https://example.net"]
+
+    async with AsyncWebCrawler(config=browser_cfg) as crawler:
+        print("Running async crawls to trigger ERR_ABORTED race condition...")
+        results = await crawler.arun_many(urls=urls, config=crawl_config)
+
+        reproduced = False
+        for res in results:
+            if not res.success and "ERR_ABORTED" in str(res.error_message):
+                print(f"Bug reproduced on {res.url}: {res.error_message}")
+                reproduced = True
+
+        if not reproduced:
+            print("Fixed or unable to reproduce: No ERR_ABORTED errors encountered.")
+
+if __name__ == "__main__":
+    asyncio.run(main())

bug_triage/repro_1455.py

@@ -0,0 +1,32 @@
+import os
+import asyncio
+from pydantic import BaseModel
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig
+from crawl4ai import LLMExtractionStrategy
+
+class Product(BaseModel):
+    name: str
+
+async def main():
+    llm_strategy = LLMExtractionStrategy(
+        llm_config=LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv('OPENAI_API_KEY', 'dummy')),
+        schema=Product.schema_json(),
+        extraction_type="schema",
+        instruction="Extract products."
+    )
+    crawl_config = CrawlerRunConfig(extraction_strategy=llm_strategy, cache_mode=CacheMode.ENABLED)
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        print("First run (caching)...")
+        await crawler.arun(url="https://example.com", config=crawl_config)
+
+        print("Second run (cache hit)...")
+        result = await crawler.arun(url="https://example.com", config=crawl_config)
+
+        if not result.extracted_content:
+            print("Bug reproduced: extracted_content is empty on cache hit.")
+        else:
+            print("Fixed: extracted_content populated from cache.")
+
+if __name__ == "__main__":
+    asyncio.run(main())

bug_triage/repro_570.py

@@ -0,0 +1,19 @@
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url="https://docs.crawl4ai.com/")
+
+        if result.success:
+            markdown = result.markdown or ""
+            if "<.>" in markdown or "<#>" in markdown:
+                print("Bug reproduced: Relative URLs are incorrectly formatted with brackets (e.g., <.>).")
+            else:
+                print("Fixed: No malformed relative URL brackets found in markdown.")
+        else:
+            print(f"Crawl failed: {result.error_message}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+

bug_triage/repro_699.py

@@ -0,0 +1,20 @@
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+    # Enable robots.txt checking
+    config = CrawlerRunConfig(check_robots_txt=True)
+
+    async with AsyncWebCrawler() as crawler:
+        # The path /awardsearch/advancedSearch.jsp is disallowed in nsf.gov/robots.txt
+        url = "https://www.nsf.gov/awardsearch/advancedSearch.jsp"
+        result = await crawler.arun(url=url, config=config)
+
+        if result.success:
+            print(f"Bug reproduced: Successfully crawled {url} despite robots.txt disallow.")
+        else:
+            print(f"Fixed/Expected behavior: Blocked from crawling. Message: {result.error_message}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+

crawl4ai/deep_crawling/bff_strategy.py

@@ -83,8 +83,6 @@ async def can_process_url(self, url: str, depth: int) -> bool:
                 raise ValueError("Missing scheme or netloc")
             if parsed.scheme not in ("http", "https"):
                 raise ValueError("Invalid scheme")
-            if "." not in parsed.netloc:
-                raise ValueError("Invalid domain")
         except Exception as e:
             self.logger.warning(f"Invalid URL: {url}, error: {e}")
             return False

crawl4ai/deep_crawling/bfs_strategy.py

@@ -70,8 +70,6 @@ async def can_process_url(self, url: str, depth: int) -> bool:
                 raise ValueError("Missing scheme or netloc")
             if parsed.scheme not in ("http", "https"):
                 raise ValueError("Invalid scheme")
-            if "." not in parsed.netloc:
-                raise ValueError("Invalid domain")
         except Exception as e:
             self.logger.warning(f"Invalid URL: {url}, error: {e}")
             return False