diff --git a/bug_triage/repro_1278.py b/bug_triage/repro_1278.py new file mode 100644 index 000000000..dab15c3f4 --- /dev/null +++ b/bug_triage/repro_1278.py @@ -0,0 +1,19 @@ +import asyncio +from crawl4ai import AsyncWebCrawler + +async def main(): + async with AsyncWebCrawler() as crawler: + # Target a page explicitly known for HTML tables + result = await crawler.arun(url="https://www.w3schools.com/html/html_tables.asp") + + if result.success: + if " tags are stripped from cleaned_html.") + else: + print("Fixed: Tables are preserved in cleaned_html.") + else: + print(f"Crawl failed: {result.error_message}") + +if __name__ == "__main__": + asyncio.run(main()) + \ No newline at end of file diff --git a/bug_triage/repro_1367.py b/bug_triage/repro_1367.py new file mode 100644 index 000000000..19c0d43e1 --- /dev/null +++ b/bug_triage/repro_1367.py @@ -0,0 +1,25 @@ +import asyncio +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig + +async def main(): + # Bug often triggers with persistent contexts and multiple concurrent navigations + browser_cfg = BrowserConfig(headless=True, use_persistent_context=True) + crawl_config = CrawlerRunConfig(wait_until="networkidle") + + urls = ["https://example.com", "https://example.org", "https://example.net"] + + async with AsyncWebCrawler(config=browser_cfg) as crawler: + print("Running async crawls to trigger ERR_ABORTED race condition...") + results = await crawler.arun_many(urls=urls, config=crawl_config) + + reproduced = False + for res in results: + if not res.success and "ERR_ABORTED" in str(res.error_message): + print(f"Bug reproduced on {res.url}: {res.error_message}") + reproduced = True + + if not reproduced: + print("Fixed or unable to reproduce: No ERR_ABORTED errors encountered.") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/bug_triage/repro_1455.py b/bug_triage/repro_1455.py new file mode 100644 index 000000000..0d3d80058 --- /dev/null +++ b/bug_triage/repro_1455.py @@ -0,0 +1,32 @@ +import os +import asyncio +from pydantic import BaseModel +from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig +from crawl4ai import LLMExtractionStrategy + +class Product(BaseModel): + name: str + +async def main(): + llm_strategy = LLMExtractionStrategy( + llm_config=LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv('OPENAI_API_KEY', 'dummy')), + schema=Product.schema_json(), + extraction_type="schema", + instruction="Extract products." + ) + crawl_config = CrawlerRunConfig(extraction_strategy=llm_strategy, cache_mode=CacheMode.ENABLED) + + async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler: + print("First run (caching)...") + await crawler.arun(url="https://example.com", config=crawl_config) + + print("Second run (cache hit)...") + result = await crawler.arun(url="https://example.com", config=crawl_config) + + if not result.extracted_content: + print("Bug reproduced: extracted_content is empty on cache hit.") + else: + print("Fixed: extracted_content populated from cache.") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/bug_triage/repro_570.py b/bug_triage/repro_570.py new file mode 100644 index 000000000..f760886c9 --- /dev/null +++ b/bug_triage/repro_570.py @@ -0,0 +1,19 @@ +import asyncio +from crawl4ai import AsyncWebCrawler + +async def main(): + async with AsyncWebCrawler() as crawler: + result = await crawler.arun(url="https://docs.crawl4ai.com/") + + if result.success: + markdown = result.markdown or "" + if "<.>" in markdown or "<#>" in markdown: + print("Bug reproduced: Relative URLs are incorrectly formatted with brackets (e.g., <.>).") + else: + print("Fixed: No malformed relative URL brackets found in markdown.") + else: + print(f"Crawl failed: {result.error_message}") + +if __name__ == "__main__": + asyncio.run(main()) + \ No newline at end of file diff --git a/bug_triage/repro_699.py b/bug_triage/repro_699.py new file mode 100644 index 000000000..e01887428 --- /dev/null +++ b/bug_triage/repro_699.py @@ -0,0 +1,20 @@ +import asyncio +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig + +async def main(): + # Enable robots.txt checking + config = CrawlerRunConfig(check_robots_txt=True) + + async with AsyncWebCrawler() as crawler: + # The path /awardsearch/advancedSearch.jsp is disallowed in nsf.gov/robots.txt + url = "https://www.nsf.gov/awardsearch/advancedSearch.jsp" + result = await crawler.arun(url=url, config=config) + + if result.success: + print(f"Bug reproduced: Successfully crawled {url} despite robots.txt disallow.") + else: + print(f"Fixed/Expected behavior: Blocked from crawling. Message: {result.error_message}") + +if __name__ == "__main__": + asyncio.run(main()) + \ No newline at end of file diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py index 511fde692..131322a00 100644 --- a/crawl4ai/deep_crawling/bff_strategy.py +++ b/crawl4ai/deep_crawling/bff_strategy.py @@ -83,8 +83,6 @@ async def can_process_url(self, url: str, depth: int) -> bool: raise ValueError("Missing scheme or netloc") if parsed.scheme not in ("http", "https"): raise ValueError("Invalid scheme") - if "." not in parsed.netloc: - raise ValueError("Invalid domain") except Exception as e: self.logger.warning(f"Invalid URL: {url}, error: {e}") return False diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py index dfb759272..70529a92c 100644 --- a/crawl4ai/deep_crawling/bfs_strategy.py +++ b/crawl4ai/deep_crawling/bfs_strategy.py @@ -70,8 +70,6 @@ async def can_process_url(self, url: str, depth: int) -> bool: raise ValueError("Missing scheme or netloc") if parsed.scheme not in ("http", "https"): raise ValueError("Invalid scheme") - if "." not in parsed.netloc: - raise ValueError("Invalid domain") except Exception as e: self.logger.warning(f"Invalid URL: {url}, error: {e}") return False