From 12e6c2b4c3ac97721efcdfa93a752bab359d1e89 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Sat, 31 Oct 2026 12:30:51 +0100 Subject: [PATCH] feat!: align `enqueue_links` URL filters and `limit` with Crawlee JS --- docs/upgrading/upgrading_to_v2.md | 16 +++++++ src/crawlee/_types.py | 9 ++-- src/crawlee/_utils/globs.py | 35 ++++++++++++-- .../_abstract_http/_abstract_http_crawler.py | 4 ++ src/crawlee/crawlers/_basic/_basic_crawler.py | 46 +++---------------- .../_playwright/_playwright_crawler.py | 4 ++ .../_sitemap_request_loader.py | 41 +++-------------- tests/unit/_utils/test_globs.py | 36 ++++++++++++++- .../crawlers/_basic/test_basic_crawler.py | 30 ++++++++++++ .../test_beautifulsoup_crawler.py | 24 ++++++++++ .../crawlers/_parsel/test_parsel_crawler.py | 24 ++++++++++ .../_playwright/test_playwright_crawler.py | 24 ++++++++++ .../test_sitemap_request_loader.py | 28 ++++++++++- 13 files changed, 239 insertions(+), 82 deletions(-) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index f7e0bd036b..16d017ba54 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -129,6 +129,22 @@ The default `Request.unique_key` lowercases only the scheme and host of the URL. As a result, a crawl can visit more pages than in v1. Requests stored in a request queue by v1 keep their lowercased unique keys, so adding the same URL again in v2 can create a duplicate request. To keep case-insensitive deduplication, pass an explicit `unique_key`. With `enqueue_links`, set `unique_key` to the lowercased URL in its `transform_request_function`, and leave `url` as it is, so the crawler still fetches the original casing. +## URL filters match the same way as in Crawlee JS + +The `include`, `exclude` and `limit` arguments of `enqueue_links`, `extract_links` and `add_requests` now behave the same as in Crawlee JS. The `include` and `exclude` arguments of `SitemapRequestLoader` follow the same matching rules. + +### Globs are case-insensitive + +A `Glob` matches URLs regardless of casing, so `Glob('https://example.com/products/**')` also matches `https://example.com/Products/ABC`. In v1, the matching was case-sensitive. For case-sensitive matching, use a regular expression instead. + +### Regular expressions match anywhere in the URL + +A regular expression in `include` or `exclude` matches if it's found anywhere in the URL, the same as `re.search`. In v1, it had to match at the start of the URL, the same as `re.match`. For example, `re.compile(r'/products/')` now matches `https://example.com/products/1`, where v1 needed `re.compile(r'https://example\.com/products/')`. To match only at the start of the URL again, start the pattern with `^`, for example `re.compile(r'^https://example\.com/products/')`. + +### `limit` is applied last + +The `limit` argument counts only the requests that pass all the filters. It's applied after the strategy, `include` and `exclude` filters, the `transform_request_function` and the `max_crawl_depth` check. In v1, it was applied before the `transform_request_function`, so the requests that `transform_request_function` skipped still counted towards the limit. For example, with `limit=5` and a `transform_request_function` that skips 2 of the first 5 links, v2 enqueues 5 requests, where v1 enqueued 3. + ## Storage data uses camelCase keys The storage backends now write their JSON data with camelCase keys, the same as Crawlee JS. This covers the metadata, request and record metadata files of `FileSystemStorageBackend`, the request and record metadata JSON stored by `RedisStorageBackend`, and the request JSON stored by `SqlStorageBackend`. For example, `item_count` in a dataset's `__metadata__.json` becomes `itemCount`. The storage metadata documents of `RedisStorageBackend` keep their snake_case keys. diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 18bb2435a3..5b40157cc0 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -148,7 +148,8 @@ class EnqueueLinksKwargs(TypedDict): """Keyword arguments for the `enqueue_links` methods.""" limit: NotRequired[int] - """Maximum number of requests to be enqueued.""" + """Maximum number of requests to be enqueued. It's applied last, after the URL filters and the + `transform_request_function`, so the requests they skip don't count towards it.""" base_url: NotRequired[str] """Base URL to be used for relative URLs.""" @@ -171,10 +172,12 @@ class EnqueueLinksKwargs(TypedDict): """ include: NotRequired[Sequence[re.Pattern | Glob]] - """List of regular expressions or globs that URLs must match to be enqueued.""" + """List of regular expressions or globs that URLs must match to be enqueued. A regular expression may match + anywhere in the URL, like `re.search`. A glob must match the whole URL and is case-insensitive.""" exclude: NotRequired[Sequence[re.Pattern | Glob]] - """List of regular expressions or globs that URLs must not match to be enqueued.""" + """List of regular expressions or globs that URLs must not match to be enqueued. A regular expression may match + anywhere in the URL, like `re.search`. A glob must match the whole URL and is case-insensitive.""" class AddRequestsKwargs(EnqueueLinksKwargs): diff --git a/src/crawlee/_utils/globs.py b/src/crawlee/_utils/globs.py index ab352113a3..e6871c85e2 100644 --- a/src/crawlee/_utils/globs.py +++ b/src/crawlee/_utils/globs.py @@ -2,18 +2,47 @@ import os import re -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from collections.abc import Sequence class Glob: - """Wraps a glob pattern (supports the `*`, `**`, `?` wildcards).""" + """Wraps a glob pattern (supports the `*`, `**`, `?` wildcards). + + The pattern must match the whole URL, and the matching is case-insensitive. For case-sensitive matching, use a + regular expression. + """ def __init__(self, glob: str) -> None: self.glob = glob - self.regexp = re.compile(_translate(self.glob, recursive=True)) + self.regexp = re.compile(_translate(self.glob, recursive=True), re.IGNORECASE) + + +def matches_url_pattern(pattern: re.Pattern[Any] | Glob, url: str) -> bool: + """Check whether `url` matches an include or exclude pattern. + + A glob must match the whole URL, while a regular expression may match anywhere in it, the same as in Crawlee JS. + """ + if isinstance(pattern, Glob): + return pattern.regexp.match(url) is not None + return pattern.search(url) is not None + + +def check_url_patterns( + url: str, + include: Sequence[re.Pattern[Any] | Glob] | None, + exclude: Sequence[re.Pattern[Any] | Glob] | None, +) -> bool: + """Check whether `url` matches no `exclude` pattern and, when `include` is given, at least one `include` pattern.""" + if any(matches_url_pattern(pattern, url) for pattern in exclude or ()): + return False + + if include is None: + return True + + return any(matches_url_pattern(pattern, url) for pattern in include) def _translate( diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py index 5f189b5b97..37a5d5929a 100644 --- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py +++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py @@ -213,6 +213,7 @@ async def extract_links( kwargs.setdefault('strategy', 'same-hostname') strategy = kwargs.get('strategy', 'same-hostname') + limit = kwargs.get('limit') links_iterator: Iterator[str] = iter( self._parser.find_links(parsed_content, selector=selector, attribute=attribute) @@ -239,6 +240,9 @@ async def extract_links( skipped = iter([]) for url in self._enqueue_links_filter_iterator(links_iterator, context.request.url, **kwargs): + if limit is not None and len(requests) >= limit: + break + request_options = RequestOptions( url=url, user_data={**base_user_data}, label=label, enqueue_strategy=strategy ) diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 4b2dc37295..850ba42e17 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -24,7 +24,7 @@ from typing_extensions import TypeVar from yarl import URL -from crawlee import Glob, RequestTransformAction, service_locator +from crawlee import RequestTransformAction, service_locator from crawlee._log_config import configure_logger, string_to_log_level from crawlee._request import Request, RequestOptions, RequestState from crawlee._service_locator import ServiceLocator @@ -44,6 +44,7 @@ ) from crawlee._utils.docs import docs_group from crawlee._utils.file import atomic_write, export_csv_to_stream, export_json_to_stream +from crawlee._utils.globs import check_url_patterns from crawlee._utils.http import parse_retry_after_header from crawlee._utils.log import LoggerOnce from crawlee._utils.recurring_task import RecurringTask @@ -80,7 +81,6 @@ ) if TYPE_CHECKING: - import re from collections.abc import Iterator, Mapping, MutableMapping from contextlib import AbstractAsyncContextManager @@ -1165,7 +1165,6 @@ def _enqueue_links_filter_iterator( self, request_iterator: Iterator[TRequestIterator], origin_url: str, **kwargs: Unpack[EnqueueLinksKwargs] ) -> Iterator[TRequestIterator]: """Filter requests based on the enqueue strategy and URL patterns.""" - limit = kwargs.get('limit') parsed_origin_url = URL(origin_url) strategy = kwargs.get('strategy', 'all') @@ -1199,44 +1198,9 @@ def _enqueue_links_filter_iterator( host_warned = True continue - if self._check_url_patterns(target_url, kwargs.get('include'), kwargs.get('exclude')): + if check_url_patterns(target_url, kwargs.get('include'), kwargs.get('exclude')): yield request - if limit is not None: - limit -= 1 - if limit <= 0: - break - - def _check_url_patterns( - self, - target_url: str, - include: Sequence[re.Pattern[Any] | Glob] | None, - exclude: Sequence[re.Pattern[Any] | Glob] | None, - ) -> bool: - """Check if a URL matches configured include/exclude patterns.""" - # If the URL matches any `exclude` pattern, reject it - for pattern in exclude or (): - if isinstance(pattern, Glob): - pattern = pattern.regexp # noqa: PLW2901 - - if pattern.match(target_url) is not None: - return False - - # If there are no `include` patterns and the URL passed all `exclude` patterns, accept the URL - if include is None: - return True - - # If the URL matches any `include` pattern, accept it - for pattern in include: - if isinstance(pattern, Glob): - pattern = pattern.regexp # noqa: PLW2901 - - if pattern.match(target_url) is not None: - return True - - # The URL does not match any `include` pattern - reject it - return False - async def _handle_error_handler_replacement( self, context: TCrawlingContext | BasicCrawlingContext, @@ -1463,11 +1427,15 @@ async def _add_requests( else: request_manager = await self.get_request_manager() + limit = kwargs.get('limit') context_aware_requests = list[Request]() base_url = kwargs.get('base_url') or context.request.loaded_url or context.request.url requests_iterator = self._convert_url_to_request_iterator(requests, base_url) filter_requests_iterator = self._enqueue_links_filter_iterator(requests_iterator, context.request.url, **kwargs) for dst_request in filter_requests_iterator: + if limit is not None and len(context_aware_requests) >= limit: + break + # Update the crawl depth of the request. dst_request.crawl_depth = context.request.crawl_depth + 1 diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 3069845b98..808b3c94c0 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -443,6 +443,7 @@ async def extract_links( kwargs.setdefault('strategy', 'same-hostname') strategy = kwargs.get('strategy', 'same-hostname') + limit = kwargs.get('limit') elements = await context.page.query_selector_all(selector) # Browsers ignore the whitespace around a URL, as `find_links` of the HTTP parsers does. @@ -465,6 +466,9 @@ async def extract_links( skipped = iter([]) for url in self._enqueue_links_filter_iterator(links_iterator, context.request.url, **kwargs): + if limit is not None and len(requests) >= limit: + break + request_options = RequestOptions( url=url, user_data={**base_user_data}, label=label, enqueue_strategy=strategy ) diff --git a/src/crawlee/request_loaders/_sitemap_request_loader.py b/src/crawlee/request_loaders/_sitemap_request_loader.py index 9f51c458ca..2bfc7b2ecf 100644 --- a/src/crawlee/request_loaders/_sitemap_request_loader.py +++ b/src/crawlee/request_loaders/_sitemap_request_loader.py @@ -13,7 +13,7 @@ from crawlee import Request, RequestOptions from crawlee._utils.docs import docs_group -from crawlee._utils.globs import Glob +from crawlee._utils.globs import Glob, check_url_patterns from crawlee._utils.recoverable_state import RecoverableState from crawlee._utils.sitemap import ( DEFAULT_MAX_DEPTH, @@ -30,7 +30,7 @@ if TYPE_CHECKING: import re - from collections.abc import Callable, Sequence + from collections.abc import Callable from types import TracebackType from crawlee import RequestTransformAction @@ -139,8 +139,9 @@ def __init__( Args: sitemap_urls: List of sitemap URLs to fetch and parse for request URLs. proxy_info: Optional proxy to use for fetching sitemaps. - include: List of glob or regex patterns to include URLs. - exclude: List of glob or regex patterns to exclude URLs. + include: List of glob or regex patterns to include URLs. A regex may match anywhere in the URL, like + `re.search`. A glob must match the whole URL and is case-insensitive. + exclude: List of glob or regex patterns to exclude URLs, matched the same way as `include`. enqueue_strategy: Strategy used to decide which sitemap-derived URLs (both nested-sitemap entries and URL entries) are kept relative to the parent sitemap URL. Defaults to `'same-hostname'`, matching the sitemap protocol's same-host expectation and the `enqueue_links` default; pass `'all'` to @@ -326,36 +327,6 @@ def _passes_filters(self, target: str, parent: URL, kind: str) -> bool: return False return True - def _check_url_patterns( - self, - target_url: str, - include: Sequence[re.Pattern[Any] | Glob] | None, - exclude: Sequence[re.Pattern[Any] | Glob] | None, - ) -> bool: - """Check if a URL matches configured include/exclude patterns.""" - # If the URL matches any `exclude` pattern, reject it - for pattern in exclude or (): - if isinstance(pattern, Glob): - pattern = pattern.regexp # noqa: PLW2901 - - if pattern.match(target_url) is not None: - return False - - # If there are no `include` patterns and the URL passed all `exclude` patterns, accept the URL - if include is None: - return True - - # If the URL matches any `include` pattern, accept it - for pattern in include: - if isinstance(pattern, Glob): - pattern = pattern.regexp # noqa: PLW2901 - - if pattern.match(target_url) is not None: - return True - - # The URL does not match any `include` pattern - reject it - return False - async def _load_sitemaps(self) -> None: """Load URLs from sitemaps in the background.""" try: @@ -420,7 +391,7 @@ async def _load_sitemaps(self) -> None: continue # Check if URL should be included - if not self._check_url_patterns(url, self._include, self._exclude): + if not check_url_patterns(url, self._include, self._exclude): continue if not self._passes_filters(url, parsed_sitemap_url, 'sitemap URL'): diff --git a/tests/unit/_utils/test_globs.py b/tests/unit/_utils/test_globs.py index 970678cfed..8ad2978d9f 100644 --- a/tests/unit/_utils/test_globs.py +++ b/tests/unit/_utils/test_globs.py @@ -1,6 +1,10 @@ from __future__ import annotations -from crawlee._utils.globs import Glob +import re + +import pytest + +from crawlee._utils.globs import Glob, matches_url_pattern def test_asterisk() -> None: @@ -15,3 +19,33 @@ def test_double_asteritsk() -> None: assert glob.regexp.match('bar/') is None assert glob.regexp.match('foo/bar') is not None assert glob.regexp.match('foo/bar/baz') is not None + + +@pytest.mark.parametrize( + 'url', + [ + pytest.param('https://example.com/products/abc', id='same casing'), + pytest.param('https://example.com/Products/ABC', id='uppercase path'), + pytest.param('HTTPS://EXAMPLE.COM/PRODUCTS/ABC', id='uppercase url'), + ], +) +def test_glob_is_case_insensitive(url: str) -> None: + """A glob matches a URL regardless of its casing.""" + assert matches_url_pattern(Glob('https://example.com/products/*'), url) + + +@pytest.mark.parametrize( + ('pattern', 'expected'), + [ + pytest.param(re.compile(r'/products/'), True, id='regex in the middle'), + pytest.param(re.compile(r'\d+$'), True, id='regex at the end'), + pytest.param(re.compile(r'^/products/'), False, id='regex anchored at the start'), + pytest.param(re.compile(r'/Products/'), False, id='regex is case-sensitive'), + pytest.param(Glob('/products/*'), False, id='glob must match the whole url'), + pytest.param(Glob('https://example.com/products'), False, id='glob prefix of the url'), + pytest.param(Glob('https://example.com/products/*'), True, id='glob matching the whole url'), + ], +) +def test_matches_url_pattern(pattern: re.Pattern[str] | Glob, *, expected: bool) -> None: + """A regex may match anywhere in the URL, while a glob must match the whole URL.""" + assert matches_url_pattern(pattern, 'https://example.com/products/123') is expected diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index f4b614b557..c312eac947 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -903,6 +903,36 @@ class AddRequestsTestInput: ), id='include_exclude_3', ), + pytest.param( + AddRequestsTestInput( + start_url=INCLUDE_TEST_URLS[0], + loaded_url=INCLUDE_TEST_URLS[0], + requests=INCLUDE_TEST_URLS, + kwargs=EnqueueLinksKwargs(include=[re.compile(r'/cats$')], exclude=[re.compile(r'/archive/')]), + expected_urls=[INCLUDE_TEST_URLS[1]], + ), + id='include_exclude_regex_matches_anywhere', + ), + pytest.param( + AddRequestsTestInput( + start_url=INCLUDE_TEST_URLS[0], + loaded_url=INCLUDE_TEST_URLS[0], + requests=INCLUDE_TEST_URLS, + kwargs=EnqueueLinksKwargs(include=[Glob('https://SomePlace.com/**/CATS')]), + expected_urls=[INCLUDE_TEST_URLS[1], INCLUDE_TEST_URLS[4]], + ), + id='include_glob_case_insensitive', + ), + pytest.param( + AddRequestsTestInput( + start_url=INCLUDE_TEST_URLS[0], + loaded_url=INCLUDE_TEST_URLS[0], + requests=INCLUDE_TEST_URLS[1:], + kwargs=EnqueueLinksKwargs(exclude=[Glob('https://**/category/**')], limit=1), + expected_urls=[INCLUDE_TEST_URLS[3]], + ), + id='limit_counts_requests_passing_filters', + ), ], ) async def test_enqueue_strategy(test_input: AddRequestsTestInput) -> None: diff --git a/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py b/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py index 7f5ee1130b..7bf738bfeb 100644 --- a/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py +++ b/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py @@ -554,6 +554,30 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: visit.assert_has_calls(expected_visit_calls, any_order=True) +async def test_enqueue_links_limit_applied_after_transform(server_url: URL, http_client: HttpClient) -> None: + """Requests skipped by `transform_request_function` don't count towards the `limit` of `enqueue_links`.""" + crawler = BeautifulSoupCrawler(http_client=http_client) + visit = mock.Mock() + + def skip_page_3(request_options: RequestOptions) -> RequestOptions | RequestTransformAction: + if request_options['url'].endswith('/page_3'): + return 'skip' + return 'unchanged' + + @crawler.router.default_handler + async def request_handler(context: BeautifulSoupCrawlingContext) -> None: + await context.enqueue_links(limit=1, transform_request_function=skip_page_3, label='child') + + @crawler.router.handler('child') + async def child_handler(context: BeautifulSoupCrawlingContext) -> None: + visit(context.request.url) + + await crawler.run([str(server_url / 'sub_index')]) + + visited = {call.args[0] for call in visit.call_args_list} + assert visited == {str(server_url / 'page_2')} + + def test_import_error_handled() -> None: # Simulate ImportError for BeautifulSoup with mock.patch.dict('sys.modules', {'bs4': None}): diff --git a/tests/unit/crawlers/_parsel/test_parsel_crawler.py b/tests/unit/crawlers/_parsel/test_parsel_crawler.py index 2b26138336..708bcef3f6 100644 --- a/tests/unit/crawlers/_parsel/test_parsel_crawler.py +++ b/tests/unit/crawlers/_parsel/test_parsel_crawler.py @@ -565,6 +565,30 @@ async def request_handler(context: ParselCrawlingContext) -> None: visit.assert_has_calls(expected_visit_calls, any_order=True) +async def test_enqueue_links_limit_applied_after_transform(server_url: URL, http_client: HttpClient) -> None: + """Requests skipped by `transform_request_function` don't count towards the `limit` of `enqueue_links`.""" + crawler = ParselCrawler(http_client=http_client) + visit = mock.Mock() + + def skip_page_3(request_options: RequestOptions) -> RequestOptions | RequestTransformAction: + if request_options['url'].endswith('/page_3'): + return 'skip' + return 'unchanged' + + @crawler.router.default_handler + async def request_handler(context: ParselCrawlingContext) -> None: + await context.enqueue_links(limit=1, transform_request_function=skip_page_3, label='child') + + @crawler.router.handler('child') + async def child_handler(context: ParselCrawlingContext) -> None: + visit(context.request.url) + + await crawler.run([str(server_url / 'sub_index')]) + + visited = {call.args[0] for call in visit.call_args_list} + assert visited == {str(server_url / 'page_2')} + + @pytest.mark.parametrize( ('texts', 'body', 'content_type'), [ diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 0c8578ce13..a095cf3157 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -1288,6 +1288,30 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: visit.assert_has_calls(expected_visit_calls, any_order=True) +async def test_enqueue_links_limit_applied_after_transform(server_url: URL) -> None: + """Requests skipped by `transform_request_function` don't count towards the `limit` of `enqueue_links`.""" + crawler = PlaywrightCrawler() + visit = mock.Mock() + + def skip_page_3(request_options: RequestOptions) -> RequestOptions | RequestTransformAction: + if request_options['url'].endswith('/page_3'): + return 'skip' + return 'unchanged' + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + await context.enqueue_links(limit=1, transform_request_function=skip_page_3, label='child') + + @crawler.router.handler('child') + async def child_handler(context: PlaywrightCrawlingContext) -> None: + visit(context.request.url) + + await crawler.run([str(server_url / 'sub_index')]) + + visited = {call.args[0] for call in visit.call_args_list} + assert visited == {str(server_url / 'page_2')} + + async def test_playwright_crawler_pre_navigation_hook_execution(server_url: URL) -> None: """Test that pre-navigation hooks are executed.""" crawler = PlaywrightCrawler(request_handler=AsyncMock()) diff --git a/tests/unit/request_loaders/test_sitemap_request_loader.py b/tests/unit/request_loaders/test_sitemap_request_loader.py index 7bc1b2be03..08908c51ba 100644 --- a/tests/unit/request_loaders/test_sitemap_request_loader.py +++ b/tests/unit/request_loaders/test_sitemap_request_loader.py @@ -1,13 +1,14 @@ import asyncio import base64 import gzip +import re from typing import TYPE_CHECKING, Any, cast from unittest.mock import AsyncMock, MagicMock, patch import pytest from yarl import URL -from crawlee import RequestOptions, RequestTransformAction +from crawlee import Glob, RequestOptions, RequestTransformAction from crawlee._types import BasicCrawlingContext from crawlee._utils.sitemap import DEFAULT_MAX_DEPTH from crawlee.crawlers import BasicCrawler @@ -555,3 +556,28 @@ async def test_sitemap_loader_drops_non_http_scheme_under_strategy_all( await loader.mark_request_as_handled(request) assert fetched == [http_url] + + +async def test_sitemap_loader_include_exclude_matching(server_url: URL, http_client: HttpClient) -> None: + """An `include` regex may match anywhere in the URL, and an `exclude` glob matches regardless of casing.""" + kept_url = str(server_url / 'products' / '1') + excluded_url = str(server_url / 'products' / 'Archive' / '2') + not_included_url = str(server_url / 'about') + sitemap_content = _make_urlset([kept_url, excluded_url, not_included_url]) + sitemap_url = (server_url / 'sitemap.xml').with_query(base64=encode_base64(sitemap_content.encode())) + + loader = SitemapRequestLoader( + [str(sitemap_url)], + http_client=http_client, + include=[re.compile(r'/products/')], + exclude=[Glob(str(server_url / 'products' / 'archive' / '**'))], + ) + + fetched: list[str] = [] + while not finished(await loader.check_readiness()): + request = await loader.fetch_next_request() + if request is not None: + fetched.append(request.url) + await loader.mark_request_as_handled(request) + + assert fetched == [kept_url]