feat!: align enqueue_links URL filters and limit with Crawlee JS

docs/upgrading/upgrading_to_v2.md

@@ -129,6 +129,22 @@ The default `Request.unique_key` lowercases only the scheme and host of the URL.
 
 As a result, a crawl can visit more pages than in v1. Requests stored in a request queue by v1 keep their lowercased unique keys, so adding the same URL again in v2 can create a duplicate request. To keep case-insensitive deduplication, pass an explicit `unique_key`. With `enqueue_links`, set `unique_key` to the lowercased URL in its `transform_request_function`, and leave `url` as it is, so the crawler still fetches the original casing.
 
+## URL filters match the same way as in Crawlee JS
+
+The `include`, `exclude` and `limit` arguments of `enqueue_links`, `extract_links` and `add_requests` now behave the same as in Crawlee JS. The `include` and `exclude` arguments of `SitemapRequestLoader` follow the same matching rules.
+
+### Globs are case-insensitive
+
+A `Glob` matches URLs regardless of casing, so `Glob('https://example.com/products/**')` also matches `https://example.com/Products/ABC`. In v1, the matching was case-sensitive. For case-sensitive matching, use a regular expression instead.
+
+### Regular expressions match anywhere in the URL
+
+A regular expression in `include` or `exclude` matches if it's found anywhere in the URL, the same as `re.search`. In v1, it had to match at the start of the URL, the same as `re.match`. For example, `re.compile(r'/products/')` now matches `https://example.com/products/1`, where v1 needed `re.compile(r'https://example\.com/products/')`. To match only at the start of the URL again, start the pattern with `^`, for example `re.compile(r'^https://example\.com/products/')`.
+
+### `limit` is applied last
+
+The `limit` argument counts only the requests that pass all the filters. It's applied after the strategy, `include` and `exclude` filters, the `transform_request_function` and the `max_crawl_depth` check. In v1, it was applied before the `transform_request_function`, so the requests that `transform_request_function` skipped still counted towards the limit. For example, with `limit=5` and a `transform_request_function` that skips 2 of the first 5 links, v2 enqueues 5 requests, where v1 enqueued 3.
+
 ## Storage data uses camelCase keys
 
 The storage backends now write their JSON data with camelCase keys, the same as Crawlee JS. This covers the metadata, request and record metadata files of `FileSystemStorageBackend`, the request and record metadata JSON stored by `RedisStorageBackend`, and the request JSON stored by `SqlStorageBackend`. For example, `item_count` in a dataset's `__metadata__.json` becomes `itemCount`. The storage metadata documents of `RedisStorageBackend` keep their snake_case keys.

src/crawlee/_types.py

@@ -148,7 +148,8 @@ class EnqueueLinksKwargs(TypedDict):
     """Keyword arguments for the `enqueue_links` methods."""
 
     limit: NotRequired[int]
-    """Maximum number of requests to be enqueued."""
+    """Maximum number of requests to be enqueued. It's applied last, after the URL filters and the
+    `transform_request_function`, so the requests they skip don't count towards it."""
 
     base_url: NotRequired[str]
     """Base URL to be used for relative URLs."""
@@ -171,10 +172,12 @@ class EnqueueLinksKwargs(TypedDict):
     """
 
     include: NotRequired[Sequence[re.Pattern | Glob]]
-    """List of regular expressions or globs that URLs must match to be enqueued."""
+    """List of regular expressions or globs that URLs must match to be enqueued. A regular expression may match
+    anywhere in the URL, like `re.search`. A glob must match the whole URL and is case-insensitive."""
 
     exclude: NotRequired[Sequence[re.Pattern | Glob]]
-    """List of regular expressions or globs that URLs must not match to be enqueued."""
+    """List of regular expressions or globs that URLs must not match to be enqueued. A regular expression may match
+    anywhere in the URL, like `re.search`. A glob must match the whole URL and is case-insensitive."""
 
 
 class AddRequestsKwargs(EnqueueLinksKwargs):

src/crawlee/_utils/globs.py

@@ -2,18 +2,47 @@
 
 import os
 import re
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
 
 class Glob:
-    """Wraps a glob pattern (supports the `*`, `**`, `?` wildcards)."""
+    """Wraps a glob pattern (supports the `*`, `**`, `?` wildcards).
+
+    The pattern must match the whole URL, and the matching is case-insensitive. For case-sensitive matching, use a
+    regular expression.
+    """
 
     def __init__(self, glob: str) -> None:
         self.glob = glob
-        self.regexp = re.compile(_translate(self.glob, recursive=True))
+        self.regexp = re.compile(_translate(self.glob, recursive=True), re.IGNORECASE)
+
+
+def matches_url_pattern(pattern: re.Pattern[Any] | Glob, url: str) -> bool:
+    """Check whether `url` matches an include or exclude pattern.
+
+    A glob must match the whole URL, while a regular expression may match anywhere in it, the same as in Crawlee JS.
+    """
+    if isinstance(pattern, Glob):
+        return pattern.regexp.match(url) is not None
+    return pattern.search(url) is not None
+
+
+def check_url_patterns(
+    url: str,
+    include: Sequence[re.Pattern[Any] | Glob] | None,
+    exclude: Sequence[re.Pattern[Any] | Glob] | None,
+) -> bool:
+    """Check whether `url` matches no `exclude` pattern and, when `include` is given, at least one `include` pattern."""
+    if any(matches_url_pattern(pattern, url) for pattern in exclude or ()):
+        return False
+
+    if include is None:
+        return True
+
+    return any(matches_url_pattern(pattern, url) for pattern in include)
 
 
 def _translate(

src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -213,6 +213,7 @@ async def extract_links(
 
             kwargs.setdefault('strategy', 'same-hostname')
             strategy = kwargs.get('strategy', 'same-hostname')
+            limit = kwargs.get('limit')
 
             links_iterator: Iterator[str] = iter(
                 self._parser.find_links(parsed_content, selector=selector, attribute=attribute)
@@ -239,6 +240,9 @@ async def extract_links(
                 skipped = iter([])
 
             for url in self._enqueue_links_filter_iterator(links_iterator, context.request.url, **kwargs):
+                if limit is not None and len(requests) >= limit:
+                    break
+
                 request_options = RequestOptions(
                     url=url, user_data={**base_user_data}, label=label, enqueue_strategy=strategy
                 )

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -24,7 +24,7 @@
 from typing_extensions import TypeVar
 from yarl import URL
 
-from crawlee import Glob, RequestTransformAction, service_locator
+from crawlee import RequestTransformAction, service_locator
 from crawlee._log_config import configure_logger, string_to_log_level
 from crawlee._request import Request, RequestOptions, RequestState
 from crawlee._service_locator import ServiceLocator
@@ -44,6 +44,7 @@
 )
 from crawlee._utils.docs import docs_group
 from crawlee._utils.file import atomic_write, export_csv_to_stream, export_json_to_stream
+from crawlee._utils.globs import check_url_patterns
 from crawlee._utils.http import parse_retry_after_header
 from crawlee._utils.log import LoggerOnce
 from crawlee._utils.recurring_task import RecurringTask
@@ -80,7 +81,6 @@
 )
 
 if TYPE_CHECKING:
-    import re
     from collections.abc import Iterator, Mapping, MutableMapping
     from contextlib import AbstractAsyncContextManager
 
@@ -1165,7 +1165,6 @@ def _enqueue_links_filter_iterator(
         self, request_iterator: Iterator[TRequestIterator], origin_url: str, **kwargs: Unpack[EnqueueLinksKwargs]
     ) -> Iterator[TRequestIterator]:
         """Filter requests based on the enqueue strategy and URL patterns."""
-        limit = kwargs.get('limit')
         parsed_origin_url = URL(origin_url)
         strategy = kwargs.get('strategy', 'all')
 
@@ -1199,44 +1198,9 @@ def _enqueue_links_filter_iterator(
                     host_warned = True
                 continue
 
-            if self._check_url_patterns(target_url, kwargs.get('include'), kwargs.get('exclude')):
+            if check_url_patterns(target_url, kwargs.get('include'), kwargs.get('exclude')):
                 yield request
 
-                if limit is not None:
-                    limit -= 1
-                    if limit <= 0:
-                        break
-
-    def _check_url_patterns(
-        self,
-        target_url: str,
-        include: Sequence[re.Pattern[Any] | Glob] | None,
-        exclude: Sequence[re.Pattern[Any] | Glob] | None,
-    ) -> bool:
-        """Check if a URL matches configured include/exclude patterns."""
-        # If the URL matches any `exclude` pattern, reject it
-        for pattern in exclude or ():
-            if isinstance(pattern, Glob):
-                pattern = pattern.regexp  # noqa: PLW2901
-
-            if pattern.match(target_url) is not None:
-                return False
-
-        # If there are no `include` patterns and the URL passed all `exclude` patterns, accept the URL
-        if include is None:
-            return True
-
-        # If the URL matches any `include` pattern, accept it
-        for pattern in include:
-            if isinstance(pattern, Glob):
-                pattern = pattern.regexp  # noqa: PLW2901
-
-            if pattern.match(target_url) is not None:
-                return True
-
-        # The URL does not match any `include` pattern - reject it
-        return False
-
     async def _handle_error_handler_replacement(
         self,
         context: TCrawlingContext | BasicCrawlingContext,
@@ -1463,11 +1427,15 @@ async def _add_requests(
         else:
             request_manager = await self.get_request_manager()
 
+        limit = kwargs.get('limit')
         context_aware_requests = list[Request]()
         base_url = kwargs.get('base_url') or context.request.loaded_url or context.request.url
         requests_iterator = self._convert_url_to_request_iterator(requests, base_url)
         filter_requests_iterator = self._enqueue_links_filter_iterator(requests_iterator, context.request.url, **kwargs)
         for dst_request in filter_requests_iterator:
+            if limit is not None and len(context_aware_requests) >= limit:
+                break
+
             # Update the crawl depth of the request.
             dst_request.crawl_depth = context.request.crawl_depth + 1
 

src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -443,6 +443,7 @@ async def extract_links(
 
             kwargs.setdefault('strategy', 'same-hostname')
             strategy = kwargs.get('strategy', 'same-hostname')
+            limit = kwargs.get('limit')
 
             elements = await context.page.query_selector_all(selector)
             # Browsers ignore the whitespace around a URL, as `find_links` of the HTTP parsers does.
@@ -465,6 +466,9 @@ async def extract_links(
                 skipped = iter([])
 
             for url in self._enqueue_links_filter_iterator(links_iterator, context.request.url, **kwargs):
+                if limit is not None and len(requests) >= limit:
+                    break
+
                 request_options = RequestOptions(
                     url=url, user_data={**base_user_data}, label=label, enqueue_strategy=strategy
                 )

src/crawlee/request_loaders/_sitemap_request_loader.py

@@ -13,7 +13,7 @@
 
 from crawlee import Request, RequestOptions
 from crawlee._utils.docs import docs_group
-from crawlee._utils.globs import Glob
+from crawlee._utils.globs import Glob, check_url_patterns
 from crawlee._utils.recoverable_state import RecoverableState
 from crawlee._utils.sitemap import (
     DEFAULT_MAX_DEPTH,
@@ -30,7 +30,7 @@
 
 if TYPE_CHECKING:
     import re
-    from collections.abc import Callable, Sequence
+    from collections.abc import Callable
     from types import TracebackType
 
     from crawlee import RequestTransformAction
@@ -139,8 +139,9 @@ def __init__(
         Args:
             sitemap_urls: List of sitemap URLs to fetch and parse for request URLs.
             proxy_info: Optional proxy to use for fetching sitemaps.
-            include: List of glob or regex patterns to include URLs.
-            exclude: List of glob or regex patterns to exclude URLs.
+            include: List of glob or regex patterns to include URLs. A regex may match anywhere in the URL, like
+                `re.search`. A glob must match the whole URL and is case-insensitive.
+            exclude: List of glob or regex patterns to exclude URLs, matched the same way as `include`.
             enqueue_strategy: Strategy used to decide which sitemap-derived URLs (both nested-sitemap entries and
                 URL entries) are kept relative to the parent sitemap URL. Defaults to `'same-hostname'`, matching
                 the sitemap protocol's same-host expectation and the `enqueue_links` default; pass `'all'` to
@@ -326,36 +327,6 @@ def _passes_filters(self, target: str, parent: URL, kind: str) -> bool:
             return False
         return True
 
-    def _check_url_patterns(
-        self,
-        target_url: str,
-        include: Sequence[re.Pattern[Any] | Glob] | None,
-        exclude: Sequence[re.Pattern[Any] | Glob] | None,
-    ) -> bool:
-        """Check if a URL matches configured include/exclude patterns."""
-        # If the URL matches any `exclude` pattern, reject it
-        for pattern in exclude or ():
-            if isinstance(pattern, Glob):
-                pattern = pattern.regexp  # noqa: PLW2901
-
-            if pattern.match(target_url) is not None:
-                return False
-
-        # If there are no `include` patterns and the URL passed all `exclude` patterns, accept the URL
-        if include is None:
-            return True
-
-        # If the URL matches any `include` pattern, accept it
-        for pattern in include:
-            if isinstance(pattern, Glob):
-                pattern = pattern.regexp  # noqa: PLW2901
-
-            if pattern.match(target_url) is not None:
-                return True
-
-        # The URL does not match any `include` pattern - reject it
-        return False
-
     async def _load_sitemaps(self) -> None:
         """Load URLs from sitemaps in the background."""
         try:
@@ -420,7 +391,7 @@ async def _load_sitemaps(self) -> None:
                                     continue
 
                                 # Check if URL should be included
-                                if not self._check_url_patterns(url, self._include, self._exclude):
+                                if not check_url_patterns(url, self._include, self._exclude):
                                     continue
 
                                 if not self._passes_filters(url, parsed_sitemap_url, 'sitemap URL'):

tests/unit/_utils/test_globs.py

@@ -1,6 +1,10 @@
 from __future__ import annotations
 
-from crawlee._utils.globs import Glob
+import re
+
+import pytest
+
+from crawlee._utils.globs import Glob, matches_url_pattern
 
 
 def test_asterisk() -> None:
@@ -15,3 +19,33 @@ def test_double_asteritsk() -> None:
     assert glob.regexp.match('bar/') is None
     assert glob.regexp.match('foo/bar') is not None
     assert glob.regexp.match('foo/bar/baz') is not None
+
+
+@pytest.mark.parametrize(
+    'url',
+    [
+        pytest.param('https://example.com/products/abc', id='same casing'),
+        pytest.param('https://example.com/Products/ABC', id='uppercase path'),
+        pytest.param('HTTPS://EXAMPLE.COM/PRODUCTS/ABC', id='uppercase url'),
+    ],
+)
+def test_glob_is_case_insensitive(url: str) -> None:
+    """A glob matches a URL regardless of its casing."""
+    assert matches_url_pattern(Glob('https://example.com/products/*'), url)
+
+
+@pytest.mark.parametrize(
+    ('pattern', 'expected'),
+    [
+        pytest.param(re.compile(r'/products/'), True, id='regex in the middle'),
+        pytest.param(re.compile(r'\d+$'), True, id='regex at the end'),
+        pytest.param(re.compile(r'^/products/'), False, id='regex anchored at the start'),
+        pytest.param(re.compile(r'/Products/'), False, id='regex is case-sensitive'),
+        pytest.param(Glob('/products/*'), False, id='glob must match the whole url'),
+        pytest.param(Glob('https://example.com/products'), False, id='glob prefix of the url'),
+        pytest.param(Glob('https://example.com/products/*'), True, id='glob matching the whole url'),
+    ],
+)
+def test_matches_url_pattern(pattern: re.Pattern[str] | Glob, *, expected: bool) -> None:
+    """A regex may match anywhere in the URL, while a glob must match the whole URL."""
+    assert matches_url_pattern(pattern, 'https://example.com/products/123') is expected

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -903,6 +903,36 @@ class AddRequestsTestInput:
             ),
             id='include_exclude_3',
         ),
+        pytest.param(
+            AddRequestsTestInput(
+                start_url=INCLUDE_TEST_URLS[0],
+                loaded_url=INCLUDE_TEST_URLS[0],
+                requests=INCLUDE_TEST_URLS,
+                kwargs=EnqueueLinksKwargs(include=[re.compile(r'/cats$')], exclude=[re.compile(r'/archive/')]),
+                expected_urls=[INCLUDE_TEST_URLS[1]],
+            ),
+            id='include_exclude_regex_matches_anywhere',
+        ),
+        pytest.param(
+            AddRequestsTestInput(
+                start_url=INCLUDE_TEST_URLS[0],
+                loaded_url=INCLUDE_TEST_URLS[0],
+                requests=INCLUDE_TEST_URLS,
+                kwargs=EnqueueLinksKwargs(include=[Glob('https://SomePlace.com/**/CATS')]),
+                expected_urls=[INCLUDE_TEST_URLS[1], INCLUDE_TEST_URLS[4]],
+            ),
+            id='include_glob_case_insensitive',
+        ),
+        pytest.param(
+            AddRequestsTestInput(
+                start_url=INCLUDE_TEST_URLS[0],
+                loaded_url=INCLUDE_TEST_URLS[0],
+                requests=INCLUDE_TEST_URLS[1:],
+                kwargs=EnqueueLinksKwargs(exclude=[Glob('https://**/category/**')], limit=1),
+                expected_urls=[INCLUDE_TEST_URLS[3]],
+            ),
+            id='limit_counts_requests_passing_filters',
+        ),
     ],
 )
 async def test_enqueue_strategy(test_input: AddRequestsTestInput) -> None: