From 39a0598a6611f37b04081a0f8c18f88c3691cd14 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 30 Oct 2026 16:02:44 +0100 Subject: [PATCH 1/2] fix!: keep URL path, query and fragment casing in the default `unique_key` --- docs/upgrading/upgrading_to_v2.md | 6 ++++ src/crawlee/_utils/requests.py | 7 ++-- tests/unit/_utils/test_requests.py | 54 ++++++++++++++++++++---------- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 41831b31e4..f7e0bd036b 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -123,6 +123,12 @@ Unlike tiers, this function doesn't track blocking per domain, so every request The `Request.forefront` property is removed. Proxy tiers were the only code that set it. The crawler now reclaims a failed request for a retry at the back of the queue, and a request deferred by a `RequestThrottledError` at the front. To enqueue a request at the front, pass `forefront=True` to `add_requests` or `reclaim_request`. +## Unique keys keep the URL casing + +The default `Request.unique_key` lowercases only the scheme and host of the URL. The path, query and fragment keep their casing, since they're case-sensitive. In v1, the whole URL was lowercased, so URLs that differed only in casing, such as `https://example.com/Product/ABC` and `https://example.com/product/abc`, got the same unique key, and the crawler visited only one of them. + +As a result, a crawl can visit more pages than in v1. Requests stored in a request queue by v1 keep their lowercased unique keys, so adding the same URL again in v2 can create a duplicate request. To keep case-insensitive deduplication, pass an explicit `unique_key`. With `enqueue_links`, set `unique_key` to the lowercased URL in its `transform_request_function`, and leave `url` as it is, so the crawler still fetches the original casing. + ## Storage data uses camelCase keys The storage backends now write their JSON data with camelCase keys, the same as Crawlee JS. This covers the metadata, request and record metadata files of `FileSystemStorageBackend`, the request and record metadata JSON stored by `RedisStorageBackend`, and the request JSON stored by `SqlStorageBackend`. For example, `item_count` in a dataset's `__metadata__.json` becomes `itemCount`. The storage metadata documents of `RedisStorageBackend` keep their snake_case keys. diff --git a/src/crawlee/_utils/requests.py b/src/crawlee/_utils/requests.py index fa31d4621d..01d8642f23 100644 --- a/src/crawlee/_utils/requests.py +++ b/src/crawlee/_utils/requests.py @@ -17,10 +17,11 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: """Normalize a URL. This function cleans and standardizes a URL by removing leading and trailing whitespaces, - converting the scheme and netloc to lower case, stripping unwanted tracking parameters + converting the scheme and host to lower case, stripping unwanted tracking parameters (specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically, and optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally - identical but differ in trivial ways (such as parameter order or casing) are treated as the same. + identical but differ in trivial ways (such as parameter order or host casing) are treated as the same. + The path, query and fragment keep their casing, since they are case-sensitive (RFC 3986, section 6.2.2.1). Args: url: The URL to be normalized. @@ -44,7 +45,7 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str: yarl_new_url.path.removesuffix('/'), keep_query=True, keep_fragment=keep_url_fragment ) - return str(yarl_new_url).lower() + return str(yarl_new_url) def compute_unique_key( diff --git a/tests/unit/_utils/test_requests.py b/tests/unit/_utils/test_requests.py index 8198909592..d9eefe38ea 100644 --- a/tests/unit/_utils/test_requests.py +++ b/tests/unit/_utils/test_requests.py @@ -9,31 +9,42 @@ @pytest.mark.parametrize( ('url', 'expected_output', 'keep_url_fragment'), [ - ('https://example.com/?utm_source=test&utm_medium=test&key=value', 'https://example.com/?key=value', False), - ( + pytest.param( + 'https://example.com/?utm_source=test&utm_medium=test&key=value', + 'https://example.com/?key=value', + False, + id='remove_utm_params', + ), + pytest.param( 'http://example.com/?key=value&another_key=another_value', 'http://example.com/?another_key=another_value&key=value', False, + id='retain_sort_non_utm_params', + ), + pytest.param( + 'HTTPS://EXAMPLE.COM/?KEY=VALUE', + 'https://example.com/?KEY=VALUE', + False, + id='lowercase_scheme_and_host_only', + ), + pytest.param('', '', False, id='handle_empty_url'), + pytest.param('http://example.com/#fragment', 'http://example.com/#fragment', True, id='retain_fragment'), + pytest.param('http://example.com/#fragment', 'http://example.com', False, id='remove_fragment'), + pytest.param(' https://example.com/ ', 'https://example.com', False, id='trim_whitespace'), + pytest.param('http://example.com/?b=2&a=1', 'http://example.com/?a=1&b=2', False, id='sort_query_params'), + pytest.param( + 'https://example.com/Product/ABC', 'https://example.com/Product/ABC', False, id='preserve_path_casing' + ), + pytest.param( + 'https://example.com/?token=SeCrEt', 'https://example.com/?token=SeCrEt', False, id='preserve_query_casing' + ), + pytest.param( + 'https://example.com/#Fragment', 'https://example.com/#Fragment', True, id='preserve_fragment_casing' ), - ('HTTPS://EXAMPLE.COM/?KEY=VALUE', 'https://example.com/?key=value', False), - ('', '', False), - ('http://example.com/#fragment', 'http://example.com/#fragment', True), - ('http://example.com/#fragment', 'http://example.com', False), - (' https://example.com/ ', 'https://example.com', False), - ('http://example.com/?b=2&a=1', 'http://example.com/?a=1&b=2', False), - ], - ids=[ - 'remove_utm_params', - 'retain_sort_non_utm_params', - 'convert_scheme_netloc_to_lowercase', - 'handle_empty_url', - 'retain_fragment', - 'remove_fragment', - 'trim_whitespace', - 'sort_query_params', ], ) def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: bool) -> None: + """`normalize_url` cleans the URL and lowercases only its scheme and host.""" output = normalize_url(url, keep_url_fragment=keep_url_fragment) assert output == expected_output @@ -45,6 +56,13 @@ def test_compute_unique_key_basic() -> None: assert url == uk_get == uk_post +def test_compute_unique_key_distinguishes_path_casing() -> None: + """URLs that differ only in path casing get distinct unique keys.""" + uk_upper = compute_unique_key('https://example.com/Product/ABC') + uk_lower = compute_unique_key('https://example.com/product/abc') + assert uk_upper != uk_lower + + def test_compute_unique_key_handles_fragments() -> None: url = 'https://crawlee.dev/#fragment' uk_with_fragment = compute_unique_key(url, keep_url_fragment=True) From 9a70beeb692e87c8d3689b50e50e5e38be5e4a0b Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 30 Oct 2026 16:23:30 +0100 Subject: [PATCH 2/2] docs: describe the default `unique_key` normalization in the intro tutorial --- docs/introduction/03_adding_more_urls.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/introduction/03_adding_more_urls.mdx b/docs/introduction/03_adding_more_urls.mdx index bfe2337d07..ce46cccd52 100644 --- a/docs/introduction/03_adding_more_urls.mdx +++ b/docs/introduction/03_adding_more_urls.mdx @@ -88,7 +88,7 @@ When you run the code, you will see the crawler log the **title** of the first p ## Skipping duplicate URLs -Skipping of duplicate URLs is critical, because visiting the same page multiple times would lead to duplicate results. This is automatically handled by the `RequestQueue` which deduplicates requests using their `unique_key`. This `unique_key` is automatically generated from the request's URL by lowercasing the URL, lexically ordering query parameters, removing fragments and a few other tweaks that ensure the queue only includes unique URLs. +Skipping of duplicate URLs is critical, because visiting the same page multiple times would lead to duplicate results. This is automatically handled by the `RequestQueue` which deduplicates requests using their `unique_key`. This `unique_key` is automatically generated from the request's URL by lowercasing its scheme and host, lexically ordering query parameters, removing fragments and a few other tweaks that ensure the queue only includes unique URLs. ## Advanced filtering arguments