fix!: keep URL path, query and fragment casing in the default unique_key

docs/introduction/03_adding_more_urls.mdx

@@ -88,7 +88,7 @@ When you run the code, you will see the crawler log the **title** of the first p
 
 ## Skipping duplicate URLs
 
-Skipping of duplicate URLs is critical, because visiting the same page multiple times would lead to duplicate results. This is automatically handled by the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink> which deduplicates requests using their `unique_key`. This `unique_key` is automatically generated from the request's URL by lowercasing the URL, lexically ordering query parameters, removing fragments and a few other tweaks that ensure the queue only includes unique URLs.
+Skipping of duplicate URLs is critical, because visiting the same page multiple times would lead to duplicate results. This is automatically handled by the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink> which deduplicates requests using their `unique_key`. This `unique_key` is automatically generated from the request's URL by lowercasing its scheme and host, lexically ordering query parameters, removing fragments and a few other tweaks that ensure the queue only includes unique URLs.
 
 ## Advanced filtering arguments
 

docs/upgrading/upgrading_to_v2.md

@@ -123,6 +123,12 @@ Unlike tiers, this function doesn't track blocking per domain, so every request
 
 The `Request.forefront` property is removed. Proxy tiers were the only code that set it. The crawler now reclaims a failed request for a retry at the back of the queue, and a request deferred by a `RequestThrottledError` at the front. To enqueue a request at the front, pass `forefront=True` to `add_requests` or `reclaim_request`.
 
+## Unique keys keep the URL casing
+
+The default `Request.unique_key` lowercases only the scheme and host of the URL. The path, query and fragment keep their casing, since they're case-sensitive. In v1, the whole URL was lowercased, so URLs that differed only in casing, such as `https://example.com/Product/ABC` and `https://example.com/product/abc`, got the same unique key, and the crawler visited only one of them.
+
+As a result, a crawl can visit more pages than in v1. Requests stored in a request queue by v1 keep their lowercased unique keys, so adding the same URL again in v2 can create a duplicate request. To keep case-insensitive deduplication, pass an explicit `unique_key`. With `enqueue_links`, set `unique_key` to the lowercased URL in its `transform_request_function`, and leave `url` as it is, so the crawler still fetches the original casing.
+
 ## Storage data uses camelCase keys
 
 The storage backends now write their JSON data with camelCase keys, the same as Crawlee JS. This covers the metadata, request and record metadata files of `FileSystemStorageBackend`, the request and record metadata JSON stored by `RedisStorageBackend`, and the request JSON stored by `SqlStorageBackend`. For example, `item_count` in a dataset's `__metadata__.json` becomes `itemCount`. The storage metadata documents of `RedisStorageBackend` keep their snake_case keys.

src/crawlee/_utils/requests.py

@@ -17,10 +17,11 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str:
     """Normalize a URL.
 
     This function cleans and standardizes a URL by removing leading and trailing whitespaces,
-    converting the scheme and netloc to lower case, stripping unwanted tracking parameters
+    converting the scheme and host to lower case, stripping unwanted tracking parameters
     (specifically those beginning with 'utm_'), sorting the remaining query parameters alphabetically,
     and optionally retaining the URL fragment. The goal is to ensure that URLs that are functionally
-    identical but differ in trivial ways (such as parameter order or casing) are treated as the same.
+    identical but differ in trivial ways (such as parameter order or host casing) are treated as the same.
+    The path, query and fragment keep their casing, since they are case-sensitive (RFC 3986, section 6.2.2.1).
 
     Args:
         url: The URL to be normalized.
@@ -44,7 +45,7 @@ def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str:
         yarl_new_url.path.removesuffix('/'), keep_query=True, keep_fragment=keep_url_fragment
     )
 
-    return str(yarl_new_url).lower()
+    return str(yarl_new_url)
 
 
 def compute_unique_key(

tests/unit/_utils/test_requests.py

@@ -9,31 +9,42 @@
 @pytest.mark.parametrize(
     ('url', 'expected_output', 'keep_url_fragment'),
     [
-        ('https://example.com/?utm_source=test&utm_medium=test&key=value', 'https://example.com/?key=value', False),
-        (
+        pytest.param(
+            'https://example.com/?utm_source=test&utm_medium=test&key=value',
+            'https://example.com/?key=value',
+            False,
+            id='remove_utm_params',
+        ),
+        pytest.param(
             'http://example.com/?key=value&another_key=another_value',
             'http://example.com/?another_key=another_value&key=value',
             False,
+            id='retain_sort_non_utm_params',
+        ),
+        pytest.param(
+            'HTTPS://EXAMPLE.COM/?KEY=VALUE',
+            'https://example.com/?KEY=VALUE',
+            False,
+            id='lowercase_scheme_and_host_only',
+        ),
+        pytest.param('', '', False, id='handle_empty_url'),
+        pytest.param('http://example.com/#fragment', 'http://example.com/#fragment', True, id='retain_fragment'),
+        pytest.param('http://example.com/#fragment', 'http://example.com', False, id='remove_fragment'),
+        pytest.param('  https://example.com/  ', 'https://example.com', False, id='trim_whitespace'),
+        pytest.param('http://example.com/?b=2&a=1', 'http://example.com/?a=1&b=2', False, id='sort_query_params'),
+        pytest.param(
+            'https://example.com/Product/ABC', 'https://example.com/Product/ABC', False, id='preserve_path_casing'
+        ),
+        pytest.param(
+            'https://example.com/?token=SeCrEt', 'https://example.com/?token=SeCrEt', False, id='preserve_query_casing'
+        ),
+        pytest.param(
+            'https://example.com/#Fragment', 'https://example.com/#Fragment', True, id='preserve_fragment_casing'
         ),
-        ('HTTPS://EXAMPLE.COM/?KEY=VALUE', 'https://example.com/?key=value', False),
-        ('', '', False),
-        ('http://example.com/#fragment', 'http://example.com/#fragment', True),
-        ('http://example.com/#fragment', 'http://example.com', False),
-        ('  https://example.com/  ', 'https://example.com', False),
-        ('http://example.com/?b=2&a=1', 'http://example.com/?a=1&b=2', False),
-    ],
-    ids=[
-        'remove_utm_params',
-        'retain_sort_non_utm_params',
-        'convert_scheme_netloc_to_lowercase',
-        'handle_empty_url',
-        'retain_fragment',
-        'remove_fragment',
-        'trim_whitespace',
-        'sort_query_params',
     ],
 )
 def test_normalize_url(url: str, expected_output: str, *, keep_url_fragment: bool) -> None:
+    """`normalize_url` cleans the URL and lowercases only its scheme and host."""
     output = normalize_url(url, keep_url_fragment=keep_url_fragment)
     assert output == expected_output
 
@@ -45,6 +56,13 @@ def test_compute_unique_key_basic() -> None:
     assert url == uk_get == uk_post
 
 
+def test_compute_unique_key_distinguishes_path_casing() -> None:
+    """URLs that differ only in path casing get distinct unique keys."""
+    uk_upper = compute_unique_key('https://example.com/Product/ABC')
+    uk_lower = compute_unique_key('https://example.com/product/abc')
+    assert uk_upper != uk_lower
+
+
 def test_compute_unique_key_handles_fragments() -> None:
     url = 'https://crawlee.dev/#fragment'
     uk_with_fragment = compute_unique_key(url, keep_url_fragment=True)