From c6c246700cfcefe0984bba348f15cd5f7c035d63 Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Fri, 21 Aug 2026 15:08:46 -0700 Subject: [PATCH 1/3] Branch prep --- sdk/storage/azure-storage-blob/CHANGELOG.md | 6 ++++++ .../azure-storage-blob/azure/storage/blob/_version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azure-storage-blob/CHANGELOG.md b/sdk/storage/azure-storage-blob/CHANGELOG.md index 76fc27d102a0..a499547cd109 100644 --- a/sdk/storage/azure-storage-blob/CHANGELOG.md +++ b/sdk/storage/azure-storage-blob/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 12.30.1 (Unreleased) + +### Bugs Fixed +- Fixed a bug where client-side encryption 2.0 could not detect a rearrangement of otherwise-untampered authenticated regions in blob content. This is now detected and exceptions are thrown. For data recovery purposes, this behavior can be reverted by setting the "AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS" environment variable. +- Fixed a bug in client-side encryption where version downgrades, and other metadata tampering, was only detected at the start of a download. + ## 12.30.0 (2026-06-08) ### Features Added diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_version.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_version.py index f2d1dea9d438..42b819748f88 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_version.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_version.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -VERSION = "12.30.0" +VERSION = "12.30.1" From 14bfcfa164bde99b771afececf06b505fc965784 Mon Sep 17 00:00:00 2001 From: Jacob Lauzon <96087589+jalauzon-msft@users.noreply.github.com> Date: Fri, 14 Aug 2026 12:06:58 -0700 Subject: [PATCH 2/3] [Storage] Detect encryption downgrade attempt during Blob download (#48539) --- .../azure/storage/blob/_download.py | 16 ++++- .../azure/storage/blob/_encryption.py | 59 +++++++++++++++++++ .../azure/storage/blob/aio/_download_async.py | 20 +++++-- .../tests/test_blob_encryption_v2.py | 49 +++++++++++++++ .../tests/test_blob_encryption_v2_async.py | 49 +++++++++++++++ .../azure-storage-blob/tests/test_retry.py | 4 +- .../tests/test_retry_async.py | 6 +- 7 files changed, 191 insertions(+), 12 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py index a2f50ebc91ec..a8d0dc5318a9 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py @@ -60,7 +60,13 @@ def process_range_and_offset( return (start_range, end_range), (start_offset, end_offset) -def process_content(data: Any, start_offset: int, end_offset: int, encryption: Dict[str, Any]) -> bytes: +def process_content( + data: Any, + start_offset: int, + end_offset: int, + encryption: Dict[str, Any], + expected_encryption_data: Optional["_EncryptionData"], +) -> bytes: if data is None: raise ValueError("Response cannot be None.") @@ -76,6 +82,7 @@ def process_content(data: Any, start_offset: int, end_offset: int, encryption: D start_offset, end_offset, data.response.headers, + expected_encryption_data, ) except Exception as error: raise HttpResponseError(message="Decryption failed.", response=data.response, error=error) from error @@ -234,7 +241,9 @@ def _download_chunk(self, chunk_start: int, chunk_end: int) -> Tuple[bytes, int] process_storage_error(error) try: - chunk_data = process_content(response, offset[0], offset[1], self.encryption_options) + chunk_data = process_content( + response, offset[0], offset[1], self.encryption_options, self.encryption_data + ) retry_active = False except (IncompleteReadError, HttpResponseError, DecodeError, ServiceResponseError) as error: retry_total -= 1 @@ -520,7 +529,8 @@ def _initial_request(self): response, self._initial_offset[0], self._initial_offset[1], - self._encryption_options + self._encryption_options, + self._encryption_data, ) retry_active = False except (IncompleteReadError, HttpResponseError, DecodeError, ServiceResponseError) as error: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py index 2153d1da1da6..31e6d189c0b4 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py @@ -52,6 +52,11 @@ _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION = ( "The require_encryption flag is set, but encryption is not supported for this method." ) +_ERROR_ENCRYPTION_METADATA_MISMATCH = ( + "The encryption metadata in the download response does not match the encryption metadata " + "retrieved at the start of the download. The blob's encryption metadata may have been modified " + "while the download was in progress." +) class KeyEncryptionKey(Protocol): @@ -216,6 +221,49 @@ def __init__( self.wrapped_content_key = wrapped_content_key self.key_wrapping_metadata = key_wrapping_metadata + def matches(self, other: "_EncryptionData") -> bool: + """ + Determines whether this encryption data refers to the same encrypted content by comparing + every field that affects decryption. This is used to detect whether a blob's encryption + metadata has changed partway through a download, which could indicate the blob was + overwritten or tampered with. + + :param _EncryptionData other: The encryption data to compare against. + :return: True if the decryption-relevant metadata matches, False otherwise. + :rtype: bool + """ + if ( + self.encryption_agent.protocol != other.encryption_agent.protocol + or self.encryption_agent.encryption_algorithm != other.encryption_agent.encryption_algorithm + ): + return False + + if ( + self.wrapped_content_key.key_id != other.wrapped_content_key.key_id + or self.wrapped_content_key.algorithm != other.wrapped_content_key.algorithm + or self.wrapped_content_key.encrypted_key != other.wrapped_content_key.encrypted_key + ): + return False + + # Compare the content encryption IV (used for AES-CBC / V1). + if self.content_encryption_IV != other.content_encryption_IV: + return False + + # Compare the encrypted region info (used for AES-GCM / V2). + self_region = self.encrypted_region_info + other_region = other.encrypted_region_info + if (self_region is None) != (other_region is None): + return False + if self_region is not None and other_region is not None: + if ( + self_region.data_length != other_region.data_length + or self_region.nonce_length != other_region.nonce_length + or self_region.tag_length != other_region.tag_length + ): + return False + + return True + class GCMBlobEncryptionStream: """ @@ -847,6 +895,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements start_offset: int, end_offset: int, response_headers: Dict[str, Any], + expected_encryption_data: Optional[_EncryptionData] = None, ) -> bytes: """ Decrypts the given blob contents and returns only the requested range. @@ -874,12 +923,18 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements :param Dict[str, Any] response_headers: A dictionary of response headers from the download request. Expected to include the 'x-ms-meta-encryptiondata' header if the blob was encrypted. + :param Optional[_EncryptionData] expected_encryption_data: + The encryption data retrieved at the start of the download. If provided, the encryption + metadata on this response is validated against it to detect the blob's encryption metadata + being modified (tampered with) partway through a download. :return: The decrypted blob content. :rtype: bytes """ try: encryption_data = _dict_to_encryption_data(loads(response_headers["x-ms-meta-encryptiondata"])) except Exception as exc: # pylint: disable=broad-except + if expected_encryption_data is not None: + raise ValueError(_ERROR_ENCRYPTION_METADATA_MISMATCH) from exc if require_encryption: raise ValueError( "Encryption required, but received data does not contain appropriate metadata." @@ -888,6 +943,10 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements return content + # Validate that the encryption metadata has not changed since the start of the download. + if expected_encryption_data is not None and not expected_encryption_data.matches(encryption_data): + raise ValueError(_ERROR_ENCRYPTION_METADATA_MISMATCH) + algorithm = encryption_data.encryption_agent.encryption_algorithm if algorithm not in (_EncryptionAlgorithm.AES_CBC_256, _EncryptionAlgorithm.AES_GCM_256): raise ValueError("Specified encryption algorithm is not supported.") diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py index 30cbb0c68fbf..a7634fe7ab4b 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py @@ -30,7 +30,7 @@ adjust_blob_size_for_encryption, decrypt_blob, is_encryption_v2, - parse_encryption_data + parse_encryption_data, ) if TYPE_CHECKING: @@ -44,7 +44,13 @@ T = TypeVar('T', bytes, str) -async def process_content(data: Any, start_offset: int, end_offset: int, encryption: Dict[str, Any]) -> bytes: +async def process_content( + data: Any, + start_offset: int, + end_offset: int, + encryption: Dict[str, Any], + expected_encryption_data: Optional["_EncryptionData"], +) -> bytes: if data is None: raise ValueError("Response cannot be None.") if hasattr(data.response, "is_stream_consumed") and data.response.is_stream_consumed: @@ -60,7 +66,8 @@ async def process_content(data: Any, start_offset: int, end_offset: int, encrypt content, start_offset, end_offset, - data.response.headers + data.response.headers, + expected_encryption_data, ) except Exception as error: raise HttpResponseError( @@ -143,7 +150,9 @@ async def _download_chunk(self, chunk_start: int, chunk_end: int) -> Tuple[bytes process_storage_error(error) try: - chunk_data = await process_content(response, offset[0], offset[1], self.encryption_options) + chunk_data = await process_content( + response, offset[0], offset[1], self.encryption_options, self.encryption_data + ) retry_active = False except (IncompleteReadError, HttpResponseError, DecodeError, ServiceResponseError) as error: retry_total -= 1 @@ -430,7 +439,8 @@ async def _initial_request(self): response, self._initial_offset[0], self._initial_offset[1], - self._encryption_options + self._encryption_options, + self._encryption_data, ) retry_active = False except (IncompleteReadError, HttpResponseError, DecodeError, ServiceResponseError) as error: diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py index f09f212ad067..64d2534c5f31 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py @@ -385,6 +385,55 @@ def test_encryption_v2_v1_downgrade(self, **kwargs): assert 'Decryption failed.' in str(e.value) + @pytest.mark.live_test_only + @BlobPreparer() + def test_encryption_v2_v1_downgrade_mid_download(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + kek = KeyWrapper("key1") + bsc = BlobServiceClient( + self.account_url(storage_account_name, "blob"), + credential=storage_account_key.secret, + max_single_get_size=4 * MiB, + max_chunk_get_size=4 * MiB, + require_encryption=True, + encryption_version="2.0", + key_encryption_key=kek, + ) + + blob = bsc.get_blob_client(self.container_name, self._get_blob_reference()) + content = b"abcd" * 2 * MiB # 8 MiB spans multiple encryption regions -> multiple download requests + + blob.upload_blob(content, overwrite=True) + + # Build tampered metadata that downgrades the blob from V2 to V1 + metadata = blob.get_blob_properties().metadata + tampered = loads(metadata["encryptiondata"]) + tampered["EncryptionAgent"]["Protocol"] = "1.0" + tampered["EncryptionAgent"]["EncryptionAlgorithm"] = "AES_CBC_256" + tampered["ContentEncryptionIV"] = base64.b64encode(os.urandom(16)).decode("utf-8") + tampered_header = dumps(tampered) + + # Simulate the service returning downgraded (V1) encryption metadata partway through the + # download by tampering with the response headers of every request after the initial one. + from azure.storage.blob import _download as download_module + + real_process_content = download_module.process_content + call_count = {"value": 0} + + def tampering_process_content(data, start_offset, end_offset, encryption, expected_encryption_data): + call_count["value"] += 1 + if call_count["value"] > 1: + data.response.headers["x-ms-meta-encryptiondata"] = tampered_header + return real_process_content(data, start_offset, end_offset, encryption, expected_encryption_data) + + # Act / Assert + with mock.patch.object(download_module, "process_content", tampering_process_content): + with pytest.raises(HttpResponseError) as e: + blob.download_blob().readall() + @BlobPreparer() @recorded_by_proxy @mock.patch('os.urandom', mock_urandom) diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py index e74482f65e26..718bd03e148b 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py @@ -392,6 +392,55 @@ async def test_encryption_v2_v1_downgrade(self, **kwargs): assert 'Decryption failed.' in str(e.value) + @pytest.mark.live_test_only + @BlobPreparer() + async def test_encryption_v2_v1_downgrade_mid_download(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + kek = KeyWrapper("key1") + bsc = BlobServiceClient( + self.account_url(storage_account_name, "blob"), + credential=storage_account_key.secret, + max_single_get_size=4 * MiB, + max_chunk_get_size=4 * MiB, + require_encryption=True, + encryption_version="2.0", + key_encryption_key=kek, + ) + + blob = bsc.get_blob_client(self.container_name, self._get_blob_reference()) + content = b"abcd" * 2 * MiB # 8 MiB spans multiple encryption regions -> multiple download requests + + await blob.upload_blob(content, overwrite=True) + + # Build tampered metadata that downgrades the blob from V2 to V1 + metadata = (await blob.get_blob_properties()).metadata + tampered = loads(metadata["encryptiondata"]) + tampered["EncryptionAgent"]["Protocol"] = "1.0" + tampered["EncryptionAgent"]["EncryptionAlgorithm"] = "AES_CBC_256" + tampered["ContentEncryptionIV"] = base64.b64encode(os.urandom(16)).decode("utf-8") + tampered_header = dumps(tampered) + + # Simulate the service returning downgraded (V1) encryption metadata partway through the + # download by tampering with the response headers of every request after the initial one. + from azure.storage.blob.aio import _download_async as download_module + + real_process_content = download_module.process_content + call_count = {"value": 0} + + async def tampering_process_content(data, start_offset, end_offset, encryption, expected_encryption_data): + call_count["value"] += 1 + if call_count["value"] > 1: + data.response.headers["x-ms-meta-encryptiondata"] = tampered_header + return await real_process_content(data, start_offset, end_offset, encryption, expected_encryption_data) + + # Act / Assert + with mock.patch.object(download_module, "process_content", tampering_process_content): + with pytest.raises(HttpResponseError) as e: + await (await blob.download_blob()).readall() + @BlobPreparer() @recorded_by_proxy_async async def test_encryption_modify_cek(self, **kwargs): diff --git a/sdk/storage/azure-storage-blob/tests/test_retry.py b/sdk/storage/azure-storage-blob/tests/test_retry.py index 68bbce9db121..8d16b77445e6 100644 --- a/sdk/storage/azure-storage-blob/tests/test_retry.py +++ b/sdk/storage/azure-storage-blob/tests/test_retry.py @@ -666,14 +666,14 @@ def test_retry_on_service_response_error(self, **kwargs): # Mock the internal response to raise ServiceResponseError on first chunk processing from azure.storage.blob._download import process_content as real_process_content - def mock_process_content_with_error(response, start_offset, end_offset, encryption): + def mock_process_content_with_error(response, start_offset, end_offset, encryption, expected_encryption_data): retry_counter.simple_count(retry) conn_error = AzureError("Connection reset by peer") if retry_counter.count == 1: raise ServiceResponseError(conn_error, error=conn_error) if retry_counter.count == 2: raise ServiceResponseTimeoutError(conn_error, error=conn_error) - return real_process_content(response, start_offset, end_offset, encryption) + return real_process_content(response, start_offset, end_offset, encryption, expected_encryption_data) # Act try: diff --git a/sdk/storage/azure-storage-blob/tests/test_retry_async.py b/sdk/storage/azure-storage-blob/tests/test_retry_async.py index faeced96d1ae..669681432f17 100644 --- a/sdk/storage/azure-storage-blob/tests/test_retry_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_retry_async.py @@ -644,14 +644,16 @@ async def test_retry_on_service_response_error(self, **kwargs): # Mock the internal response to raise ServiceResponseError on first chunk processing from azure.storage.blob.aio._download_async import process_content as real_process_content - async def mock_process_content_with_error(response, start_offset, end_offset, encryption): + async def mock_process_content_with_error( + response, start_offset, end_offset, encryption, expected_encryption_data + ): retry_counter.simple_count(retry) conn_error = AzureError("Connection reset by peer") if retry_counter.count == 1: raise ServiceResponseError(conn_error, error=conn_error) if retry_counter.count == 2: raise ServiceResponseTimeoutError(conn_error, error=conn_error) - return await real_process_content(response, start_offset, end_offset, encryption) + return await real_process_content(response, start_offset, end_offset, encryption, expected_encryption_data) # Act try: From 8f04a89f532c2e968cb2d266309cc0f40411e4cc Mon Sep 17 00:00:00 2001 From: Jacob Lauzon <96087589+jalauzon-msft@users.noreply.github.com> Date: Fri, 21 Aug 2026 11:51:08 -0700 Subject: [PATCH 3/3] [Storage] Detect region reorder during Blob decryption (#48559) --- .../azure/storage/blob/_download.py | 6 +- .../azure/storage/blob/_encryption.py | 136 ++++++++++- .../azure/storage/blob/aio/_download_async.py | 4 + .../tests/test_blob_encryption_v2.py | 217 ++++++++++++++++- .../tests/test_blob_encryption_v2_async.py | 41 +++- .../azure/storage/queue/_encryption.py | 229 +++++++++++++++--- 6 files changed, 578 insertions(+), 55 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py index a8d0dc5318a9..8ce2d813b432 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py @@ -27,7 +27,8 @@ decrypt_blob, get_adjusted_download_range_and_offset, is_encryption_v2, - parse_encryption_data + parse_encryption_data, + _GCMRegionNonceValidator, ) if TYPE_CHECKING: @@ -83,6 +84,7 @@ def process_content( end_offset, data.response.headers, expected_encryption_data, + encryption.get("gcm_nonce_validator"), ) except Exception as error: raise HttpResponseError(message="Decryption failed.", response=data.response, error=error) from error @@ -389,6 +391,8 @@ def __init__( if self._encryption_options.get("key") is not None or self._encryption_options.get("resolver") is not None: self._get_encryption_data_request() + if is_encryption_v2(self._encryption_data): + self._encryption_options["gcm_nonce_validator"] = _GCMRegionNonceValidator() # The service only provides transactional MD5s for chunks under 4MB. # If validate_content is on, get only self.MAX_CHUNK_GET_SIZE for the first diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py index 31e6d189c0b4..5c4832a4b672 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_encryption.py @@ -8,6 +8,7 @@ import math import os import sys +import threading import warnings from collections import OrderedDict from io import BytesIO @@ -58,6 +59,8 @@ "while the download was in progress." ) +_ERROR_INVALID_ENCRYPTION_METADATA = "The encryption metadata is not valid and may have been modified." + class KeyEncryptionKey(Protocol): @@ -713,7 +716,7 @@ def _validate_and_unwrap_cek( version_2_bytes = encryption_data.encryption_agent.protocol.encode().ljust(8, b"\0") cek_version_bytes = content_encryption_key[: len(version_2_bytes)] if cek_version_bytes != version_2_bytes: - raise ValueError("The encryption metadata is not valid and may have been modified.") + raise ValueError(_ERROR_INVALID_ENCRYPTION_METADATA) # Remove version from the start of the cek. content_encryption_key = content_encryption_key[len(version_2_bytes) :] @@ -887,7 +890,96 @@ def generate_blob_encryption_data( return content_encryption_key, initialization_vector, encryption_data -def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements +def _parse_content_range(content_range: str) -> Tuple[int, int, int]: + """ + Parses a Content-Range header of the form 'bytes x-y/size' into its + start, end, and total size components. + + :param str content_range: The Content-Range header value. + :return: A tuple of (start, end, total size). + :rtype: Tuple[int, int, int] + """ + # Format: 'bytes x-y/size' -- ignore the leading 'bytes' word. + byte_range, size = content_range.split(" ")[1].split("/") + start, end = byte_range.split("-") + return int(start), int(end), int(size) + + +def _region_nonce_encodings(nonce_length: int) -> Dict[str, Callable[[int], bytes]]: + """ + Returns the supported per-region nonce encodings, keyed by the SDK that produces them. + + The per-region nonce is a counter of the region's position, but each SDK encodes it + differently, so all supported encodings must be understood for interoperability: + + * Python: zero-based counter, big-endian across the whole nonce (value in trailing bytes). + * Java: zero-based counter, big-endian in the leading 8 bytes, trailing bytes zeroed. + * .NET: one-based counter, little-endian in the trailing 8 bytes, leading bytes zeroed. + + These encodings share the same value space, so they must not be accepted independently + per region (for example Java's nonce for region 1 is identical to .NET's nonce for + region 16,777,215). A single encoding is instead selected and enforced across the whole + download; see ``decrypt_blob``. + + :param int nonce_length: The length of the nonce in bytes. + :return: A mapping of SDK name to a function returning that SDK's nonce for a region index. + :rtype: Dict[str, Callable[[int], bytes]] + """ + encodings: Dict[str, Callable[[int], bytes]] = { + "python": lambda index: index.to_bytes(nonce_length, "big"), + } + + counter_length = 8 + pad = nonce_length - counter_length + encodings["java"] = lambda index: index.to_bytes(counter_length, "big") + b"\x00" * pad + encodings["dotnet"] = lambda index: b"\x00" * pad + (index + 1).to_bytes(counter_length, "little") + + return encodings + + +class _GCMRegionNonceValidator: + """ + Enforces that every region across a whole download uses a single nonce encoding. + + ``decrypt_blob`` runs once per HTTP chunk, so the candidate encodings are shared and + intersected across all chunks (including concurrent ones) rather than reset per call. + Otherwise the encoding could change at a chunk boundary and, at an encoding collision, + let a relocated region pass validation. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._candidates: Optional[Dict[str, Callable[[int], bytes]]] = None + # Set once the candidates collapse to a single encoding; read lock-free thereafter. + self._encoding: Optional[Callable[[int], bytes]] = None + + def validate_region(self, region_index: int, nonce: bytes, nonce_length: int) -> None: + """ + Narrows the shared candidate encodings to those consistent with this region. + + :param int region_index: The zero-based index of the region within the blob. + :param bytes nonce: The nonce read from the region. + :param int nonce_length: The length of the nonce in bytes. + """ + encoding = self._encoding + if encoding is not None: + if encoding(region_index) != nonce: + raise ValueError(_ERROR_INVALID_ENCRYPTION_METADATA) + return + + with self._lock: + if self._candidates is None: + self._candidates = _region_nonce_encodings(nonce_length) + self._candidates = { + name: encode for name, encode in self._candidates.items() if encode(region_index) == nonce + } + if not self._candidates: + raise ValueError(_ERROR_INVALID_ENCRYPTION_METADATA) + if len(self._candidates) == 1: + self._encoding = next(iter(self._candidates.values())) + + +def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements,too-many-branches require_encryption: bool, key_encryption_key: Optional[KeyEncryptionKey], key_resolver: Optional[Callable[[str], KeyEncryptionKey]], @@ -896,6 +988,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements end_offset: int, response_headers: Dict[str, Any], expected_encryption_data: Optional[_EncryptionData] = None, + nonce_validator: Optional["_GCMRegionNonceValidator"] = None, ) -> bytes: """ Decrypts the given blob contents and returns only the requested range. @@ -927,6 +1020,10 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements The encryption data retrieved at the start of the download. If provided, the encryption metadata on this response is validated against it to detect the blob's encryption metadata being modified (tampered with) partway through a download. + :param Optional[_GCMRegionNonceValidator] nonce_validator: + Shared state used to enforce a single V2 nonce encoding across every chunk of a Blob download. + Required for V2 decryption unless the caller has set the + AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS environment variable. :return: The decrypted blob content. :rtype: bytes """ @@ -963,16 +1060,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements iv: Optional[bytes] = None unpad = False if "content-range" in response_headers: - content_range = response_headers["content-range"] - # Format: 'bytes x-y/size' - - # Ignore the word 'bytes' - content_range = content_range.split(" ") - - content_range = content_range[1].split("-") - content_range = content_range[1].split("/") - end_range = int(content_range[0]) - blob_size = int(content_range[1]) + _, end_range, blob_size = _parse_content_range(response_headers["content-range"]) if start_offset >= 16: iv = content[:16] @@ -1016,6 +1104,25 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements tag_length = encryption_data.encrypted_region_info.tag_length region_length = nonce_length + data_length + tag_length + # The per-region nonce is a counter of the region's index within the blob. The + # downloaded content always begins on a region boundary, so derive the first + # region's index from the download range (0 when the whole blob was downloaded). + # This lets us validate each nonce and detect reordered regions. + start_range = 0 + if "content-range" in response_headers: + start_range, _, _ = _parse_content_range(response_headers["content-range"]) + nonce_counter = start_range // region_length + + # Bypass nonce validation via an environment variable for data-recovery scenarios + # where regions were reordered. Not recommended: it can allow tampered data through. + validate_nonce = os.environ.get( + "AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS", "" + ).strip().lower() not in ("true", "1") + + # A validator is required to enforce a single nonce encoding across the whole download. + if validate_nonce and nonce_validator is None: + raise ValueError("A nonce validator is required to decrypt Encryption V2 content.") + decrypted_content = bytearray() while offset < total_size: # Process one encryption region at a time @@ -1024,6 +1131,10 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements # First bytes are the nonce nonce = encrypted_region[:nonce_length] + # Validate the nonce matches the expected counter for this region under a single + # consistent encoding. A mismatch indicates the regions were reordered or tampered with. + if nonce_validator is not None and validate_nonce: + nonce_validator.validate_region(nonce_counter, nonce, nonce_length) ciphertext_with_tag = encrypted_region[nonce_length:] aesgcm = AESGCM(content_encryption_key) @@ -1031,6 +1142,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements decrypted_content.extend(decrypted_data) offset += process_size + nonce_counter += 1 # Read the caller requested data from the decrypted content return decrypted_content[start_offset:end_offset] diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py index a7634fe7ab4b..b69b6b28501a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py @@ -31,6 +31,7 @@ decrypt_blob, is_encryption_v2, parse_encryption_data, + _GCMRegionNonceValidator, ) if TYPE_CHECKING: @@ -68,6 +69,7 @@ async def process_content( end_offset, data.response.headers, expected_encryption_data, + encryption.get("gcm_nonce_validator"), ) except Exception as error: raise HttpResponseError( @@ -325,6 +327,8 @@ async def _get_encryption_data_request(self) -> None: async def _setup(self) -> None: if self._encryption_options.get("key") is not None or self._encryption_options.get("resolver") is not None: await self._get_encryption_data_request() + if is_encryption_v2(self._encryption_data): + self._encryption_options["gcm_nonce_validator"] = _GCMRegionNonceValidator() # The service only provides transactional MD5s for chunks under 4MB. # If validate_content is on, get only self.MAX_CHUNK_GET_SIZE for the first diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py index 64d2534c5f31..d455f5196516 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2.py @@ -22,9 +22,15 @@ from azure.core import MatchConditions from azure.core.exceptions import HttpResponseError, ResourceExistsError -from azure.storage.blob import BlobServiceClient, BlobType, ContentSettings +from azure.storage.blob import BlobBlock, BlobServiceClient, BlobType, ContentSettings from azure.storage.blob._encryption import ( - _dict_to_encryption_data, _GCM_NONCE_LENGTH, _GCM_TAG_LENGTH, _validate_and_unwrap_cek, + _dict_to_encryption_data, + _GCM_NONCE_LENGTH, + _GCM_TAG_LENGTH, + _GCMRegionNonceValidator, + _region_nonce_encodings, + _validate_and_unwrap_cek, + decrypt_blob, ) @@ -467,6 +473,45 @@ def test_encryption_modify_cek(self, **kwargs): assert 'Decryption failed.' in str(e.value) + @pytest.mark.live_test_only + @BlobPreparer() + def test_encryption_reordered_regions(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + self._setup(storage_account_name, storage_account_key) + kek = KeyWrapper("key1") + # Each encrypted region is the plaintext region plus a nonce and tag. Size each + # block to a full encrypted region so every committed block is exactly one region. + region_length = 4 * MiB + _GCM_NONCE_LENGTH + _GCM_TAG_LENGTH + bsc = BlobServiceClient( + self.account_url(storage_account_name, "blob"), + credential=storage_account_key.secret, + max_single_put_size=1024, + max_block_size=region_length, + require_encryption=True, + encryption_version="2.0", + key_encryption_key=kek, + ) + + blob = bsc.get_blob_client(self.container_name, self._get_blob_reference()) + content = b"abcd" * 3 * MiB # 12 MiB -- three full 4 MiB encryption regions + blob.upload_blob(content, overwrite=True) + + # Reorder the committed blocks so the encryption regions are out of order. + plain_blob = self.bsc.get_blob_client(self.container_name, self._get_blob_reference()) + metadata = plain_blob.get_blob_properties().metadata + committed, _ = plain_blob.get_block_list(block_list_type="committed") + reordered = committed[:-2] + committed[-1:] + committed[-2:-1] + reordered = [BlobBlock(block_id=block.id) for block in reordered] + plain_blob.commit_block_list(reordered, metadata=metadata) + + # Act / Assert -- a region's nonce no longer matches its position + with pytest.raises(HttpResponseError) as e: + blob.download_blob().readall() + + assert "Decryption failed." in str(e.value) + @BlobPreparer() @recorded_by_proxy @mock.patch('os.urandom', mock_urandom) @@ -1285,3 +1330,171 @@ def assert_user_agent(request): blob.upload_blob(content, overwrite=True, raw_request_hook=assert_user_agent) blob.download_blob(raw_request_hook=assert_user_agent).readall() + + +class TestGCMRegionNonceValidation: + REGION_DATA_LENGTH = 32 + + @staticmethod + def _encryption_headers(kek, cek, protocol, library, data_length=REGION_DATA_LENGTH): + # Wrap the CEK the way the V2 protocol requires (version prefix padded to 8 bytes). + wrapped_cek = kek.wrap_key(protocol.encode().ljust(8, b"\x00") + cek) + encryption_data = { + "WrappedContentKey": { + "KeyId": kek.get_kid(), + "EncryptedKey": base64.b64encode(wrapped_cek).decode(), + "Algorithm": kek.get_key_wrap_algorithm(), + }, + "EncryptionAgent": {"Protocol": protocol, "EncryptionAlgorithm": "AES_GCM_256"}, + "EncryptedRegionInfo": {"DataLength": data_length, "NonceLength": _GCM_NONCE_LENGTH}, + "KeyWrappingMetadata": {"EncryptionLibrary": library}, + } + return {"x-ms-meta-encryptiondata": dumps(encryption_data)} + + @staticmethod + def _encrypt_regions(cek, nonce_for_region, plaintext_regions): + aesgcm = AESGCM(cek) + return [ + nonce_for_region(i) + aesgcm.encrypt(nonce_for_region(i), region, None) + for i, region in enumerate(plaintext_regions) + ] + + @staticmethod + def _decrypt(kek, headers, content, end_offset, nonce_validator): + return decrypt_blob( + require_encryption=True, + key_encryption_key=kek, + key_resolver=None, + content=content, + start_offset=0, + end_offset=end_offset, + response_headers=headers, + nonce_validator=nonce_validator, + ) + + def test_decrypt_dotnet_v2_1_nonce_encoding(self): + # Regression test for cross-SDK interoperability. The .NET Storage SDK encodes each + # region's GCM nonce as a one-based counter written little-endian into the final 8 + # nonce bytes. Python must still decrypt these .NET-produced V2.1 blobs while + # continuing to detect reordered regions. + kek = KeyWrapper("key1") + cek = os.urandom(32) + num_regions = 3 + plaintext_regions = [bytes([i]) * self.REGION_DATA_LENGTH for i in range(num_regions)] + + def dotnet_nonce(region_index): + # 4 zero bytes + one-based counter, little-endian, 8 bytes -- see .NET + # GcmAuthenticatedCryptographicTransform.GetNewNonce(). + return b"\x00\x00\x00\x00" + (region_index + 1).to_bytes(8, "little") + + encrypted_regions = self._encrypt_regions(cek, dotnet_nonce, plaintext_regions) + headers = self._encryption_headers(kek, cek, "2.1", "Dotnet") + plaintext = b"".join(plaintext_regions) + + # Act / Assert -- the .NET nonce encoding is accepted and the content round-trips. + decrypted = self._decrypt(kek, headers, b"".join(encrypted_regions), len(plaintext), _GCMRegionNonceValidator()) + assert decrypted == plaintext + + # Reordering the .NET-produced regions must still be detected. + reordered = encrypted_regions[0] + encrypted_regions[2] + encrypted_regions[1] + with pytest.raises(ValueError): + self._decrypt(kek, headers, reordered, len(plaintext), _GCMRegionNonceValidator()) + + def test_decrypt_java_v2_nonce_encoding(self): + # Regression test for cross-SDK interoperability. The Java Storage SDK encodes each + # region's GCM nonce as a zero-based counter written big-endian into the leading 8 + # nonce bytes (ByteBuffer.allocate(12).putLong(index)), which differs from Python's + # full-width big-endian counter. Python must still decrypt Java-produced V2 blobs + # while continuing to detect reordered regions. + kek = KeyWrapper("key1") + cek = os.urandom(32) + num_regions = 3 + plaintext_regions = [bytes([i]) * self.REGION_DATA_LENGTH for i in range(num_regions)] + + def java_nonce(region_index): + # Zero-based counter, big-endian, in the leading 8 bytes; trailing bytes zeroed. + return region_index.to_bytes(8, "big") + b"\x00" * (_GCM_NONCE_LENGTH - 8) + + encrypted_regions = self._encrypt_regions(cek, java_nonce, plaintext_regions) + headers = self._encryption_headers(kek, cek, "2.0", "Java") + plaintext = b"".join(plaintext_regions) + + # Act / Assert -- the Java nonce encoding is accepted and the content round-trips. + decrypted = self._decrypt(kek, headers, b"".join(encrypted_regions), len(plaintext), _GCMRegionNonceValidator()) + assert decrypted == plaintext + + # Reordering the Java-produced regions must still be detected. + reordered = encrypted_regions[0] + encrypted_regions[2] + encrypted_regions[1] + with pytest.raises(ValueError): + self._decrypt(kek, headers, reordered, len(plaintext), _GCMRegionNonceValidator()) + + def test_decrypt_rejects_mixed_nonce_encodings(self): + # Regression test: the supported SDK nonce encodings share a value space, so accepting + # them independently per region would weaken reorder detection. For example Java's + # nonce for region 1 is identical to .NET's nonce for region 16,777,215, so a Java + # region could be moved to that position and still pass a per-region union check. + # decrypt_blob must instead select a single encoding and enforce it consistently. + encodings = _region_nonce_encodings(_GCM_NONCE_LENGTH) + # Document the overlap that motivates single-encoding enforcement. + assert encodings["java"](1) == encodings["dotnet"](16_777_215) + + kek = KeyWrapper("key1") + cek = os.urandom(32) + aesgcm = AESGCM(cek) + + # Region 0 uses the Java/Python encoding (all zeros); region 1 uses the .NET encoding. + # A per-region union check would accept both; single-encoding enforcement rejects the mix. + region0_nonce = encodings["java"](0) + region1_nonce = encodings["dotnet"](1) + region0 = region0_nonce + aesgcm.encrypt(region0_nonce, b"\x00" * self.REGION_DATA_LENGTH, None) + region1 = region1_nonce + aesgcm.encrypt(region1_nonce, b"\x11" * self.REGION_DATA_LENGTH, None) + headers = self._encryption_headers(kek, cek, "2.0", "Mixed") + + # Act / Assert -- the mixed encoding is rejected rather than silently accepted. + with pytest.raises(ValueError): + self._decrypt(kek, headers, region0 + region1, 2 * self.REGION_DATA_LENGTH, _GCMRegionNonceValidator()) + + def test_nonce_validator_enforces_single_encoding_across_chunks(self): + # decrypt_blob runs once per download chunk, so a shared validator must intersect the + # candidate encodings across chunks; otherwise the encoding could change at a chunk + # boundary and, at a collision, let a relocated region pass. + encodings = _region_nonce_encodings(_GCM_NONCE_LENGTH) + # The collision that makes per-chunk validation unsound: Java region 1 == .NET region 16,777,215. + assert encodings["java"](1) == encodings["dotnet"](16_777_215) + + validator = _GCMRegionNonceValidator() + # First chunk: two Java regions resolve the encoding to Java. + validator.validate_region(0, encodings["java"](0), _GCM_NONCE_LENGTH) + validator.validate_region(1, encodings["java"](1), _GCM_NONCE_LENGTH) + + # Later chunk: a region relocated to the colliding .NET index carries Java's region-1 + # nonce. Its only consistent encoding is .NET, which conflicts with the resolved Java + # encoding, so the shared validator rejects it. + with pytest.raises(ValueError): + validator.validate_region(16_777_215, encodings["java"](1), _GCM_NONCE_LENGTH) + + def test_env_var_bypasses_nonce_validation(self): + # The AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS escape hatch disables nonce + # validation for data-recovery scenarios. With it set, a mix of nonce encodings that + # would normally be rejected must decrypt, and no validator is required. + encodings = _region_nonce_encodings(_GCM_NONCE_LENGTH) + kek = KeyWrapper("key1") + cek = os.urandom(32) + aesgcm = AESGCM(cek) + + # Two regions using incompatible encodings (Java for region 0, .NET for region 1). + region0_plaintext = b"\x00" * self.REGION_DATA_LENGTH + region1_plaintext = b"\x11" * self.REGION_DATA_LENGTH + region0_nonce = encodings["java"](0) + region1_nonce = encodings["dotnet"](1) + region0 = region0_nonce + aesgcm.encrypt(region0_nonce, region0_plaintext, None) + region1 = region1_nonce + aesgcm.encrypt(region1_nonce, region1_plaintext, None) + headers = self._encryption_headers(kek, cek, "2.0", "Mixed") + plaintext = region0_plaintext + region1_plaintext + + # Act / Assert -- with the bypass set, decryption succeeds without a validator. + with mock.patch.dict(os.environ, {"AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS": "true"}): + decrypted = self._decrypt( + kek, headers, region0 + region1, 2 * self.REGION_DATA_LENGTH, nonce_validator=None + ) + assert decrypted == plaintext diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py index 718bd03e148b..6b1097cb9f77 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_v2_async.py @@ -23,7 +23,7 @@ from azure.core import MatchConditions from azure.core.exceptions import HttpResponseError, ResourceExistsError -from azure.storage.blob import BlobType, ContentSettings +from azure.storage.blob import BlobBlock, BlobType, ContentSettings from azure.storage.blob._encryption import ( _dict_to_encryption_data, _GCM_NONCE_LENGTH, @@ -474,6 +474,45 @@ async def test_encryption_modify_cek(self, **kwargs): assert 'Decryption failed.' in str(e.value) + @pytest.mark.live_test_only + @BlobPreparer() + async def test_encryption_reordered_regions(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + await self._setup(storage_account_name, storage_account_key) + kek = KeyWrapper("key1") + # Each encrypted region is the plaintext region plus a nonce and tag. Size each + # block to a full encrypted region so every committed block is exactly one region. + region_length = 4 * MiB + _GCM_NONCE_LENGTH + _GCM_TAG_LENGTH + bsc = BlobServiceClient( + self.account_url(storage_account_name, "blob"), + credential=storage_account_key.secret, + max_single_put_size=1024, + max_block_size=region_length, + require_encryption=True, + encryption_version="2.0", + key_encryption_key=kek, + ) + + blob = bsc.get_blob_client(self.container_name, self._get_blob_reference()) + content = b"abcd" * 3 * MiB # 12 MiB -- three full 4 MiB encryption regions + await blob.upload_blob(content, overwrite=True) + + # Reorder the committed blocks so the encryption regions are out of order. + plain_blob = self.bsc.get_blob_client(self.container_name, self._get_blob_reference()) + metadata = (await plain_blob.get_blob_properties()).metadata + committed, _ = await plain_blob.get_block_list(block_list_type="committed") + reordered = committed[:-2] + committed[-1:] + committed[-2:-1] + reordered = [BlobBlock(block_id=block.id) for block in reordered] + await plain_blob.commit_block_list(reordered, metadata=metadata) + + # Act / Assert -- a region's nonce no longer matches its position + with pytest.raises(HttpResponseError) as e: + await (await blob.download_blob()).readall() + + assert "Decryption failed." in str(e.value) + @BlobPreparer() @recorded_by_proxy_async async def test_case_insensitive_metadata_key(self, **kwargs): diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_encryption.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_encryption.py index 5d9fcb187987..5c4832a4b672 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_encryption.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_encryption.py @@ -8,6 +8,7 @@ import math import os import sys +import threading import warnings from collections import OrderedDict from io import BytesIO @@ -41,11 +42,7 @@ _ENCRYPTION_PROTOCOL_V1 = "1.0" _ENCRYPTION_PROTOCOL_V2 = "2.0" _ENCRYPTION_PROTOCOL_V2_1 = "2.1" -_VALID_ENCRYPTION_PROTOCOLS = [ - _ENCRYPTION_PROTOCOL_V1, - _ENCRYPTION_PROTOCOL_V2, - _ENCRYPTION_PROTOCOL_V2_1, -] +_VALID_ENCRYPTION_PROTOCOLS = [_ENCRYPTION_PROTOCOL_V1, _ENCRYPTION_PROTOCOL_V2, _ENCRYPTION_PROTOCOL_V2_1] _ENCRYPTION_V2_PROTOCOLS = [_ENCRYPTION_PROTOCOL_V2, _ENCRYPTION_PROTOCOL_V2_1] _GCM_REGION_DATA_LENGTH = 4 * 1024 * 1024 _GCM_NONCE_LENGTH = 12 @@ -56,6 +53,13 @@ _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION = ( "The require_encryption flag is set, but encryption is not supported for this method." ) +_ERROR_ENCRYPTION_METADATA_MISMATCH = ( + "The encryption metadata in the download response does not match the encryption metadata " + "retrieved at the start of the download. The blob's encryption metadata may have been modified " + "while the download was in progress." +) + +_ERROR_INVALID_ENCRYPTION_METADATA = "The encryption metadata is not valid and may have been modified." class KeyEncryptionKey(Protocol): @@ -220,6 +224,49 @@ def __init__( self.wrapped_content_key = wrapped_content_key self.key_wrapping_metadata = key_wrapping_metadata + def matches(self, other: "_EncryptionData") -> bool: + """ + Determines whether this encryption data refers to the same encrypted content by comparing + every field that affects decryption. This is used to detect whether a blob's encryption + metadata has changed partway through a download, which could indicate the blob was + overwritten or tampered with. + + :param _EncryptionData other: The encryption data to compare against. + :return: True if the decryption-relevant metadata matches, False otherwise. + :rtype: bool + """ + if ( + self.encryption_agent.protocol != other.encryption_agent.protocol + or self.encryption_agent.encryption_algorithm != other.encryption_agent.encryption_algorithm + ): + return False + + if ( + self.wrapped_content_key.key_id != other.wrapped_content_key.key_id + or self.wrapped_content_key.algorithm != other.wrapped_content_key.algorithm + or self.wrapped_content_key.encrypted_key != other.wrapped_content_key.encrypted_key + ): + return False + + # Compare the content encryption IV (used for AES-CBC / V1). + if self.content_encryption_IV != other.content_encryption_IV: + return False + + # Compare the encrypted region info (used for AES-GCM / V2). + self_region = self.encrypted_region_info + other_region = other.encrypted_region_info + if (self_region is None) != (other_region is None): + return False + if self_region is not None and other_region is not None: + if ( + self_region.data_length != other_region.data_length + or self_region.nonce_length != other_region.nonce_length + or self_region.tag_length != other_region.tag_length + ): + return False + + return True + class GCMBlobEncryptionStream: """ @@ -312,10 +359,7 @@ def is_encryption_v2(encryption_data: Optional[_EncryptionData]) -> bool: def modify_user_agent_for_encryption( - user_agent: str, - moniker: str, - encryption_version: str, - request_options: Dict[str, Any], + user_agent: str, moniker: str, encryption_version: str, request_options: Dict[str, Any] ) -> None: """ Modifies the request options to contain a user agent string updated with encryption information. @@ -367,10 +411,7 @@ def get_adjusted_upload_size(length: int, encryption_version: str) -> int: def get_adjusted_download_range_and_offset( - start: int, - end: int, - length: Optional[int], - encryption_data: Optional[_EncryptionData], + start: int, end: int, length: Optional[int], encryption_data: Optional[_EncryptionData] ) -> Tuple[Tuple[int, int], Tuple[int, int]]: """ Gets the new download range and offsets into the decrypted data for @@ -589,17 +630,11 @@ def _dict_to_encryption_data(encryption_data_dict: Dict[str, Any]) -> _Encryptio if "EncryptedRegionInfo" in encryption_data_dict: encrypted_region_info = encryption_data_dict["EncryptedRegionInfo"] region_info = _EncryptedRegionInfo( - encrypted_region_info["DataLength"], - encrypted_region_info["NonceLength"], - _GCM_TAG_LENGTH, + encrypted_region_info["DataLength"], encrypted_region_info["NonceLength"], _GCM_TAG_LENGTH ) encryption_data = _EncryptionData( - encryption_iv, - region_info, - encryption_agent, - wrapped_content_key, - key_wrapping_metadata, + encryption_iv, region_info, encryption_agent, wrapped_content_key, key_wrapping_metadata ) return encryption_data @@ -672,8 +707,7 @@ def _validate_and_unwrap_cek( raise ValueError("Provided or resolved key-encryption-key does not match the id of key used to encrypt.") # Will throw an exception if the specified algorithm is not supported. content_encryption_key = key_encryption_key.unwrap_key( - encryption_data.wrapped_content_key.encrypted_key, - encryption_data.wrapped_content_key.algorithm, + encryption_data.wrapped_content_key.encrypted_key, encryption_data.wrapped_content_key.algorithm ) # For V2, the version is included with the cek. We need to validate it @@ -682,7 +716,7 @@ def _validate_and_unwrap_cek( version_2_bytes = encryption_data.encryption_agent.protocol.encode().ljust(8, b"\0") cek_version_bytes = content_encryption_key[: len(version_2_bytes)] if cek_version_bytes != version_2_bytes: - raise ValueError("The encryption metadata is not valid and may have been modified.") + raise ValueError(_ERROR_INVALID_ENCRYPTION_METADATA) # Remove version from the start of the cek. content_encryption_key = content_encryption_key[len(version_2_bytes) :] @@ -856,7 +890,96 @@ def generate_blob_encryption_data( return content_encryption_key, initialization_vector, encryption_data -def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements +def _parse_content_range(content_range: str) -> Tuple[int, int, int]: + """ + Parses a Content-Range header of the form 'bytes x-y/size' into its + start, end, and total size components. + + :param str content_range: The Content-Range header value. + :return: A tuple of (start, end, total size). + :rtype: Tuple[int, int, int] + """ + # Format: 'bytes x-y/size' -- ignore the leading 'bytes' word. + byte_range, size = content_range.split(" ")[1].split("/") + start, end = byte_range.split("-") + return int(start), int(end), int(size) + + +def _region_nonce_encodings(nonce_length: int) -> Dict[str, Callable[[int], bytes]]: + """ + Returns the supported per-region nonce encodings, keyed by the SDK that produces them. + + The per-region nonce is a counter of the region's position, but each SDK encodes it + differently, so all supported encodings must be understood for interoperability: + + * Python: zero-based counter, big-endian across the whole nonce (value in trailing bytes). + * Java: zero-based counter, big-endian in the leading 8 bytes, trailing bytes zeroed. + * .NET: one-based counter, little-endian in the trailing 8 bytes, leading bytes zeroed. + + These encodings share the same value space, so they must not be accepted independently + per region (for example Java's nonce for region 1 is identical to .NET's nonce for + region 16,777,215). A single encoding is instead selected and enforced across the whole + download; see ``decrypt_blob``. + + :param int nonce_length: The length of the nonce in bytes. + :return: A mapping of SDK name to a function returning that SDK's nonce for a region index. + :rtype: Dict[str, Callable[[int], bytes]] + """ + encodings: Dict[str, Callable[[int], bytes]] = { + "python": lambda index: index.to_bytes(nonce_length, "big"), + } + + counter_length = 8 + pad = nonce_length - counter_length + encodings["java"] = lambda index: index.to_bytes(counter_length, "big") + b"\x00" * pad + encodings["dotnet"] = lambda index: b"\x00" * pad + (index + 1).to_bytes(counter_length, "little") + + return encodings + + +class _GCMRegionNonceValidator: + """ + Enforces that every region across a whole download uses a single nonce encoding. + + ``decrypt_blob`` runs once per HTTP chunk, so the candidate encodings are shared and + intersected across all chunks (including concurrent ones) rather than reset per call. + Otherwise the encoding could change at a chunk boundary and, at an encoding collision, + let a relocated region pass validation. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._candidates: Optional[Dict[str, Callable[[int], bytes]]] = None + # Set once the candidates collapse to a single encoding; read lock-free thereafter. + self._encoding: Optional[Callable[[int], bytes]] = None + + def validate_region(self, region_index: int, nonce: bytes, nonce_length: int) -> None: + """ + Narrows the shared candidate encodings to those consistent with this region. + + :param int region_index: The zero-based index of the region within the blob. + :param bytes nonce: The nonce read from the region. + :param int nonce_length: The length of the nonce in bytes. + """ + encoding = self._encoding + if encoding is not None: + if encoding(region_index) != nonce: + raise ValueError(_ERROR_INVALID_ENCRYPTION_METADATA) + return + + with self._lock: + if self._candidates is None: + self._candidates = _region_nonce_encodings(nonce_length) + self._candidates = { + name: encode for name, encode in self._candidates.items() if encode(region_index) == nonce + } + if not self._candidates: + raise ValueError(_ERROR_INVALID_ENCRYPTION_METADATA) + if len(self._candidates) == 1: + self._encoding = next(iter(self._candidates.values())) + + +def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements,too-many-branches require_encryption: bool, key_encryption_key: Optional[KeyEncryptionKey], key_resolver: Optional[Callable[[str], KeyEncryptionKey]], @@ -864,6 +987,8 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements start_offset: int, end_offset: int, response_headers: Dict[str, Any], + expected_encryption_data: Optional[_EncryptionData] = None, + nonce_validator: Optional["_GCMRegionNonceValidator"] = None, ) -> bytes: """ Decrypts the given blob contents and returns only the requested range. @@ -891,12 +1016,22 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements :param Dict[str, Any] response_headers: A dictionary of response headers from the download request. Expected to include the 'x-ms-meta-encryptiondata' header if the blob was encrypted. + :param Optional[_EncryptionData] expected_encryption_data: + The encryption data retrieved at the start of the download. If provided, the encryption + metadata on this response is validated against it to detect the blob's encryption metadata + being modified (tampered with) partway through a download. + :param Optional[_GCMRegionNonceValidator] nonce_validator: + Shared state used to enforce a single V2 nonce encoding across every chunk of a Blob download. + Required for V2 decryption unless the caller has set the + AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS environment variable. :return: The decrypted blob content. :rtype: bytes """ try: encryption_data = _dict_to_encryption_data(loads(response_headers["x-ms-meta-encryptiondata"])) except Exception as exc: # pylint: disable=broad-except + if expected_encryption_data is not None: + raise ValueError(_ERROR_ENCRYPTION_METADATA_MISMATCH) from exc if require_encryption: raise ValueError( "Encryption required, but received data does not contain appropriate metadata." @@ -905,11 +1040,12 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements return content + # Validate that the encryption metadata has not changed since the start of the download. + if expected_encryption_data is not None and not expected_encryption_data.matches(encryption_data): + raise ValueError(_ERROR_ENCRYPTION_METADATA_MISMATCH) + algorithm = encryption_data.encryption_agent.encryption_algorithm - if algorithm not in ( - _EncryptionAlgorithm.AES_CBC_256, - _EncryptionAlgorithm.AES_GCM_256, - ): + if algorithm not in (_EncryptionAlgorithm.AES_CBC_256, _EncryptionAlgorithm.AES_GCM_256): raise ValueError("Specified encryption algorithm is not supported.") version = encryption_data.encryption_agent.protocol @@ -924,16 +1060,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements iv: Optional[bytes] = None unpad = False if "content-range" in response_headers: - content_range = response_headers["content-range"] - # Format: 'bytes x-y/size' - - # Ignore the word 'bytes' - content_range = content_range.split(" ") - - content_range = content_range[1].split("-") - content_range = content_range[1].split("/") - end_range = int(content_range[0]) - blob_size = int(content_range[1]) + _, end_range, blob_size = _parse_content_range(response_headers["content-range"]) if start_offset >= 16: iv = content[:16] @@ -977,6 +1104,25 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements tag_length = encryption_data.encrypted_region_info.tag_length region_length = nonce_length + data_length + tag_length + # The per-region nonce is a counter of the region's index within the blob. The + # downloaded content always begins on a region boundary, so derive the first + # region's index from the download range (0 when the whole blob was downloaded). + # This lets us validate each nonce and detect reordered regions. + start_range = 0 + if "content-range" in response_headers: + start_range, _, _ = _parse_content_range(response_headers["content-range"]) + nonce_counter = start_range // region_length + + # Bypass nonce validation via an environment variable for data-recovery scenarios + # where regions were reordered. Not recommended: it can allow tampered data through. + validate_nonce = os.environ.get( + "AZURE_STORAGE_CSE_V2_ALLOW_MISORDERED_AUTH_REGIONS", "" + ).strip().lower() not in ("true", "1") + + # A validator is required to enforce a single nonce encoding across the whole download. + if validate_nonce and nonce_validator is None: + raise ValueError("A nonce validator is required to decrypt Encryption V2 content.") + decrypted_content = bytearray() while offset < total_size: # Process one encryption region at a time @@ -985,6 +1131,10 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements # First bytes are the nonce nonce = encrypted_region[:nonce_length] + # Validate the nonce matches the expected counter for this region under a single + # consistent encoding. A mismatch indicates the regions were reordered or tampered with. + if nonce_validator is not None and validate_nonce: + nonce_validator.validate_region(nonce_counter, nonce, nonce_length) ciphertext_with_tag = encrypted_region[nonce_length:] aesgcm = AESGCM(content_encryption_key) @@ -992,6 +1142,7 @@ def decrypt_blob( # pylint: disable=too-many-locals,too-many-statements decrypted_content.extend(decrypted_data) offset += process_size + nonce_counter += 1 # Read the caller requested data from the decrypted content return decrypted_content[start_offset:end_offset]