diff --git a/docs/source/python/parquet/parquet_encryption.rst b/docs/source/python/parquet/parquet_encryption.rst index be46b4850463..529af0b619ba 100644 --- a/docs/source/python/parquet/parquet_encryption.rst +++ b/docs/source/python/parquet/parquet_encryption.rst @@ -269,10 +269,8 @@ Direct Key Encryption (without KMS) encryption/decryption properties directly from a plaintext key, bypassing :class:`~pyarrow.parquet.encryption.CryptoFactory` and the KMS-based flow. -.. note:: - Only uniform encryption (a single key for the footer and all columns) is - supported by these functions. For per-column keys, use the - ``CryptoFactory``/``EncryptionConfiguration`` flow described above. +Without ``column_keys``, these functions use uniform encryption: the footer +key also encrypts every column. .. code-block:: python @@ -288,3 +286,30 @@ encryption/decryption properties directly from a plaintext key, bypassing ... aad_prefix=b'table_id', ... ) >>> pq.read_table('encrypted.parquet', decryption_properties=decryption_props) # doctest: +SKIP + +To encrypt selected columns with separate keys, pass ``column_keys`` as a +dictionary mapping full Parquet column paths to raw keys. Unlike +``EncryptionConfiguration.column_keys``, this maps column paths to keys, +rather than master key IDs to lists of column names. Unlisted columns remain +unencrypted. Nested fields use dot-separated leaf paths, such as +``"person.email"``. The dictionary must not be empty. +Specify each nested leaf separately; parent-field aliases cannot be used +for decryption. + +.. code-block:: python + + >>> import pyarrow as pa + >>> table = pa.table({"sensitive": [1, 2], "public": ["a", "b"]}) # doctest: +SKIP + >>> column_keys = {"sensitive": b'abcdef0123456789'} + >>> props = pe.create_encryption_properties( # doctest: +SKIP + ... footer_key=b'0123456789abcdef', column_keys=column_keys) + >>> pq.write_table(table, 'columns.parquet', encryption_properties=props) # doctest: +SKIP + >>> decryption_props = pe.create_decryption_properties( # doctest: +SKIP + ... footer_key=b'0123456789abcdef', column_keys=column_keys) + >>> pq.read_table('columns.parquet', decryption_properties=decryption_props) # doctest: +SKIP + +Readers only need keys for the encrypted columns they project. The footer key +is still required by these functions, and all supplied keys must be 16, 24, +or 32 bytes. The caller remains responsible for managing and protecting raw +keys; ``CryptoFactory`` provides the higher-level KMS-based key management +flow described above. diff --git a/python/pyarrow/_parquet_encryption.pxd b/python/pyarrow/_parquet_encryption.pxd index 1a12a6d6785e..ba4fd730b1ab 100644 --- a/python/pyarrow/_parquet_encryption.pxd +++ b/python/pyarrow/_parquet_encryption.pxd @@ -21,6 +21,10 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libparquet_encryption cimport * from pyarrow.includes.libparquet cimport ( + c_map, + CColumnDecryptionProperties, + CColumnDecryptionPropertiesBuilder, + CColumnEncryptionProperties, CSecureString, CFileDecryptionPropertiesBuilder, CFileEncryptionPropertiesBuilder, diff --git a/python/pyarrow/_parquet_encryption.pyx b/python/pyarrow/_parquet_encryption.pyx index ed88a98b0070..60402a6f98a6 100644 --- a/python/pyarrow/_parquet_encryption.pyx +++ b/python/pyarrow/_parquet_encryption.pyx @@ -749,12 +749,29 @@ cdef shared_ptr[CDecryptionConfiguration] pyarrow_unwrap_decryptionconfig(object raise TypeError("Expected DecryptionConfiguration, got %s" % type(decryptionconfig)) +cdef _validate_column_keys(column_keys): + if not isinstance(column_keys, dict): + raise TypeError("column_keys must be a dict of column paths to bytes") + if not column_keys: + raise ValueError("column_keys must not be empty") + for column_path, key in column_keys.items(): + if not isinstance(column_path, str): + raise TypeError("column paths in column_keys must be strings") + if not column_path: + raise ValueError("column paths in column_keys must not be empty") + if not isinstance(key, bytes): + raise TypeError("column keys must be bytes") + if len(key) not in (16, 24, 32): + raise ValueError("column keys must be 16, 24, or 32 bytes") + + def create_decryption_properties( footer_key, *, aad_prefix=None, bint check_footer_integrity=True, bint allow_plaintext_files=False, + column_keys=None, ): """ Create FileDecryptionProperties using a direct footer key. @@ -769,18 +786,12 @@ def create_decryption_properties( Parquet key management specification and is interoperable with other tools and frameworks. - .. note:: - Currently only uniform encryption (single key for footer and all - columns) is supported with this method. Per-column keys are not - yet available; files encrypted with per-column keys cannot be - decrypted using this function. - Parameters ---------- footer_key : bytes - The decryption key for the file footer and all columns (uniform - encryption). Must be 16, 24, or 32 bytes for AES-128, AES-192, - or AES-256 respectively. + The decryption key for the file footer and any columns encrypted + with the footer key. Must be 16, 24, or 32 bytes for AES-128, + AES-192, or AES-256 respectively. aad_prefix : bytes, optional Additional Authenticated Data prefix. Must match the AAD prefix that was used during encryption. Required if the AAD prefix was @@ -791,6 +802,12 @@ def create_decryption_properties( allow_plaintext_files : bool, default False Whether to allow reading plaintext (unencrypted) files with these decryption properties without raising an error. + column_keys : dict[str, bytes], optional + Explicit keys for encrypted columns, mapped by their full Parquet + column paths (dot-separated for nested fields). Keys must be 16, 24, + or 32 bytes. Only columns read need to have keys supplied. Columns + encrypted with the footer key continue to use ``footer_key``. + An empty dictionary is not allowed. Returns ------- @@ -814,6 +831,11 @@ def create_decryption_properties( CSecureString c_footer_key CFileDecryptionPropertiesBuilder builder shared_ptr[CFileDecryptionProperties] props + c_map[c_string, shared_ptr[CColumnDecryptionProperties]] c_column_keys + unique_ptr[CColumnDecryptionPropertiesBuilder] column_builder + c_string c_column_path + c_string c_column_key_str + CSecureString c_column_key if not isinstance(footer_key, bytes): raise TypeError( @@ -828,6 +850,18 @@ def create_decryption_properties( c_footer_key = CSecureString(move(c_footer_key_str)) builder.footer_key(c_footer_key) + if column_keys is not None: + _validate_column_keys(column_keys) + for column_path, key in column_keys.items(): + c_column_path = tobytes(column_path) + c_column_key_str = key + c_column_key = CSecureString(move(c_column_key_str)) + column_builder.reset( + new CColumnDecryptionPropertiesBuilder(c_column_path)) + column_builder.get().key(c_column_key) + c_column_keys[c_column_path] = column_builder.get().build() + builder.column_keys(c_column_keys) + if aad_prefix is not None: if not isinstance(aad_prefix, bytes): raise TypeError( @@ -853,6 +887,7 @@ def create_encryption_properties( bint store_aad_prefix=True, encryption_algorithm="AES_GCM_V1", bint plaintext_footer=False, + column_keys=None, ): """ Create FileEncryptionProperties using a direct footer key. @@ -871,18 +906,12 @@ def create_encryption_properties( prefer it unless you have a specific reason to manage keys yourself. - .. note:: - Currently only uniform encryption (single key for footer and all - columns) is supported with this method. Per-column keys are not - yet available; the provided key encrypts both the footer and - every column. - Parameters ---------- footer_key : bytes - The encryption key for the file footer and all columns (uniform - encryption). Must be 16, 24, or 32 bytes for AES-128, AES-192, - or AES-256 respectively. + The encryption key for the file footer. Also encrypts every column + when ``column_keys`` is not specified (uniform encryption). Must be + 16, 24, or 32 bytes for AES-128, AES-192, or AES-256 respectively. aad_prefix : bytes, optional Additional Authenticated Data prefix for cryptographic binding. store_aad_prefix : bool, default True @@ -896,6 +925,12 @@ def create_encryption_properties( plaintext_footer : bool, default False Whether to leave the file footer unencrypted. When True, file schema and metadata are readable without a key. + column_keys : dict[str, bytes], optional + Keys for columns to encrypt, mapped by their full Parquet column + paths (dot-separated for nested fields). Keys must be 16, 24, or + 32 bytes. When specified, columns not in this dictionary remain + unencrypted. If omitted, all columns use ``footer_key``. + An empty dictionary is not allowed. Returns ------- @@ -922,6 +957,10 @@ def create_encryption_properties( CFileEncryptionPropertiesBuilder* builder shared_ptr[CFileEncryptionProperties] props ParquetCipher cipher + c_map[c_string, shared_ptr[CColumnEncryptionProperties]] c_column_keys + c_string c_column_path + c_string c_column_key_str + CSecureString c_column_key if not isinstance(footer_key, bytes): raise TypeError( @@ -939,6 +978,16 @@ def create_encryption_properties( try: builder.algorithm(cipher) + if column_keys is not None: + _validate_column_keys(column_keys) + for column_path, key in column_keys.items(): + c_column_path = tobytes(column_path) + c_column_key_str = key + c_column_key = CSecureString(move(c_column_key_str)) + c_column_keys[c_column_path] = \ + CColumnEncryptionProperties.WithColumnKey(c_column_key, b"") + builder.encrypted_columns(c_column_keys) + if aad_prefix is not None: if not isinstance(aad_prefix, bytes): raise TypeError( diff --git a/python/pyarrow/includes/libparquet.pxd b/python/pyarrow/includes/libparquet.pxd index 3a5f91716c17..cfb48f60fca7 100644 --- a/python/pyarrow/includes/libparquet.pxd +++ b/python/pyarrow/includes/libparquet.pxd @@ -17,6 +17,8 @@ # distutils: language = c++ +from libcpp.map cimport map as c_map + from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport (Type, CChunkedArray, CScalar, CSchema, CStatus, CTable, CMemoryPool, CBuffer, @@ -638,6 +640,22 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil: const COutputStream* sink) cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil: + cdef cppclass CColumnEncryptionProperties\ + " parquet::ColumnEncryptionProperties": + @staticmethod + shared_ptr[CColumnEncryptionProperties] WithColumnKey( + CSecureString key, c_string key_metadata) except + + + cdef cppclass CColumnDecryptionProperties\ + " parquet::ColumnDecryptionProperties": + pass + + cdef cppclass CColumnDecryptionPropertiesBuilder\ + " parquet::ColumnDecryptionProperties::Builder": + CColumnDecryptionPropertiesBuilder(c_string name) except + + CColumnDecryptionPropertiesBuilder* key(CSecureString key) except + + shared_ptr[CColumnDecryptionProperties] build() except + + cdef cppclass CFileDecryptionProperties\ " parquet::FileDecryptionProperties": pass @@ -647,6 +665,8 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil: CFileDecryptionPropertiesBuilder() except + CFileDecryptionPropertiesBuilder* footer_key( CSecureString footer_key) except + + CFileDecryptionPropertiesBuilder* column_keys( + c_map[c_string, shared_ptr[CColumnDecryptionProperties]] column_keys) except + CFileDecryptionPropertiesBuilder* aad_prefix( c_string aad_prefix) except + CFileDecryptionPropertiesBuilder* disable_footer_signature_verification() except + @@ -666,4 +686,6 @@ cdef extern from "parquet/encryption/encryption.h" namespace "parquet" nogil: CFileEncryptionPropertiesBuilder* aad_prefix( c_string aad_prefix) except + CFileEncryptionPropertiesBuilder* disable_aad_prefix_storage() except + + CFileEncryptionPropertiesBuilder* encrypted_columns( + c_map[c_string, shared_ptr[CColumnEncryptionProperties]] columns) except + shared_ptr[CFileEncryptionProperties] build() except + diff --git a/python/pyarrow/tests/parquet/test_encryption.py b/python/pyarrow/tests/parquet/test_encryption.py index 5185bf6a3013..7d68c07d17d8 100644 --- a/python/pyarrow/tests/parquet/test_encryption.py +++ b/python/pyarrow/tests/parquet/test_encryption.py @@ -880,6 +880,114 @@ def test_roundtrip_key_sizes(self, tempdir, data_table, key): result = pq.read_table(path, decryption_properties=dec_props) assert data_table.equals(result) + @pytest.mark.parametrize("plaintext_footer", [False, True]) + @pytest.mark.parametrize("store_aad_prefix", [False, True]) + def test_per_column_keys_roundtrip( + self, tempdir, data_table, plaintext_footer, store_aad_prefix + ): + path = tempdir / "direct_columns.parquet" + column_keys = {"a": DIRECT_KEY_192, "b": DIRECT_KEY_256} + enc_props = pe.create_encryption_properties( + DIRECT_KEY_128, + column_keys=column_keys, + plaintext_footer=plaintext_footer, + aad_prefix=DIRECT_AAD_PREFIX, + store_aad_prefix=store_aad_prefix, + ) + pq.write_table(data_table, path, encryption_properties=enc_props) + + dec_props = pe.create_decryption_properties( + DIRECT_KEY_128, + column_keys=column_keys, + aad_prefix=DIRECT_AAD_PREFIX, + ) + result = pq.read_table(path, decryption_properties=dec_props) + assert result.equals(data_table) + + def test_per_column_keys_projection(self, tempdir, data_table): + path = tempdir / "direct_projection.parquet" + enc_props = pe.create_encryption_properties( + DIRECT_KEY_128, + column_keys={"a": DIRECT_KEY_192, "b": DIRECT_KEY_256}, + ) + pq.write_table(data_table, path, encryption_properties=enc_props) + + # A key is only required for an encrypted column being read. + result = pq.read_table( + path, columns=["c"], + decryption_properties=pe.create_decryption_properties(DIRECT_KEY_128), + ) + assert result.equals(data_table.select(["c"])) + result = pq.read_table( + path, columns=["a", "c"], + decryption_properties=pe.create_decryption_properties( + DIRECT_KEY_128, column_keys={"a": DIRECT_KEY_192}), + ) + assert result.equals(data_table.select(["a", "c"])) + + with pytest.raises(IOError, match="key"): + pq.read_table( + path, + decryption_properties=pe.create_decryption_properties( + DIRECT_KEY_128, column_keys={"a": DIRECT_KEY_192}), + ) + + def test_per_column_wrong_key(self, tempdir, data_table): + path = tempdir / "direct_wrong_column_key.parquet" + enc_props = pe.create_encryption_properties( + DIRECT_KEY_128, column_keys={"a": DIRECT_KEY_192}) + pq.write_table(data_table, path, encryption_properties=enc_props) + + with pytest.raises(IOError, match="decrypt"): + pq.read_table( + path, + decryption_properties=pe.create_decryption_properties( + DIRECT_KEY_128, column_keys={"a": DIRECT_KEY_256}), + ) + + def test_per_column_nested_path(self, tempdir): + table = pa.table({"details": [{"private": 1, "public": "a"}, + {"private": 2, "public": "b"}]}) + path = tempdir / "direct_nested.parquet" + column_keys = {"details.private": DIRECT_KEY_192} + pq.write_table( + table, path, + encryption_properties=pe.create_encryption_properties( + DIRECT_KEY_128, column_keys=column_keys), + ) + result = pq.read_table( + path, + decryption_properties=pe.create_decryption_properties( + DIRECT_KEY_128, column_keys=column_keys), + ) + assert result.equals(table) + + def test_per_column_unknown_path(self, tempdir, data_table): + enc_props = pe.create_encryption_properties( + DIRECT_KEY_128, column_keys={"missing": DIRECT_KEY_192}) + with pytest.raises( + IOError, match="Encrypted column missing not in file schema" + ): + pq.write_table( + data_table, tempdir / "direct_unknown_column.parquet", + encryption_properties=enc_props, + ) + + @pytest.mark.parametrize("factory", [ + "create_encryption_properties", "create_decryption_properties", + ]) + @pytest.mark.parametrize("column_keys, error, message", [ + ([], TypeError, "column_keys must be a dict"), + ({}, ValueError, "column_keys must not be empty"), + ({1: DIRECT_KEY_128}, TypeError, "column paths.*must be strings"), + ({"": DIRECT_KEY_128}, ValueError, "column paths.*must not be empty"), + ({"a": "0123456789abcdef"}, TypeError, "column keys must be bytes"), + ({"a": b"short"}, ValueError, "column keys must be 16, 24, or 32 bytes"), + ]) + def test_invalid_column_keys(self, factory, column_keys, error, message): + with pytest.raises(error, match=message): + getattr(pe, factory)(DIRECT_KEY_128, column_keys=column_keys) + def test_roundtrip_with_aad_prefix(self, tempdir, data_table): path = tempdir / "direct_aad.parquet"