From 046e1052f45a92573a888333c954bd985789f4ae Mon Sep 17 00:00:00 2001 From: shangxinli Date: Fri, 16 Oct 2026 12:08:56 -0700 Subject: [PATCH 1/4] feat: add AES GCM cipher and AES GCM Stream (AGS1) files Add the crypto primitives of table encryption to the bundle library, built when ICEBERG_ENCRYPTION is ON: - AesGcmCipher (OpenSSL EVP): AES-GCM with a random 12-byte nonce and the nonce | ciphertext | tag layout of Java's Ciphers. - AesGcmInputFile / AesGcmOutputFile: AES GCM Stream files (format/gcm-stream-spec.md), used for manifests, manifest lists, Avro data files and Puffin files. 1 MiB plaintext blocks, block AAD = file AAD prefix followed by the block index. The reader is seekable, takes the encrypted length from trusted key metadata, and verifies every block. The writer reports the plaintext Position() and the encrypted StoredLength(). aes_gcm_stream.h is the first public header that needs ICEBERG_ENCRYPTION while sharing a directory with unconditional ones, so the public-header generator grows a per-header override and the C++20 compatibility test a matching ICEBERG_PUBLIC_HEADERS_ENCRYPTION define. Tests decrypt AGS1 streams and KEK-encrypted key metadata produced by Java, and cover multi-block round trips, random access and tampering. --- ci/scripts/generate_public_headers.py | 19 +- src/iceberg/CMakeLists.txt | 5 + src/iceberg/cpp20_compatibility_internal.h | 4 + src/iceberg/encryption/aes_gcm_internal.cc | 174 +++++++++++ src/iceberg/encryption/aes_gcm_internal.h | 71 +++++ src/iceberg/encryption/aes_gcm_stream.cc | 344 +++++++++++++++++++++ src/iceberg/encryption/aes_gcm_stream.h | 89 ++++++ src/iceberg/test/CMakeLists.txt | 8 + src/iceberg/test/aes_gcm_test.cc | 284 +++++++++++++++++ 9 files changed, 994 insertions(+), 4 deletions(-) create mode 100644 src/iceberg/encryption/aes_gcm_internal.cc create mode 100644 src/iceberg/encryption/aes_gcm_internal.h create mode 100644 src/iceberg/encryption/aes_gcm_stream.cc create mode 100644 src/iceberg/encryption/aes_gcm_stream.h create mode 100644 src/iceberg/test/aes_gcm_test.cc diff --git a/ci/scripts/generate_public_headers.py b/ci/scripts/generate_public_headers.py index 56f196d87b..5a515871c9 100755 --- a/ci/scripts/generate_public_headers.py +++ b/ci/scripts/generate_public_headers.py @@ -51,6 +51,13 @@ ("iceberg/parquet", "ICEBERG_PUBLIC_HEADERS_BUNDLE"), ] +# Individual headers of an optional component that lives in a directory which is +# otherwise always built. iceberg/encryption holds both the metadata types, which are +# part of core, and the crypto implementation, which needs ICEBERG_ENCRYPTION. +OPTIONAL_HEADERS = { + "iceberg/encryption/aes_gcm_stream.h": "ICEBERG_PUBLIC_HEADERS_ENCRYPTION", +} + INSTALL_CALL = re.compile(r"iceberg_install_all_headers\(\s*([^\s)]+)") LICENSE = """\ @@ -75,7 +82,9 @@ """ -def component_of(install_path: str) -> str | None: +def component_of(install_path: str, header_path: str | None = None) -> str | None: + if header_path is not None and header_path in OPTIONAL_HEADERS: + return OPTIONAL_HEADERS[header_path] for prefix, macro in OPTIONAL_COMPONENTS: if install_path == prefix or install_path.startswith(prefix + "/"): return macro @@ -89,8 +98,9 @@ def collect_public_headers() -> dict[str | None, list[str]]: for header in cmake_file.parent.iterdir(): if header.suffix != ".h" or "internal" in header.name: continue - headers.setdefault(component_of(install_path), []).append( - f"{install_path}/{header.name}" + header_path = f"{install_path}/{header.name}" + headers.setdefault(component_of(install_path, header_path), []).append( + header_path ) return headers @@ -107,7 +117,8 @@ def render(headers: dict[str | None, list[str]]) -> str: "", ] lines += [f'#include "{h}"' for h in sorted(headers.get(None, []))] - for macro in sorted({m for _, m in OPTIONAL_COMPONENTS}): + optional_macros = {m for _, m in OPTIONAL_COMPONENTS} | set(OPTIONAL_HEADERS.values()) + for macro in sorted(optional_macros): if macro not in headers: continue lines += ["", f"#ifdef {macro}"] diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index c283b8af9d..d3133adc25 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -326,6 +326,11 @@ if(ICEBERG_BUILD_BUNDLE) parquet/parquet_writer.cc parquet/row_number_reader.cc) + if(ICEBERG_ENCRYPTION) + list(APPEND ICEBERG_BUNDLE_SOURCES encryption/aes_gcm_internal.cc + encryption/aes_gcm_stream.cc) + endif() + # Libraries to link with exported libiceberg_bundle.{so,a}. set(ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS) set(ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS) diff --git a/src/iceberg/cpp20_compatibility_internal.h b/src/iceberg/cpp20_compatibility_internal.h index 359e5ddff8..bd9a9475d8 100644 --- a/src/iceberg/cpp20_compatibility_internal.h +++ b/src/iceberg/cpp20_compatibility_internal.h @@ -216,6 +216,10 @@ # include "iceberg/parquet/parquet_writer.h" #endif // ICEBERG_PUBLIC_HEADERS_BUNDLE +#ifdef ICEBERG_PUBLIC_HEADERS_ENCRYPTION +# include "iceberg/encryption/aes_gcm_stream.h" +#endif // ICEBERG_PUBLIC_HEADERS_ENCRYPTION + #ifdef ICEBERG_PUBLIC_HEADERS_HIVE # include "iceberg/catalog/hive/hive_catalog.h" # include "iceberg/catalog/hive/hive_catalog_properties.h" diff --git a/src/iceberg/encryption/aes_gcm_internal.cc b/src/iceberg/encryption/aes_gcm_internal.cc new file mode 100644 index 0000000000..63fe630365 --- /dev/null +++ b/src/iceberg/encryption/aes_gcm_internal.cc @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/encryption/aes_gcm_internal.h" + +#include + +#include +#include + +#include "iceberg/util/macros.h" + +namespace iceberg::encryption { + +namespace { + +const EVP_CIPHER* CipherForKey(size_t key_length) { + switch (key_length) { + case 16: + return EVP_aes_128_gcm(); + case 24: + return EVP_aes_192_gcm(); + case 32: + return EVP_aes_256_gcm(); + default: + return nullptr; + } +} + +EVP_CIPHER_CTX* Ctx(void* ctx) { return static_cast(ctx); } + +} // namespace + +Status SecureRandomBytes(std::span out) { + if (out.size() > static_cast(std::numeric_limits::max()) || + RAND_bytes(out.data(), static_cast(out.size())) != 1) { + return IOError("Failed to generate secure random bytes"); + } + return {}; +} + +Result> AesGcmCipher::Make(std::span key) { + if (CipherForKey(key.size()) == nullptr) { + return InvalidArgument("Invalid key length: {} (must be 16, 24, or 32 bytes)", + key.size()); + } + EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); + if (ctx == nullptr) { + return IOError("Failed to create cipher context"); + } + return std::unique_ptr( + new AesGcmCipher(std::vector(key.begin(), key.end()), ctx)); +} + +AesGcmCipher::AesGcmCipher(std::vector key, void* ctx) + : key_(std::move(key)), ctx_(ctx) {} + +AesGcmCipher::~AesGcmCipher() { EVP_CIPHER_CTX_free(Ctx(ctx_)); } + +Status AesGcmCipher::Encrypt(std::span plaintext, + std::span aad, std::span out) { + if (out.size() != plaintext.size() + kNonceLength + kGcmTagLength) { + return InvalidArgument("Invalid ciphertext buffer size: {}", out.size()); + } + if (plaintext.size() > static_cast(std::numeric_limits::max())) { + return InvalidArgument("Plaintext too large: {}", plaintext.size()); + } + + auto nonce = out.first(kNonceLength); + auto ciphertext = out.subspan(kNonceLength, plaintext.size()); + auto tag = out.last(kGcmTagLength); + ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(nonce)); + + auto* ctx = Ctx(ctx_); + int len = 0; + if (EVP_EncryptInit_ex(ctx, CipherForKey(key_.size()), nullptr, nullptr, nullptr) != + 1 || + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, kNonceLength, nullptr) != 1 || + EVP_EncryptInit_ex(ctx, nullptr, nullptr, key_.data(), nonce.data()) != 1) { + return IOError("Failed to initialize AES GCM encryption"); + } + if (!aad.empty() && EVP_EncryptUpdate(ctx, nullptr, &len, aad.data(), + static_cast(aad.size())) != 1) { + return IOError("Failed to set AES GCM AAD"); + } + if (!plaintext.empty() && + EVP_EncryptUpdate(ctx, ciphertext.data(), &len, plaintext.data(), + static_cast(plaintext.size())) != 1) { + return IOError("Failed to encrypt"); + } + if (EVP_EncryptFinal_ex(ctx, ciphertext.data() + plaintext.size(), &len) != 1 || + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, kGcmTagLength, tag.data()) != 1) { + return IOError("Failed to finalize AES GCM encryption"); + } + return {}; +} + +Status AesGcmCipher::Decrypt(std::span ciphertext, + std::span aad, std::span out) { + if (ciphertext.size() < static_cast(kNonceLength + kGcmTagLength)) { + return Invalid( + "Cannot decrypt cipher text of length {}: shorter than nonce and tag, text may " + "not be encrypted with AES GCM", + ciphertext.size()); + } + size_t plaintext_length = ciphertext.size() - kNonceLength - kGcmTagLength; + if (out.size() != plaintext_length) { + return InvalidArgument("Invalid plaintext buffer size: {}", out.size()); + } + + auto nonce = ciphertext.first(kNonceLength); + auto data = ciphertext.subspan(kNonceLength, plaintext_length); + auto tag = ciphertext.last(kGcmTagLength); + + auto* ctx = Ctx(ctx_); + int len = 0; + if (EVP_DecryptInit_ex(ctx, CipherForKey(key_.size()), nullptr, nullptr, nullptr) != + 1 || + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, kNonceLength, nullptr) != 1 || + EVP_DecryptInit_ex(ctx, nullptr, nullptr, key_.data(), nonce.data()) != 1) { + return IOError("Failed to initialize AES GCM decryption"); + } + if (!aad.empty() && EVP_DecryptUpdate(ctx, nullptr, &len, aad.data(), + static_cast(aad.size())) != 1) { + return IOError("Failed to set AES GCM AAD"); + } + if (!data.empty() && EVP_DecryptUpdate(ctx, out.data(), &len, data.data(), + static_cast(data.size())) != 1) { + return IOError("Failed to decrypt"); + } + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, kGcmTagLength, + const_cast(tag.data())) != 1 || + EVP_DecryptFinal_ex(ctx, out.data() + plaintext_length, &len) != 1) { + return Invalid( + "GCM tag check failed. Possible reasons: wrong decryption key; or " + "corrupt/tampered data. AES GCM doesn't differentiate between these two."); + } + return {}; +} + +Result> AesGcmCipher::Encrypt(std::span plaintext, + std::span aad) { + std::vector out(plaintext.size() + kNonceLength + kGcmTagLength); + ICEBERG_RETURN_UNEXPECTED(Encrypt(plaintext, aad, out)); + return out; +} + +Result> AesGcmCipher::Decrypt(std::span ciphertext, + std::span aad) { + if (ciphertext.size() < static_cast(kNonceLength + kGcmTagLength)) { + return Invalid("Cannot decrypt cipher text of length {}", ciphertext.size()); + } + std::vector out(ciphertext.size() - kNonceLength - kGcmTagLength); + ICEBERG_RETURN_UNEXPECTED(Decrypt(ciphertext, aad, out)); + return out; +} + +} // namespace iceberg::encryption diff --git a/src/iceberg/encryption/aes_gcm_internal.h b/src/iceberg/encryption/aes_gcm_internal.h new file mode 100644 index 0000000000..e09a7802e7 --- /dev/null +++ b/src/iceberg/encryption/aes_gcm_internal.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/encryption/aes_gcm_internal.h +/// AES GCM cipher compatible with Java's org.apache.iceberg.encryption.Ciphers. + +#include +#include +#include +#include + +#include "iceberg/result.h" + +namespace iceberg::encryption { + +inline constexpr int32_t kNonceLength = 12; +inline constexpr int32_t kGcmTagLength = 16; + +/// \brief Fill `out` with cryptographically secure random bytes. +Status SecureRandomBytes(std::span out); + +/// \brief AES GCM with a random 12-byte nonce. +/// +/// Ciphertext layout: nonce (12) | encrypted data | tag (16). +class AesGcmCipher { + public: + /// \brief Create a cipher for a 16, 24 or 32 byte key. + static Result> Make(std::span key); + + ~AesGcmCipher(); + + /// \brief Encrypt `plaintext` into `out`, which must hold plaintext.size() + 28 bytes. + Status Encrypt(std::span plaintext, std::span aad, + std::span out); + + /// \brief Decrypt and verify `ciphertext` into `out`, which must hold + /// ciphertext.size() - 28 bytes. + Status Decrypt(std::span ciphertext, std::span aad, + std::span out); + + Result> Encrypt(std::span plaintext, + std::span aad); + Result> Decrypt(std::span ciphertext, + std::span aad); + + private: + AesGcmCipher(std::vector key, void* ctx); + + std::vector key_; + void* ctx_; // EVP_CIPHER_CTX +}; + +} // namespace iceberg::encryption diff --git a/src/iceberg/encryption/aes_gcm_stream.cc b/src/iceberg/encryption/aes_gcm_stream.cc new file mode 100644 index 0000000000..7fb417c115 --- /dev/null +++ b/src/iceberg/encryption/aes_gcm_stream.cc @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/encryption/aes_gcm_stream.h" + +#include +#include +#include +#include +#include + +#include "iceberg/encryption/aes_gcm_internal.h" +#include "iceberg/util/macros.h" + +namespace iceberg { + +namespace { + +using encryption::AesGcmCipher; + +constexpr int64_t kPlainBlock = AesGcmStream::kPlainBlockSize; +constexpr int64_t kCipherBlock = AesGcmStream::kCipherBlockSize; +constexpr int64_t kBlockOverhead = + AesGcmStream::kNonceLength + AesGcmStream::kGcmTagLength; + +/// \brief Block AAD: file AAD prefix followed by the block index as a 4-byte LE int. +std::vector BlockAad(const std::vector& aad_prefix, + int32_t block_index) { + std::vector aad(aad_prefix); + for (int i = 0; i < 4; ++i) { + aad.push_back( + static_cast((static_cast(block_index) >> (8 * i)) & 0xFF)); + } + return aad; +} + +std::array HeaderBytes() { + std::array header{}; + std::memcpy(header.data(), AesGcmStream::kMagic.data(), AesGcmStream::kMagic.size()); + for (int i = 0; i < 4; ++i) { + header[4 + i] = static_cast( + (static_cast(AesGcmStream::kPlainBlockSize) >> (8 * i)) & 0xFF); + } + return header; +} + +class AesGcmInputStream : public SeekableInputStream { + public: + AesGcmInputStream(std::unique_ptr source, + std::unique_ptr cipher, std::vector aad_prefix, + int64_t encrypted_length) + : source_(std::move(source)), + cipher_(std::move(cipher)), + aad_prefix_(std::move(aad_prefix)), + cipher_block_(kCipherBlock), + plain_block_(kPlainBlock) { + int64_t stream_length = encrypted_length - AesGcmStream::kHeaderLength; + int64_t num_full_blocks = stream_length / kCipherBlock; + int64_t last_bytes = stream_length - num_full_blocks * kCipherBlock; + bool full_blocks_only = last_bytes == 0; + num_blocks_ = full_blocks_only ? num_full_blocks : num_full_blocks + 1; + last_cipher_block_size_ = full_blocks_only ? kCipherBlock : last_bytes; + plain_size_ = AesGcmStream::PlaintextLength(encrypted_length); + } + + Status ValidateHeader() { + std::array header{}; + ICEBERG_RETURN_UNEXPECTED(source_->ReadFully(0, header)); + if (std::memcmp(header.data(), AesGcmStream::kMagic.data(), + AesGcmStream::kMagic.size()) != 0) { + return Invalid("Invalid GCM stream: magic does not match AGS1"); + } + uint32_t plain_block_size = 0; + for (int i = 0; i < 4; ++i) { + plain_block_size |= static_cast(header[4 + i]) << (8 * i); + } + if (plain_block_size != static_cast(AesGcmStream::kPlainBlockSize)) { + return Invalid("Invalid GCM stream: block size {} != {}", plain_block_size, + AesGcmStream::kPlainBlockSize); + } + if (last_cipher_block_size_ < kBlockOverhead) { + return Invalid("Invalid GCM stream: truncated last block"); + } + return {}; + } + + Result Position() const override { return position_; } + + Status Seek(int64_t position) override { + if (position < 0 || position > plain_size_) { + return IOError("Invalid position {} for stream of length {}", position, + plain_size_); + } + position_ = position; + return {}; + } + + Result Read(std::span out) override { + int64_t total = 0; + while (std::cmp_less(total, out.size()) && position_ < plain_size_) { + int64_t block_index = position_ / kPlainBlock; + ICEBERG_RETURN_UNEXPECTED(DecryptBlock(block_index)); + int64_t offset = position_ % kPlainBlock; + int64_t n = std::min(current_block_size_ - offset, + static_cast(out.size()) - total); + std::memcpy(out.data() + total, plain_block_.data() + offset, n); + total += n; + position_ += n; + } + return total; + } + + Status ReadFully(int64_t position, std::span out) override { + if (position < 0 || position + static_cast(out.size()) > plain_size_) { + return IOError("Cannot read {} bytes at {} from stream of length {}", out.size(), + position, plain_size_); + } + ICEBERG_RETURN_UNEXPECTED(Seek(position)); + ICEBERG_ASSIGN_OR_RAISE(auto n, Read(out)); + if (std::cmp_not_equal(n, out.size())) { + return IOError("Short read from GCM stream"); + } + return {}; + } + + Status Close() override { return source_->Close(); } + + private: + Status DecryptBlock(int64_t block_index) { + if (block_index == current_block_index_) { + return {}; + } + if (block_index > std::numeric_limits::max()) { + return Invalid("GCM stream block index overflow"); + } + int64_t size = + block_index == num_blocks_ - 1 ? last_cipher_block_size_ : kCipherBlock; + auto cipher_block = std::span(cipher_block_).first(size); + ICEBERG_RETURN_UNEXPECTED( + source_->ReadFully(AesGcmStream::kHeaderLength + block_index * kCipherBlock, + std::as_writable_bytes(cipher_block))); + auto aad = BlockAad(aad_prefix_, static_cast(block_index)); + int64_t plain_size = size - kBlockOverhead; + // Decryption writes plaintext before the tag is checked, so the cached block is + // invalid until the whole block authenticates. + current_block_index_ = -1; + current_block_size_ = 0; + ICEBERG_RETURN_UNEXPECTED( + cipher_->Decrypt(cipher_block, aad, std::span(plain_block_).first(plain_size))); + current_block_size_ = plain_size; + current_block_index_ = block_index; + return {}; + } + + std::unique_ptr source_; + std::unique_ptr cipher_; + std::vector aad_prefix_; + std::vector cipher_block_; + std::vector plain_block_; + int64_t num_blocks_ = 0; + int64_t last_cipher_block_size_ = 0; + int64_t plain_size_ = 0; + int64_t position_ = 0; + int64_t current_block_index_ = -1; + int64_t current_block_size_ = 0; +}; + +class AesGcmOutputStream : public PositionOutputStream { + public: + AesGcmOutputStream(std::unique_ptr target, + std::unique_ptr cipher, + std::vector aad_prefix) + : target_(std::move(target)), + cipher_(std::move(cipher)), + aad_prefix_(std::move(aad_prefix)), + plain_block_(kPlainBlock), + cipher_block_(kCipherBlock) {} + + Result Position() const override { + // Closing flushes the last block, which resets the in-block position, so the + // plaintext length is captured before that happens. + if (closed_) { + return final_position_; + } + return block_index_ * kPlainBlock + position_in_block_; + } + + Result StoredLength() const override { return target_->StoredLength(); } + + Status Write(std::span data) override { + if (closed_) { + return IOError("Writing to closed stream"); + } + ICEBERG_RETURN_UNEXPECTED(WriteHeaderIfNeeded()); + while (!data.empty()) { + int64_t n = std::min(kPlainBlock - position_in_block_, data.size()); + std::memcpy(plain_block_.data() + position_in_block_, data.data(), n); + position_in_block_ += n; + data = data.subspan(n); + if (position_in_block_ == kPlainBlock) { + ICEBERG_RETURN_UNEXPECTED(EncryptAndWriteBlock()); + } + } + return {}; + } + + Status Flush() override { return target_->Flush(); } + + Status Close() override { + if (closed_) { + return {}; + } + ICEBERG_RETURN_UNEXPECTED(WriteHeaderIfNeeded()); + ICEBERG_ASSIGN_OR_RAISE(final_position_, Position()); + // Like Java, an empty stream gets one empty block; otherwise only a non-empty + // partial block is written. + if (position_in_block_ > 0 || block_index_ == 0) { + ICEBERG_RETURN_UNEXPECTED(EncryptAndWriteBlock()); + } + closed_ = true; + return target_->Close(); + } + + private: + Status WriteHeaderIfNeeded() { + if (!header_written_) { + auto header = HeaderBytes(); + ICEBERG_RETURN_UNEXPECTED(target_->Write(header)); + header_written_ = true; + } + return {}; + } + + Status EncryptAndWriteBlock() { + if (last_block_written_) { + return Invalid("Cannot encrypt block: a partial block has already been written"); + } + if (block_index_ == std::numeric_limits::max()) { + return IOError("Cannot write block: exceeded Integer.MAX_VALUE blocks"); + } + if (position_in_block_ != kPlainBlock) { + last_block_written_ = true; + } + auto aad = BlockAad(aad_prefix_, static_cast(block_index_)); + auto out = std::span(cipher_block_).first(position_in_block_ + kBlockOverhead); + ICEBERG_RETURN_UNEXPECTED( + cipher_->Encrypt(std::span(plain_block_).first(position_in_block_), aad, out)); + ICEBERG_RETURN_UNEXPECTED(target_->Write(std::as_bytes(out))); + position_in_block_ = 0; + ++block_index_; + return {}; + } + + std::unique_ptr target_; + std::unique_ptr cipher_; + std::vector aad_prefix_; + std::vector plain_block_; + std::vector cipher_block_; + int64_t position_in_block_ = 0; + int64_t block_index_ = 0; + int64_t final_position_ = 0; + bool header_written_ = false; + bool last_block_written_ = false; + bool closed_ = false; +}; + +} // namespace + +int64_t AesGcmStream::PlaintextLength(int64_t encrypted_length) { + int64_t stream_length = encrypted_length - kHeaderLength; + if (stream_length <= 0) { + return 0; + } + int64_t num_full_blocks = stream_length / kCipherBlockSize; + int64_t last_bytes = stream_length - num_full_blocks * kCipherBlockSize; + int64_t last_plain = last_bytes == 0 ? 0 : last_bytes - kNonceLength - kGcmTagLength; + return num_full_blocks * kPlainBlockSize + std::max(last_plain, 0); +} + +AesGcmInputFile::AesGcmInputFile(std::unique_ptr source, + std::vector key, + std::vector aad_prefix, + int64_t encrypted_length) + : source_(std::move(source)), + key_(std::move(key)), + aad_prefix_(std::move(aad_prefix)), + encrypted_length_(encrypted_length) {} + +Result AesGcmInputFile::Size() const { + return AesGcmStream::PlaintextLength(encrypted_length_); +} + +Result> AesGcmInputFile::Open() { + if (encrypted_length_ < AesGcmStream::kMinStreamLength) { + return Invalid( + "Invalid encrypted stream: {} is shorter than the minimum possible stream length", + encrypted_length_); + } + ICEBERG_ASSIGN_OR_RAISE(auto cipher, AesGcmCipher::Make(key_)); + ICEBERG_ASSIGN_OR_RAISE(auto source, source_->Open()); + auto stream = std::make_unique(std::move(source), std::move(cipher), + aad_prefix_, encrypted_length_); + ICEBERG_RETURN_UNEXPECTED(stream->ValidateHeader()); + return stream; +} + +AesGcmOutputFile::AesGcmOutputFile(std::unique_ptr target, + std::vector key, + std::vector aad_prefix) + : target_(std::move(target)), + key_(std::move(key)), + aad_prefix_(std::move(aad_prefix)) {} + +Result> AesGcmOutputFile::Create() { + ICEBERG_ASSIGN_OR_RAISE(auto cipher, encryption::AesGcmCipher::Make(key_)); + ICEBERG_ASSIGN_OR_RAISE(auto target, target_->Create()); + return std::make_unique(std::move(target), std::move(cipher), + aad_prefix_); +} + +Result> AesGcmOutputFile::CreateOrOverwrite() { + ICEBERG_ASSIGN_OR_RAISE(auto cipher, encryption::AesGcmCipher::Make(key_)); + ICEBERG_ASSIGN_OR_RAISE(auto target, target_->CreateOrOverwrite()); + return std::make_unique(std::move(target), std::move(cipher), + aad_prefix_); +} + +} // namespace iceberg diff --git a/src/iceberg/encryption/aes_gcm_stream.h b/src/iceberg/encryption/aes_gcm_stream.h new file mode 100644 index 0000000000..99404fa480 --- /dev/null +++ b/src/iceberg/encryption/aes_gcm_stream.h @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/encryption/aes_gcm_stream.h +/// AES GCM Stream (AGS1) files, see format/gcm-stream-spec.md. + +#include +#include +#include +#include +#include + +#include "iceberg/file_io.h" +#include "iceberg/iceberg_bundle_export.h" +#include "iceberg/result.h" + +namespace iceberg { + +/// \brief AES GCM Stream format constants. +struct AesGcmStream { + static constexpr int32_t kPlainBlockSize = 1024 * 1024; + static constexpr int32_t kNonceLength = 12; + static constexpr int32_t kGcmTagLength = 16; + static constexpr int32_t kCipherBlockSize = + kPlainBlockSize + kNonceLength + kGcmTagLength; + static constexpr std::array kMagic = {'A', 'G', 'S', '1'}; + static constexpr int32_t kHeaderLength = 8; // magic + plain block size (LE int32) + static constexpr int64_t kMinStreamLength = + kHeaderLength + kNonceLength + kGcmTagLength; + + /// \brief Plaintext length of a stream with the given encrypted length. + static int64_t PlaintextLength(int64_t encrypted_length); +}; + +/// \brief An InputFile that decrypts an AGS1 file. +/// +/// The encrypted length must come from a trusted source (the key metadata), so that +/// truncation of the file is detected. +class ICEBERG_BUNDLE_EXPORT AesGcmInputFile : public InputFile { + public: + AesGcmInputFile(std::unique_ptr source, std::vector key, + std::vector aad_prefix, int64_t encrypted_length); + + std::string_view location() const override { return source_->location(); } + Result Size() const override; + Result> Open() override; + + private: + std::unique_ptr source_; + std::vector key_; + std::vector aad_prefix_; + int64_t encrypted_length_; +}; + +/// \brief An OutputFile that writes an AGS1 file. +class ICEBERG_BUNDLE_EXPORT AesGcmOutputFile : public OutputFile { + public: + AesGcmOutputFile(std::unique_ptr target, std::vector key, + std::vector aad_prefix); + + std::string_view location() const override { return target_->location(); } + Result> Create() override; + Result> CreateOrOverwrite() override; + + private: + std::unique_ptr target_; + std::vector key_; + std::vector aad_prefix_; +}; + +} // namespace iceberg diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index a86bc9087b..2b2988573c 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -228,6 +228,10 @@ if(ICEBERG_BUILD_BUNDLE) target_sources(parquet_test PRIVATE parquet_encryption_test.cc) endif() + if(ICEBERG_ENCRYPTION) + add_iceberg_test(crypto_test USE_BUNDLE SOURCES aes_gcm_test.cc) + endif() + add_iceberg_test(scan_test USE_BUNDLE SOURCES @@ -408,6 +412,10 @@ if(ICEBERG_CXX20_COMPAT) ) target_compile_definitions(cpp20_compatibility_test PRIVATE ICEBERG_PUBLIC_HEADERS_BUNDLE) + if(ICEBERG_ENCRYPTION) + target_compile_definitions(cpp20_compatibility_test + PRIVATE ICEBERG_PUBLIC_HEADERS_ENCRYPTION) + endif() endif() if(ICEBERG_BUILD_REST) target_link_libraries(cpp20_compatibility_test diff --git a/src/iceberg/test/aes_gcm_test.cc b/src/iceberg/test/aes_gcm_test.cc new file mode 100644 index 0000000000..fd53c694c8 --- /dev/null +++ b/src/iceberg/test/aes_gcm_test.cc @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "iceberg/arrow/arrow_io_util.h" +#include "iceberg/encryption/aes_gcm_internal.h" +#include "iceberg/encryption/aes_gcm_stream.h" +#include "iceberg/encryption/standard_key_metadata.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/temp_file_test_base.h" + +namespace iceberg { + +namespace { + +using encryption::AesGcmCipher; + +/// \brief Deterministic test bytes: byte i is (i * 31 + seed). +std::vector TestBytes(size_t length, int seed) { + std::vector bytes(length); + for (size_t i = 0; i < length; ++i) { + bytes[i] = static_cast(static_cast(i) * 31 + seed); + } + return bytes; +} + +std::span AsBytes(std::string_view s) { + return {reinterpret_cast(s.data()), s.size()}; +} + +// The fixtures below were produced by Iceberg Java. It used TestBytes(16, 1) as the +// data key, TestBytes(16, 2) as the file AAD prefix and TestBytes(16, 3) as the key +// encryption key. +const std::vector kDataKey = TestBytes(16, 1); +const std::vector kAadPrefix = TestBytes(16, 2); +const std::vector kKek = TestBytes(16, 3); + +// StandardEncryptionManager encrypts manifest list key metadata under the key +// encryption key, with the KEK timestamp as AAD. +constexpr std::string_view kKekTimestamp = "1791455219054"; + +// Manifest list key metadata encrypted by Java under kKek with kKekTimestamp +// as AAD: encryption key TestBytes(16, 1), AAD prefix TestBytes(16, 2), +// file length 4242. +const std::vector kJavaManifestListKeyMetadata = { + 0xf2, 0x80, 0x57, 0xac, 0x14, 0xdc, 0x8d, 0xa7, 0xb2, 0x25, 0x58, 0x24, 0xa3, 0xe1, + 0x88, 0xb0, 0xde, 0x2e, 0x94, 0x09, 0x9b, 0x7d, 0xe4, 0x7d, 0x0e, 0xd6, 0xb0, 0x7a, + 0xb1, 0x86, 0x7b, 0xca, 0x61, 0x11, 0x2a, 0x11, 0x4e, 0xc1, 0xa8, 0x23, 0xd8, 0xa5, + 0xca, 0x43, 0x9c, 0x33, 0x6a, 0xe9, 0x6b, 0xf2, 0xc6, 0xdb, 0xa8, 0xe0, 0x2e, 0x7f, + 0xc0, 0xcc, 0x0e, 0xf0, 0xb7, 0xec, 0xb9, 0x1f, 0x06, 0x3a, 0xe7}; +// AES GCM stream of TestBytes(0, 5) written by Java's AesGcmOutputStream +// with kDataKey and kAadPrefix. +const std::vector kJavaAgs1Empty = { + 0x41, 0x47, 0x53, 0x31, 0x00, 0x00, 0x10, 0x00, 0x92, 0xd6, 0x53, 0x8c, + 0x2b, 0x71, 0x45, 0x37, 0x37, 0x86, 0x76, 0x8f, 0xef, 0xd4, 0xdc, 0x98, + 0x15, 0xf1, 0x93, 0x9d, 0x6c, 0x56, 0x4b, 0xae, 0x24, 0xff, 0xdc, 0xb4}; +// AES GCM stream of TestBytes(1, 5) written by Java's AesGcmOutputStream +// with kDataKey and kAadPrefix. +const std::vector kJavaAgs1OneByte = { + 0x41, 0x47, 0x53, 0x31, 0x00, 0x00, 0x10, 0x00, 0xf8, 0xde, 0xe0, 0x44, 0x00, + 0x8d, 0xbb, 0x2b, 0x6b, 0x29, 0x58, 0xb2, 0x70, 0x62, 0x46, 0xbb, 0xb1, 0xf8, + 0x76, 0x29, 0x69, 0x86, 0x14, 0x6c, 0x08, 0x98, 0x84, 0xdc, 0x3b}; +// AES GCM stream of TestBytes(100, 5) written by Java's AesGcmOutputStream +// with kDataKey and kAadPrefix. +const std::vector kJavaAgs1Hundred = { + 0x41, 0x47, 0x53, 0x31, 0x00, 0x00, 0x10, 0x00, 0x02, 0xb0, 0x1c, 0x1f, 0x7e, 0x37, + 0xeb, 0x85, 0xd3, 0x4b, 0x14, 0x75, 0x91, 0x83, 0xcf, 0x68, 0xde, 0xdd, 0xe0, 0xff, + 0xd3, 0xe0, 0xeb, 0x31, 0x10, 0x4b, 0x95, 0x86, 0x5c, 0x67, 0x1f, 0x68, 0xdf, 0x24, + 0xbd, 0x0a, 0x07, 0xf1, 0x06, 0xc7, 0x13, 0x1e, 0x42, 0xd4, 0x7a, 0xff, 0x87, 0x32, + 0xc3, 0xe5, 0xea, 0x75, 0x6a, 0x53, 0xaa, 0xab, 0xab, 0xe0, 0x81, 0x2c, 0x71, 0x4c, + 0xb4, 0x89, 0x32, 0x89, 0xc6, 0xbb, 0x84, 0x5d, 0x58, 0x93, 0x6b, 0x80, 0xbd, 0xa3, + 0xe6, 0xc3, 0xef, 0x53, 0x83, 0xa4, 0x00, 0x9c, 0x30, 0x02, 0xd4, 0xca, 0x7e, 0x0e, + 0xb3, 0xa3, 0xff, 0x4c, 0x96, 0xe3, 0x31, 0xa1, 0x6a, 0xc1, 0xf2, 0xb7, 0xa0, 0x40, + 0xc4, 0x0b, 0x5c, 0x45, 0x09, 0xb7, 0x35, 0xda, 0x6c, 0x03, 0xce, 0xd5, 0x59, 0xd0, + 0x47, 0xf9, 0x0c, 0x9d, 0x1f, 0x89, 0x2f, 0x1b, 0xcb, 0x44}; + +Result> ReadAll(InputFile& file) { + ICEBERG_ASSIGN_OR_RAISE(auto size, file.Size()); + ICEBERG_ASSIGN_OR_RAISE(auto stream, file.Open()); + std::vector out(size); + if (size > 0) { + ICEBERG_RETURN_UNEXPECTED( + stream->ReadFully(0, std::as_writable_bytes(std::span(out)))); + } + return out; +} + +} // namespace + +TEST(AesGcmCipherTest, RoundTrip) { + for (size_t key_length : {16, 24, 32}) { + ICEBERG_UNWRAP_OR_FAIL(auto cipher, AesGcmCipher::Make(TestBytes(key_length, 1))); + auto plaintext = TestBytes(100, 2); + ICEBERG_UNWRAP_OR_FAIL(auto ciphertext, cipher->Encrypt(plaintext, AsBytes("aad"))); + EXPECT_EQ(ciphertext.size(), plaintext.size() + 28); + ICEBERG_UNWRAP_OR_FAIL(auto decrypted, cipher->Decrypt(ciphertext, AsBytes("aad"))); + EXPECT_EQ(decrypted, plaintext); + + // Random nonces: encrypting twice gives different ciphertexts + ICEBERG_UNWRAP_OR_FAIL(auto again, cipher->Encrypt(plaintext, AsBytes("aad"))); + EXPECT_NE(again, ciphertext); + + // Wrong AAD, tampered ciphertext and truncated ciphertext fail authentication + EXPECT_THAT(cipher->Decrypt(ciphertext, AsBytes("other")), + IsError(ErrorKind::kInvalid)); + ciphertext[20] ^= 1; + EXPECT_THAT(cipher->Decrypt(ciphertext, AsBytes("aad")), + IsError(ErrorKind::kInvalid)); + EXPECT_THAT(cipher->Decrypt(std::span(ciphertext).first(27), AsBytes("aad")), + IsError(ErrorKind::kInvalid)); + } + EXPECT_THAT(AesGcmCipher::Make(TestBytes(15, 1)), IsError(ErrorKind::kInvalidArgument)); +} + +TEST(AesGcmCipherTest, DecryptsJavaManifestListKeyMetadata) { + ICEBERG_UNWRAP_OR_FAIL(auto cipher, AesGcmCipher::Make(kKek)); + ICEBERG_UNWRAP_OR_FAIL(auto decrypted, cipher->Decrypt(kJavaManifestListKeyMetadata, + AsBytes(kKekTimestamp))); + ICEBERG_UNWRAP_OR_FAIL(auto metadata, StandardKeyMetadata::Parse(decrypted)); + EXPECT_EQ(metadata.encryption_key, TestBytes(16, 1)); + EXPECT_EQ(metadata.aad_prefix, TestBytes(16, 2)); + EXPECT_EQ(metadata.file_length, 4242); +} + +class AesGcmStreamTest : public TempFileTestBase { + protected: + void SetUp() override { + TempFileTestBase::SetUp(); + io_ = std::shared_ptr(arrow::MakeLocalFileIO()); + } + + // Write `plain` as an AGS1 stream and return the stored length. + int64_t Write(const std::string& path, std::span plain, + const std::vector& key, const std::vector& aad, + size_t chunk = 1 << 20) { + auto out = io_->NewOutputFile(path); + EXPECT_THAT(out, IsOk()); + AesGcmOutputFile file(std::move(out.value()), key, aad); + auto stream = file.CreateOrOverwrite(); + EXPECT_THAT(stream, IsOk()); + for (size_t pos = 0; pos < plain.size(); pos += chunk) { + auto part = plain.subspan(pos, std::min(chunk, plain.size() - pos)); + EXPECT_THAT((*stream)->Write(std::as_bytes(part)), IsOk()); + } + EXPECT_EQ((*stream)->Position().value(), static_cast(plain.size())); + EXPECT_THAT((*stream)->Close(), IsOk()); + // Java's AesGcmOutputStream keeps reporting the plaintext length once closed + EXPECT_EQ((*stream)->Position().value(), static_cast(plain.size())); + auto stored = (*stream)->StoredLength().value(); + EXPECT_EQ(stored, static_cast(std::filesystem::file_size(path))); + return stored; + } + + std::unique_ptr Open(const std::string& path, + const std::vector& key, + const std::vector& aad, int64_t length) { + auto in = io_->NewInputFile(path); + EXPECT_THAT(in, IsOk()); + return std::make_unique(std::move(in.value()), key, aad, length); + } + + // Materialize an inlined Java fixture as a file the reader can open. + std::string WriteRaw(const std::vector& bytes) { + auto path = CreateNewTempFilePath(); + std::ofstream out(path, std::ios::binary); + out.write(reinterpret_cast(bytes.data()), + static_cast(bytes.size())); + out.close(); + return path; + } + + std::shared_ptr io_; +}; + +TEST_F(AesGcmStreamTest, ReadsJavaVectors) { + // Streams of TestBytes(0, 5), TestBytes(1, 5) and TestBytes(100, 5) written by Java + const std::vector*>> vectors = { + {0, &kJavaAgs1Empty}, {1, &kJavaAgs1OneByte}, {100, &kJavaAgs1Hundred}}; + for (const auto& [length, encrypted] : vectors) { + auto path = WriteRaw(*encrypted); + auto file = Open(path, kDataKey, kAadPrefix, static_cast(encrypted->size())); + ICEBERG_UNWRAP_OR_FAIL(auto plain, ReadAll(*file)); + EXPECT_EQ(plain, TestBytes(length, 5)) << "plaintext length " << length; + } +} + +TEST_F(AesGcmStreamTest, WritesJavaCompatibleLayout) { + // Same plaintext, key and AAD prefix as Java's 100-byte vector: same header and + // length (the ciphertext differs only by the random nonce). + auto path = CreateNewTempFilePath(); + auto stored = Write(path, TestBytes(100, 5), kDataKey, kAadPrefix); + ASSERT_EQ(stored, static_cast(kJavaAgs1Hundred.size())); + std::ifstream in(path, std::ios::binary); + std::vector header(8); + in.read(reinterpret_cast(header.data()), 8); + EXPECT_EQ(header, + std::vector(kJavaAgs1Hundred.begin(), kJavaAgs1Hundred.begin() + 8)); +} + +TEST_F(AesGcmStreamTest, RoundTripAcrossBlocks) { + auto key = TestBytes(32, 7); + auto aad = TestBytes(16, 9); + constexpr size_t kBlock = AesGcmStream::kPlainBlockSize; + for (size_t size : + {size_t{0}, size_t{1}, kBlock - 1, kBlock, kBlock + 1, 2 * kBlock + 17}) { + auto path = CreateNewTempFilePath(); + auto plain = TestBytes(size, 5); + // Odd write sizes exercise writes that straddle blocks + int64_t stored = Write(path, plain, key, aad, /*chunk=*/333331); + size_t blocks = size == 0 ? 1 : (size + kBlock - 1) / kBlock; + EXPECT_EQ(stored, static_cast(8 + size + blocks * 28)); + EXPECT_EQ(AesGcmStream::PlaintextLength(stored), static_cast(size)); + + auto file = Open(path, key, aad, stored); + ICEBERG_UNWRAP_OR_FAIL(auto read, ReadAll(*file)); + EXPECT_EQ(read, plain); + + if (size > kBlock + 10) { + // Random access across a block boundary + ICEBERG_UNWRAP_OR_FAIL(auto stream, file->Open()); + std::vector window(10); + ASSERT_THAT(stream->ReadFully(kBlock - 5, window), IsOk()); + EXPECT_EQ(std::memcmp(window.data(), plain.data() + kBlock - 5, 10), 0); + } + } +} + +TEST_F(AesGcmStreamTest, DetectsTampering) { + auto key = TestBytes(16, 7); + auto aad = TestBytes(16, 9); + auto path = CreateNewTempFilePath(); + auto plain = TestBytes(AesGcmStream::kPlainBlockSize + 100, 5); + int64_t stored = Write(path, plain, key, aad); + + // Wrong key or AAD prefix (a file swapped in from another manifest entry) + EXPECT_THAT(ReadAll(*Open(path, TestBytes(16, 8), aad, stored)), + IsError(ErrorKind::kInvalid)); + EXPECT_THAT(ReadAll(*Open(path, key, TestBytes(16, 10), stored)), + IsError(ErrorKind::kInvalid)); + + // Truncation: the trusted length from the key metadata no longer matches the file + std::filesystem::resize_file(path, 8 + AesGcmStream::kCipherBlockSize); + EXPECT_THAT(ReadAll(*Open(path, key, aad, stored)), IsError(ErrorKind::kIOError)); + // AGS1 blocks do not mark the end of the stream: a truncated file read with its own + // (untrusted) length still decrypts. This is why readers must take the length from + // the key metadata. + auto truncated = Open(path, key, aad, 8 + AesGcmStream::kCipherBlockSize); + ICEBERG_UNWRAP_OR_FAIL(auto first_block, ReadAll(*truncated)); + EXPECT_EQ(first_block.size(), AesGcmStream::kPlainBlockSize); + + // Bad magic + std::filesystem::resize_file(path, 0); + std::ofstream(path, std::ios::binary) << "XXXX0000" << std::string(28, '\0'); + EXPECT_THAT(Open(path, key, aad, 36)->Open(), IsError(ErrorKind::kInvalid)); + // Shorter than an empty stream + EXPECT_THAT(Open(path, key, aad, 35)->Open(), IsError(ErrorKind::kInvalid)); +} + +} // namespace iceberg From 7fa6c020c3fe80a196fdb6b283b05dbb33617350 Mon Sep 17 00:00:00 2001 From: shangxinli Date: Sat, 17 Oct 2026 11:35:27 -0700 Subject: [PATCH 2/4] fix: validate AGS1 encrypted length before constructing the stream AesGcmStream::PlaintextLength now returns a Result and rejects a length shorter than an empty stream, one implying more blocks than the int32 block index in the block AAD can address, and one whose last block is too short to hold a nonce and a tag. Java bounds the block count with Math.toIntExact in the AesGcmInputStream constructor; the C++ side only caught it when a block was decrypted, so AesGcmInputFile::Size() could return a length that could never be read back. --- src/iceberg/encryption/aes_gcm_stream.cc | 49 ++++++++++++++---------- src/iceberg/encryption/aes_gcm_stream.h | 5 ++- src/iceberg/test/aes_gcm_test.cc | 40 ++++++++++++++++++- 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/src/iceberg/encryption/aes_gcm_stream.cc b/src/iceberg/encryption/aes_gcm_stream.cc index 7fb417c115..1dea2bcb7f 100644 --- a/src/iceberg/encryption/aes_gcm_stream.cc +++ b/src/iceberg/encryption/aes_gcm_stream.cc @@ -62,21 +62,23 @@ std::array HeaderBytes() { class AesGcmInputStream : public SeekableInputStream { public: + /// \brief Construct over an encrypted length already accepted by + /// AesGcmStream::PlaintextLength, which bounds the block count. AesGcmInputStream(std::unique_ptr source, std::unique_ptr cipher, std::vector aad_prefix, - int64_t encrypted_length) + int64_t encrypted_length, int64_t plain_size) : source_(std::move(source)), cipher_(std::move(cipher)), aad_prefix_(std::move(aad_prefix)), cipher_block_(kCipherBlock), - plain_block_(kPlainBlock) { + plain_block_(kPlainBlock), + plain_size_(plain_size) { int64_t stream_length = encrypted_length - AesGcmStream::kHeaderLength; int64_t num_full_blocks = stream_length / kCipherBlock; int64_t last_bytes = stream_length - num_full_blocks * kCipherBlock; bool full_blocks_only = last_bytes == 0; num_blocks_ = full_blocks_only ? num_full_blocks : num_full_blocks + 1; last_cipher_block_size_ = full_blocks_only ? kCipherBlock : last_bytes; - plain_size_ = AesGcmStream::PlaintextLength(encrypted_length); } Status ValidateHeader() { @@ -94,9 +96,6 @@ class AesGcmInputStream : public SeekableInputStream { return Invalid("Invalid GCM stream: block size {} != {}", plain_block_size, AesGcmStream::kPlainBlockSize); } - if (last_cipher_block_size_ < kBlockOverhead) { - return Invalid("Invalid GCM stream: truncated last block"); - } return {}; } @@ -146,9 +145,7 @@ class AesGcmInputStream : public SeekableInputStream { if (block_index == current_block_index_) { return {}; } - if (block_index > std::numeric_limits::max()) { - return Invalid("GCM stream block index overflow"); - } + // num_blocks_ is bounded by int32 max at construction, so the cast below is safe. int64_t size = block_index == num_blocks_ - 1 ? last_cipher_block_size_ : kCipherBlock; auto cipher_block = std::span(cipher_block_).first(size); @@ -282,15 +279,28 @@ class AesGcmOutputStream : public PositionOutputStream { } // namespace -int64_t AesGcmStream::PlaintextLength(int64_t encrypted_length) { - int64_t stream_length = encrypted_length - kHeaderLength; - if (stream_length <= 0) { - return 0; +Result AesGcmStream::PlaintextLength(int64_t encrypted_length) { + if (encrypted_length < kMinStreamLength) { + return Invalid( + "Invalid encrypted stream: {} is shorter than the minimum possible stream " + "length {}", + encrypted_length, kMinStreamLength); } + int64_t stream_length = encrypted_length - kHeaderLength; int64_t num_full_blocks = stream_length / kCipherBlockSize; int64_t last_bytes = stream_length - num_full_blocks * kCipherBlockSize; + int64_t num_blocks = last_bytes == 0 ? num_full_blocks : num_full_blocks + 1; + // Block AAD carries the block index as an int32, so Java bounds the block count + // with Math.toIntExact. Reject here rather than when the block is decrypted. + if (num_blocks > std::numeric_limits::max()) { + return Invalid("Invalid encrypted stream: {} bytes exceeds the maximum of {} blocks", + encrypted_length, std::numeric_limits::max()); + } + if (last_bytes != 0 && last_bytes < kNonceLength + kGcmTagLength) { + return Invalid("Invalid GCM stream: truncated last block"); + } int64_t last_plain = last_bytes == 0 ? 0 : last_bytes - kNonceLength - kGcmTagLength; - return num_full_blocks * kPlainBlockSize + std::max(last_plain, 0); + return num_full_blocks * kPlainBlockSize + last_plain; } AesGcmInputFile::AesGcmInputFile(std::unique_ptr source, @@ -307,15 +317,12 @@ Result AesGcmInputFile::Size() const { } Result> AesGcmInputFile::Open() { - if (encrypted_length_ < AesGcmStream::kMinStreamLength) { - return Invalid( - "Invalid encrypted stream: {} is shorter than the minimum possible stream length", - encrypted_length_); - } + ICEBERG_ASSIGN_OR_RAISE(auto plain_size, + AesGcmStream::PlaintextLength(encrypted_length_)); ICEBERG_ASSIGN_OR_RAISE(auto cipher, AesGcmCipher::Make(key_)); ICEBERG_ASSIGN_OR_RAISE(auto source, source_->Open()); - auto stream = std::make_unique(std::move(source), std::move(cipher), - aad_prefix_, encrypted_length_); + auto stream = std::make_unique( + std::move(source), std::move(cipher), aad_prefix_, encrypted_length_, plain_size); ICEBERG_RETURN_UNEXPECTED(stream->ValidateHeader()); return stream; } diff --git a/src/iceberg/encryption/aes_gcm_stream.h b/src/iceberg/encryption/aes_gcm_stream.h index 99404fa480..49d3022a57 100644 --- a/src/iceberg/encryption/aes_gcm_stream.h +++ b/src/iceberg/encryption/aes_gcm_stream.h @@ -47,7 +47,10 @@ struct AesGcmStream { kHeaderLength + kNonceLength + kGcmTagLength; /// \brief Plaintext length of a stream with the given encrypted length. - static int64_t PlaintextLength(int64_t encrypted_length); + /// + /// Returns an error if the length is shorter than an empty stream, implies more + /// than int32 max blocks, or ends in a block too short to hold a nonce and tag. + static Result PlaintextLength(int64_t encrypted_length); }; /// \brief An InputFile that decrypts an AGS1 file. diff --git a/src/iceberg/test/aes_gcm_test.cc b/src/iceberg/test/aes_gcm_test.cc index fd53c694c8..6adebc2c28 100644 --- a/src/iceberg/test/aes_gcm_test.cc +++ b/src/iceberg/test/aes_gcm_test.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -234,7 +235,8 @@ TEST_F(AesGcmStreamTest, RoundTripAcrossBlocks) { int64_t stored = Write(path, plain, key, aad, /*chunk=*/333331); size_t blocks = size == 0 ? 1 : (size + kBlock - 1) / kBlock; EXPECT_EQ(stored, static_cast(8 + size + blocks * 28)); - EXPECT_EQ(AesGcmStream::PlaintextLength(stored), static_cast(size)); + ICEBERG_UNWRAP_OR_FAIL(auto plain_size, AesGcmStream::PlaintextLength(stored)); + EXPECT_EQ(plain_size, static_cast(size)); auto file = Open(path, key, aad, stored); ICEBERG_UNWRAP_OR_FAIL(auto read, ReadAll(*file)); @@ -281,4 +283,40 @@ TEST_F(AesGcmStreamTest, DetectsTampering) { EXPECT_THAT(Open(path, key, aad, 35)->Open(), IsError(ErrorKind::kInvalid)); } +TEST_F(AesGcmStreamTest, RejectsInvalidEncryptedLength) { + constexpr int64_t kHeader = AesGcmStream::kHeaderLength; + constexpr int64_t kCipherBlock = AesGcmStream::kCipherBlockSize; + constexpr int64_t kOverhead = AesGcmStream::kNonceLength + AesGcmStream::kGcmTagLength; + + // Shorter than an empty stream + EXPECT_THAT(AesGcmStream::PlaintextLength(-1), IsError(ErrorKind::kInvalid)); + EXPECT_THAT(AesGcmStream::PlaintextLength(0), IsError(ErrorKind::kInvalid)); + EXPECT_THAT(AesGcmStream::PlaintextLength(AesGcmStream::kMinStreamLength - 1), + IsError(ErrorKind::kInvalid)); + EXPECT_THAT(AesGcmStream::PlaintextLength(AesGcmStream::kMinStreamLength), + HasValue(::testing::Eq(0))); + + // A last block too short to hold a nonce and a tag + EXPECT_THAT(AesGcmStream::PlaintextLength(kHeader + kCipherBlock + kOverhead - 1), + IsError(ErrorKind::kInvalid)); + EXPECT_THAT(AesGcmStream::PlaintextLength(kHeader + kCipherBlock + kOverhead), + HasValue(::testing::Eq(AesGcmStream::kPlainBlockSize))); + + // More blocks than the int32 block index in the block AAD can address. The check + // must happen here rather than when a block is decrypted, so that a caller asking + // only for the size never sees a length it could not read back. + constexpr int64_t kMaxBlocks = std::numeric_limits::max(); + EXPECT_THAT(AesGcmStream::PlaintextLength(kHeader + kMaxBlocks * kCipherBlock), + HasValue(::testing::Eq(kMaxBlocks * AesGcmStream::kPlainBlockSize))); + EXPECT_THAT(AesGcmStream::PlaintextLength(kHeader + kMaxBlocks * kCipherBlock + 1), + IsError(ErrorKind::kInvalid)); + + // Both Size() and Open() reject it, rather than Open() alone + auto path = CreateNewTempFilePath(); + Write(path, TestBytes(10, 5), kDataKey, kAadPrefix); + auto file = Open(path, kDataKey, kAadPrefix, kHeader + kMaxBlocks * kCipherBlock + 1); + EXPECT_THAT(file->Size(), IsError(ErrorKind::kInvalid)); + EXPECT_THAT(file->Open(), IsError(ErrorKind::kInvalid)); +} + } // namespace iceberg From cf549f1809926ab8bfd21565db4b5b65cfb51547 Mon Sep 17 00:00:00 2001 From: shangxinli Date: Mon, 19 Oct 2026 08:09:34 -0700 Subject: [PATCH 3/4] fix: authenticate empty AGS1 streams and harden the ReadFully range check An empty AGS1 file still carries one block with a nonce and a tag, which the writer emits like Java does, but the reader stopped at a plaintext length of zero and never authenticated it. A wrong key, a wrong AAD prefix, a corrupt tag or a file truncated behind its trusted length was accepted as an empty file. Java authenticates that block on the first read; an empty stream is never read from, so it is authenticated when the stream is opened. ReadFully added the output length to the position before bounding it, so a position near int64 max was signed overflow before Seek could reject it. The position is now bounded first and the length compared against the remainder. AesGcmStream::PlaintextLength is defined out of line in a bundle source built with hidden visibility, so it needs ICEBERG_BUNDLE_EXPORT to link from outside the shared bundle. The static-first tests did not cover this. --- src/iceberg/encryption/aes_gcm_stream.cc | 20 ++++++++- src/iceberg/encryption/aes_gcm_stream.h | 2 +- src/iceberg/test/aes_gcm_test.cc | 54 ++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/src/iceberg/encryption/aes_gcm_stream.cc b/src/iceberg/encryption/aes_gcm_stream.cc index 1dea2bcb7f..7fb545dbf9 100644 --- a/src/iceberg/encryption/aes_gcm_stream.cc +++ b/src/iceberg/encryption/aes_gcm_stream.cc @@ -99,6 +99,20 @@ class AesGcmInputStream : public SeekableInputStream { return {}; } + /// \brief Authenticate the sole block of an empty stream. + /// + /// An empty AGS1 file still carries one empty block with a nonce and a tag, so a + /// reader that stops at plain_size_ == 0 would accept a wrong key, a wrong AAD + /// prefix, a corrupt tag or a file truncated behind its trusted length. Java + /// authenticates the block on the first read; an empty stream is never read from, + /// so it is authenticated when the stream is opened. + Status ValidateEmptyStream() { + if (plain_size_ != 0) { + return {}; + } + return DecryptBlock(0); + } + Result Position() const override { return position_; } Status Seek(int64_t position) override { @@ -126,7 +140,10 @@ class AesGcmInputStream : public SeekableInputStream { } Status ReadFully(int64_t position, std::span out) override { - if (position < 0 || position + static_cast(out.size()) > plain_size_) { + // The position is bounded before the length is added to it, so that a position + // near int64 max does not overflow the range check. + if (position < 0 || position > plain_size_ || + std::cmp_greater(out.size(), plain_size_ - position)) { return IOError("Cannot read {} bytes at {} from stream of length {}", out.size(), position, plain_size_); } @@ -324,6 +341,7 @@ Result> AesGcmInputFile::Open() { auto stream = std::make_unique( std::move(source), std::move(cipher), aad_prefix_, encrypted_length_, plain_size); ICEBERG_RETURN_UNEXPECTED(stream->ValidateHeader()); + ICEBERG_RETURN_UNEXPECTED(stream->ValidateEmptyStream()); return stream; } diff --git a/src/iceberg/encryption/aes_gcm_stream.h b/src/iceberg/encryption/aes_gcm_stream.h index 49d3022a57..63b5c13933 100644 --- a/src/iceberg/encryption/aes_gcm_stream.h +++ b/src/iceberg/encryption/aes_gcm_stream.h @@ -50,7 +50,7 @@ struct AesGcmStream { /// /// Returns an error if the length is shorter than an empty stream, implies more /// than int32 max blocks, or ends in a block too short to hold a nonce and tag. - static Result PlaintextLength(int64_t encrypted_length); + ICEBERG_BUNDLE_EXPORT static Result PlaintextLength(int64_t encrypted_length); }; /// \brief An InputFile that decrypts an AGS1 file. diff --git a/src/iceberg/test/aes_gcm_test.cc b/src/iceberg/test/aes_gcm_test.cc index 6adebc2c28..9b21a2c493 100644 --- a/src/iceberg/test/aes_gcm_test.cc +++ b/src/iceberg/test/aes_gcm_test.cc @@ -283,6 +283,60 @@ TEST_F(AesGcmStreamTest, DetectsTampering) { EXPECT_THAT(Open(path, key, aad, 35)->Open(), IsError(ErrorKind::kInvalid)); } +TEST_F(AesGcmStreamTest, AuthenticatesEmptyStream) { + auto key = TestBytes(16, 7); + auto aad = TestBytes(16, 9); + auto path = CreateNewTempFilePath(); + int64_t stored = Write(path, {}, key, aad); + ASSERT_EQ(stored, AesGcmStream::kMinStreamLength); + + // An empty stream still carries one block with a nonce and a tag, so opening it + // authenticates that block even though there is no plaintext to read. + ICEBERG_UNWRAP_OR_FAIL(auto plain, ReadAll(*Open(path, key, aad, stored))); + EXPECT_TRUE(plain.empty()); + + // Wrong key or AAD prefix (a file swapped in from another manifest entry) + EXPECT_THAT(Open(path, TestBytes(16, 8), aad, stored)->Open(), + IsError(ErrorKind::kInvalid)); + EXPECT_THAT(Open(path, key, TestBytes(16, 10), stored)->Open(), + IsError(ErrorKind::kInvalid)); + + // Truncated to its header behind a trusted length that says otherwise + std::filesystem::resize_file(path, AesGcmStream::kHeaderLength); + EXPECT_THAT(Open(path, key, aad, stored)->Open(), IsError(ErrorKind::kIOError)); + + // A corrupt tag + auto corrupt = WriteRaw(kJavaAgs1Empty); + { + std::fstream out(corrupt, std::ios::binary | std::ios::in | std::ios::out); + out.seekp(static_cast(kJavaAgs1Empty.size() - 1)); + out.put(static_cast(kJavaAgs1Empty.back() ^ 0xFF)); + } + EXPECT_THAT( + Open(corrupt, kDataKey, kAadPrefix, static_cast(kJavaAgs1Empty.size())) + ->Open(), + IsError(ErrorKind::kInvalid)); +} + +TEST_F(AesGcmStreamTest, RejectsOutOfRangeReadFully) { + auto key = TestBytes(16, 7); + auto aad = TestBytes(16, 9); + auto path = CreateNewTempFilePath(); + int64_t stored = Write(path, TestBytes(100, 5), key, aad); + ICEBERG_UNWRAP_OR_FAIL(auto stream, Open(path, key, aad, stored)->Open()); + + // A position near int64 max must be rejected without overflowing the range check + std::vector one(1); + EXPECT_THAT(stream->ReadFully(std::numeric_limits::max(), one), + IsError(ErrorKind::kIOError)); + EXPECT_THAT(stream->ReadFully(-1, one), IsError(ErrorKind::kIOError)); + + // Reading up to the last byte is allowed; one byte past it is not + EXPECT_THAT(stream->ReadFully(99, one), IsOk()); + EXPECT_THAT(stream->ReadFully(100, one), IsError(ErrorKind::kIOError)); + EXPECT_THAT(stream->ReadFully(100, std::span{}), IsOk()); +} + TEST_F(AesGcmStreamTest, RejectsInvalidEncryptedLength) { constexpr int64_t kHeader = AesGcmStream::kHeaderLength; constexpr int64_t kCipherBlock = AesGcmStream::kCipherBlockSize; From 7466520dd73d0eb24ff6feba2ea22d775a3c7dd3 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Wed, 21 Oct 2026 15:23:34 +0800 Subject: [PATCH 4/4] build(encryption): align public header installation and checks Read header patterns and optional groups from the CMake installation rules so AGS1 headers are installed only with the encryption bundle. Rename the AES-GCM source, label OpenSSL null arguments, and document the Java-compatible format constraints. --- ci/scripts/generate_public_headers.py | 65 +++++++++++-------- cmake_modules/IcebergBuildUtils.cmake | 3 +- src/iceberg/CMakeLists.txt | 3 +- src/iceberg/encryption/CMakeLists.txt | 10 ++- .../{aes_gcm_internal.cc => aes_gcm.cc} | 23 ++++--- src/iceberg/encryption/aes_gcm_stream.h | 7 ++ .../encryption/standard_key_metadata.h | 4 ++ 7 files changed, 74 insertions(+), 41 deletions(-) rename src/iceberg/encryption/{aes_gcm_internal.cc => aes_gcm.cc} (89%) diff --git a/ci/scripts/generate_public_headers.py b/ci/scripts/generate_public_headers.py index 5a515871c9..d4b80be824 100755 --- a/ci/scripts/generate_public_headers.py +++ b/ci/scripts/generate_public_headers.py @@ -19,20 +19,21 @@ """Regenerate src/iceberg/cpp20_compatibility_internal.h. -The generated header includes every public header, i.e. every header that -iceberg_install_all_headers() installs: *.h files in a directory whose -CMakeLists.txt calls it, except those with "internal" in their name. This script -applies the same rule, so the list never has to be maintained by hand. The -generate-public-headers pre-commit hook runs it and fails when the checked-in -header is out of date. - -Headers of optional components are wrapped in a macro that -cpp20_compatibility_test defines only when that component is built. +The generated header covers all possible public headers installed by +iceberg_install_all_headers(): *.h and *.hpp files matching its PATTERN arguments, +except those with "internal" in their name. The generate-public-headers pre-commit +hook runs this script and fails when the checked-in header is out of date. + +Headers of optional components are grouped by directory or an explicit +HEADER_GROUP in the installation call. cpp20_compatibility_test defines each +group's macro only when the corresponding component is built. """ from __future__ import annotations +import fnmatch import re +import shlex import sys from pathlib import Path @@ -51,14 +52,7 @@ ("iceberg/parquet", "ICEBERG_PUBLIC_HEADERS_BUNDLE"), ] -# Individual headers of an optional component that lives in a directory which is -# otherwise always built. iceberg/encryption holds both the metadata types, which are -# part of core, and the crypto implementation, which needs ICEBERG_ENCRYPTION. -OPTIONAL_HEADERS = { - "iceberg/encryption/aes_gcm_stream.h": "ICEBERG_PUBLIC_HEADERS_ENCRYPTION", -} - -INSTALL_CALL = re.compile(r"iceberg_install_all_headers\(\s*([^\s)]+)") +INSTALL_CALL = re.compile(r"iceberg_install_all_headers\((.*?)\)", re.DOTALL) LICENSE = """\ /* @@ -82,9 +76,9 @@ """ -def component_of(install_path: str, header_path: str | None = None) -> str | None: - if header_path is not None and header_path in OPTIONAL_HEADERS: - return OPTIONAL_HEADERS[header_path] +def component_of(install_path: str, header_group: str | None = None) -> str | None: + if header_group: + return header_group for prefix, macro in OPTIONAL_COMPONENTS: if install_path == prefix or install_path.startswith(prefix + "/"): return macro @@ -94,14 +88,29 @@ def component_of(install_path: str, header_path: str | None = None) -> str | Non def collect_public_headers() -> dict[str | None, list[str]]: headers: dict[str | None, list[str]] = {} for cmake_file in sorted(SRC.rglob("CMakeLists.txt")): - for install_path in INSTALL_CALL.findall(cmake_file.read_text()): + for call in INSTALL_CALL.findall(cmake_file.read_text()): + tokens = shlex.split(call, comments=True) + if not tokens: + continue + install_path = tokens[0] + try: + pattern_start = tokens.index("PATTERN") + 1 + except ValueError: + pattern_start = len(tokens) + try: + group_start = tokens.index("HEADER_GROUP") + header_group = tokens[group_start + 1] + except ValueError: + group_start = len(tokens) + header_group = None + patterns = tokens[pattern_start:group_start] or ["*.h", "*.hpp"] + group = component_of(install_path, header_group) for header in cmake_file.parent.iterdir(): - if header.suffix != ".h" or "internal" in header.name: + if not any(fnmatch.fnmatch(header.name, pattern) for pattern in patterns): + continue + if header.suffix not in {".h", ".hpp"} or "internal" in header.name: continue - header_path = f"{install_path}/{header.name}" - headers.setdefault(component_of(install_path, header_path), []).append( - header_path - ) + headers.setdefault(group, []).append(f"{install_path}/{header.name}") return headers @@ -117,7 +126,9 @@ def render(headers: dict[str | None, list[str]]) -> str: "", ] lines += [f'#include "{h}"' for h in sorted(headers.get(None, []))] - optional_macros = {m for _, m in OPTIONAL_COMPONENTS} | set(OPTIONAL_HEADERS.values()) + optional_macros = {m for _, m in OPTIONAL_COMPONENTS} | { + group for group in headers if group is not None + } for macro in sorted(optional_macros): if macro not in headers: continue diff --git a/cmake_modules/IcebergBuildUtils.cmake b/cmake_modules/IcebergBuildUtils.cmake index 035c312dae..fa1accd253 100644 --- a/cmake_modules/IcebergBuildUtils.cmake +++ b/cmake_modules/IcebergBuildUtils.cmake @@ -275,7 +275,8 @@ endfunction() function(iceberg_install_all_headers PATH) set(options) - set(one_value_args) + # HEADER_GROUP labels optional headers for ci/scripts/generate_public_headers.py. + set(one_value_args HEADER_GROUP) set(multi_value_args PATTERN) cmake_parse_arguments(ARG "${options}" diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index d3133adc25..af6189ea0b 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -327,8 +327,7 @@ if(ICEBERG_BUILD_BUNDLE) parquet/row_number_reader.cc) if(ICEBERG_ENCRYPTION) - list(APPEND ICEBERG_BUNDLE_SOURCES encryption/aes_gcm_internal.cc - encryption/aes_gcm_stream.cc) + list(APPEND ICEBERG_BUNDLE_SOURCES encryption/aes_gcm.cc encryption/aes_gcm_stream.cc) endif() # Libraries to link with exported libiceberg_bundle.{so,a}. diff --git a/src/iceberg/encryption/CMakeLists.txt b/src/iceberg/encryption/CMakeLists.txt index c493ee250e..73072dbc14 100644 --- a/src/iceberg/encryption/CMakeLists.txt +++ b/src/iceberg/encryption/CMakeLists.txt @@ -15,4 +15,12 @@ # specific language governing permissions and limitations # under the License. -iceberg_install_all_headers(iceberg/encryption) +iceberg_install_all_headers(iceberg/encryption PATTERN encrypted_key.h + standard_key_metadata.h) +if(ICEBERG_BUILD_BUNDLE AND ICEBERG_ENCRYPTION) + iceberg_install_all_headers(iceberg/encryption + PATTERN + aes_gcm_stream.h + HEADER_GROUP + ICEBERG_PUBLIC_HEADERS_ENCRYPTION) +endif() diff --git a/src/iceberg/encryption/aes_gcm_internal.cc b/src/iceberg/encryption/aes_gcm.cc similarity index 89% rename from src/iceberg/encryption/aes_gcm_internal.cc rename to src/iceberg/encryption/aes_gcm.cc index 63fe630365..eefb7654bb 100644 --- a/src/iceberg/encryption/aes_gcm_internal.cc +++ b/src/iceberg/encryption/aes_gcm.cc @@ -17,13 +17,12 @@ * under the License. */ -#include "iceberg/encryption/aes_gcm_internal.h" - #include #include #include +#include "iceberg/encryption/aes_gcm_internal.h" #include "iceberg/util/macros.h" namespace iceberg::encryption { @@ -89,13 +88,15 @@ Status AesGcmCipher::Encrypt(std::span plaintext, auto* ctx = Ctx(ctx_); int len = 0; - if (EVP_EncryptInit_ex(ctx, CipherForKey(key_.size()), nullptr, nullptr, nullptr) != + if (EVP_EncryptInit_ex(ctx, CipherForKey(key_.size()), /*impl=*/nullptr, + /*key=*/nullptr, /*iv=*/nullptr) != 1 || + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, kNonceLength, /*ptr=*/nullptr) != 1 || - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, kNonceLength, nullptr) != 1 || - EVP_EncryptInit_ex(ctx, nullptr, nullptr, key_.data(), nonce.data()) != 1) { + EVP_EncryptInit_ex(ctx, /*cipher=*/nullptr, /*impl=*/nullptr, key_.data(), + nonce.data()) != 1) { return IOError("Failed to initialize AES GCM encryption"); } - if (!aad.empty() && EVP_EncryptUpdate(ctx, nullptr, &len, aad.data(), + if (!aad.empty() && EVP_EncryptUpdate(ctx, /*out=*/nullptr, &len, aad.data(), static_cast(aad.size())) != 1) { return IOError("Failed to set AES GCM AAD"); } @@ -130,13 +131,15 @@ Status AesGcmCipher::Decrypt(std::span ciphertext, auto* ctx = Ctx(ctx_); int len = 0; - if (EVP_DecryptInit_ex(ctx, CipherForKey(key_.size()), nullptr, nullptr, nullptr) != + if (EVP_DecryptInit_ex(ctx, CipherForKey(key_.size()), /*impl=*/nullptr, + /*key=*/nullptr, /*iv=*/nullptr) != 1 || + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, kNonceLength, /*ptr=*/nullptr) != 1 || - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, kNonceLength, nullptr) != 1 || - EVP_DecryptInit_ex(ctx, nullptr, nullptr, key_.data(), nonce.data()) != 1) { + EVP_DecryptInit_ex(ctx, /*cipher=*/nullptr, /*impl=*/nullptr, key_.data(), + nonce.data()) != 1) { return IOError("Failed to initialize AES GCM decryption"); } - if (!aad.empty() && EVP_DecryptUpdate(ctx, nullptr, &len, aad.data(), + if (!aad.empty() && EVP_DecryptUpdate(ctx, /*out=*/nullptr, &len, aad.data(), static_cast(aad.size())) != 1) { return IOError("Failed to set AES GCM AAD"); } diff --git a/src/iceberg/encryption/aes_gcm_stream.h b/src/iceberg/encryption/aes_gcm_stream.h index 63b5c13933..766eb85eb4 100644 --- a/src/iceberg/encryption/aes_gcm_stream.h +++ b/src/iceberg/encryption/aes_gcm_stream.h @@ -35,6 +35,13 @@ namespace iceberg { /// \brief AES GCM Stream format constants. +/// +/// AGS1 stores the block length in its header. This implementation only reads and +/// writes 1 MiB blocks, matching Java, rather than supporting other block lengths. +/// +/// For Java compatibility, an empty file contains one zero-length plaintext block +/// with a nonce and tag. The AGS1 spec describes a non-empty last block and does not +/// explicitly define this empty-file representation. struct AesGcmStream { static constexpr int32_t kPlainBlockSize = 1024 * 1024; static constexpr int32_t kNonceLength = 12; diff --git a/src/iceberg/encryption/standard_key_metadata.h b/src/iceberg/encryption/standard_key_metadata.h index ebad2d80a2..7e788eae56 100644 --- a/src/iceberg/encryption/standard_key_metadata.h +++ b/src/iceberg/encryption/standard_key_metadata.h @@ -38,6 +38,10 @@ namespace iceberg { /// and (encrypted by a key encryption key) in the table metadata `encryption-keys` for /// manifest lists. /// +/// The Iceberg table spec leaves the key metadata representation to the encryption +/// scheme; it mandates neither this encoding nor AES-GCM. The schema below follows +/// Java's standard scheme. +/// /// The serialized form is compatible with Java's StandardKeyMetadata: a one-byte schema /// version (1) followed by an Avro binary encoded record with the fields /// `encryption_key: bytes`, `aad_prefix: union{null, bytes}` and