diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 08de48a71..513570461 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -87,7 +87,7 @@ jobs: - name: Install dependencies on Ubuntu if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} shell: bash - run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libjitterentropy3-dev + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev libjitterentropy3-dev - name: Cache vcpkg packages if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }} uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 5ca6c4b19..be59a610d 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -60,7 +60,7 @@ jobs: shell: bash run: | sudo apt-get update - sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev + sudo apt-get install -y libcurl4-openssl-dev libssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index ba494db46..6cbcbeacb 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -104,7 +104,7 @@ jobs: - name: Install dependencies on Ubuntu if: ${{ startsWith(matrix.os, 'ubuntu') }} shell: bash - run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev - name: Verify run: | diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index 343e661ef..98612c6c3 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -51,7 +51,7 @@ jobs: persist-credentials: false - name: Install dependencies shell: bash - run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 91b5d60e9..28ddaae32 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -96,7 +96,7 @@ jobs: if: ${{ startsWith(matrix.runs-on, 'windows') && steps.vcpkg-cache.outputs.cache-hit != 'true' }} shell: pwsh run: | - vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows + vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows openssl:x64-windows - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eb289f077..a85e6f24a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,10 +53,10 @@ jobs: matrix: include: - cmake_build_type: Debug - extra_cmake_args: "-DICEBERG_CXX20_COMPAT=ON" + extra_cmake_args: "-DICEBERG_CXX20_COMPAT=ON -DICEBERG_ENCRYPTION=ON" api_mode: C++20 Compat - cmake_build_type: RelWithDebInfo - extra_cmake_args: "" + extra_cmake_args: "-DICEBERG_ENCRYPTION=ON" api_mode: C++23 steps: - name: Checkout iceberg-cpp @@ -65,7 +65,7 @@ jobs: persist-credentials: false - name: Install dependencies shell: bash - run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev - name: Set up sccache uses: ./.github/actions/setup-sccache with: @@ -130,7 +130,7 @@ jobs: persist-credentials: false - name: Install dependencies shell: bash - run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev - name: Restore sccache cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -179,7 +179,8 @@ jobs: - name: Build Iceberg shell: bash env: - ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON" + # Enables the Homebrew keg-only OpenSSL lookup, which only runs on macOS. + ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON -DICEBERG_ENCRYPTION=ON" run: | ci/scripts/build_iceberg.sh \ --source-dir "$(pwd)" \ @@ -231,7 +232,7 @@ jobs: if: ${{ steps.vcpkg-cache.outputs.cache-hit != 'true' }} shell: pwsh run: | - vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows + vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows openssl:x64-windows - name: Set up sccache uses: ./.github/actions/setup-sccache with: @@ -239,7 +240,7 @@ jobs: - name: Build Iceberg shell: pwsh env: - ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON" + ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON -DICEBERG_ENCRYPTION=ON" run: | $ErrorActionPreference = "Stop" bash -lc ' diff --git a/CMakeLists.txt b/CMakeLists.txt index 315af8077..7a9c3b2ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,8 @@ option(ICEBERG_BUILD_SHARED "Build shared library" OFF) option(ICEBERG_BUILD_TESTS "Build tests" ON) option(ICEBERG_BUILD_BENCHMARKS "Build benchmarks" OFF) option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON) +option(ICEBERG_ENCRYPTION + "Build table encryption support (Parquet modular encryption, AES GCM streams)" OFF) option(ICEBERG_BUILD_REST "Build rest catalog client" ON) option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration tests" OFF) option(ICEBERG_BUILD_HIVE "Build hive (HMS) catalog client" OFF) diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index aa1a88e00..fef63f651 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -163,7 +163,28 @@ endmacro() # ---------------------------------------------------------------------- # Apache Arrow +# Find OpenSSL for table encryption. Called before Arrow is configured, so that the +# imported OpenSSL targets are visible to the vendored Arrow subdirectories. +macro(resolve_openssl_for_encryption) + if(APPLE + AND NOT DEFINED OPENSSL_ROOT_DIR + AND NOT DEFINED ENV{OPENSSL_ROOT_DIR}) + # Homebrew's OpenSSL is keg-only and not on the default search path. + execute_process(COMMAND brew --prefix openssl@3 + OUTPUT_VARIABLE ICEBERG_BREW_OPENSSL_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(ICEBERG_BREW_OPENSSL_PREFIX AND EXISTS "${ICEBERG_BREW_OPENSSL_PREFIX}") + set(OPENSSL_ROOT_DIR "${ICEBERG_BREW_OPENSSL_PREFIX}") + endif() + endif() + find_package(OpenSSL REQUIRED) +endmacro() + function(resolve_arrow_dependency) + if(ICEBERG_ENCRYPTION) + resolve_openssl_for_encryption() + endif() + prepare_fetchcontent() # Prevent Arrow from injecting -Werror into CMAKE_CXX_FLAGS_DEBUG via @@ -178,12 +199,21 @@ function(resolve_arrow_dependency) set(ARROW_S3 ${ICEBERG_S3}) set(ARROW_JSON ON) set(ARROW_PARQUET ON) + if(ICEBERG_ENCRYPTION) + # Parquet modular encryption, used for encrypted Iceberg data and delete files. + set(PARQUET_REQUIRE_ENCRYPTION ON) + endif() set(ARROW_ENABLE_THREADING ON) set(ARROW_SIMD_LEVEL "NONE") set(ARROW_RUNTIME_SIMD_LEVEL "NONE") set(ARROW_POSITION_INDEPENDENT_CODE ON) set(ARROW_DEPENDENCY_SOURCE "BUNDLED") set(ARROW_WITH_ZLIB ON) + # zstd is the default value of the write.parquet.compression-codec table property, + # so a vendored Arrow without it cannot write Iceberg's default Parquet files. + # Not required by modular encryption itself; parquet_encryption_test covers the + # encrypted-plus-zstd combination to keep the two features verified together. + set(ARROW_WITH_ZSTD ON) if(ICEBERG_S3 AND NOT ICEBERG_AWSSDK_BUNDLED) set(AWSSDK_SOURCE "SYSTEM") endif() @@ -217,6 +247,25 @@ function(resolve_arrow_dependency) fetchcontent_makeavailable(VendoredArrow) if(vendoredarrow_SOURCE_DIR) + # Arrow's Parquet targets do not inherit the OpenSSL include directory when + # Arrow is vendored; add it for the Parquet encryption sources. Depending on the + # platform and build type, they are compiled in the object library or directly in + # the static/shared library. The bundle library links OpenSSL::Crypto. + # + # Only vendored targets are touched here. With system Arrow the same names can + # be ALIAS targets created by a parent project, and target_include_directories + # cannot modify an ALIAS. + if(ICEBERG_ENCRYPTION) + foreach(parquet_target parquet_objlib parquet_static parquet_shared) + if(TARGET ${parquet_target}) + get_target_property(parquet_target_alias ${parquet_target} ALIASED_TARGET) + if(NOT parquet_target_alias) + target_include_directories(${parquet_target} PRIVATE ${OPENSSL_INCLUDE_DIR}) + endif() + endif() + endforeach() + endif() + if(NOT TARGET Arrow::arrow_static) add_library(Arrow::arrow_static INTERFACE IMPORTED) target_link_libraries(Arrow::arrow_static INTERFACE arrow_static) @@ -268,6 +317,12 @@ function(resolve_arrow_dependency) list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Arrow Parquet) endif() + # The bundle exports OpenSSL::Crypto in its link interface for both vendored and + # system Arrow, so consumers must find OpenSSL in both modes. + if(ICEBERG_ENCRYPTION) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL) + endif() + set(ICEBERG_SYSTEM_DEPENDENCIES ${ICEBERG_SYSTEM_DEPENDENCIES} PARENT_SCOPE) diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 657d487c2..6939b2bef 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -360,6 +360,16 @@ if(ICEBERG_BUILD_BUNDLE) "$,iceberg::avrocpp_s,$,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>>" ) + if(ICEBERG_ENCRYPTION) + foreach(libs + ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS + ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS + ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS + ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS) + list(APPEND ${libs} OpenSSL::Crypto) + endforeach() + endif() + add_iceberg_lib(iceberg_bundle SOURCES ${ICEBERG_BUNDLE_SOURCES} diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 659db4f61..3a38db35d 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -221,6 +221,9 @@ if(ICEBERG_BUILD_BUNDLE) parquet_metrics_test.cc parquet_schema_test.cc parquet_test.cc) + if(ICEBERG_ENCRYPTION) + target_sources(parquet_test PRIVATE parquet_encryption_test.cc) + endif() add_iceberg_test(scan_test USE_BUNDLE diff --git a/src/iceberg/test/parquet_encryption_test.cc b/src/iceberg/test/parquet_encryption_test.cc new file mode 100644 index 000000000..10ec64f84 --- /dev/null +++ b/src/iceberg/test/parquet_encryption_test.cc @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace iceberg::parquet { + +namespace { + +const std::string kKey = "0123456789012345"; +const std::string kAadPrefix = "file-aad-prefix-"; + +std::shared_ptr<::arrow::Table> MakeTable() { + auto schema = ::arrow::schema({::arrow::field("id", ::arrow::int64())}); + auto ids = + ::arrow::json::ArrayFromJSONString(::arrow::int64(), "[1, 2, 3]").ValueOrDie(); + return ::arrow::Table::Make(schema, {ids}); +} + +std::shared_ptr<::arrow::Buffer> WriteEncrypted(::parquet::Compression::type codec) { + // Same settings as Iceberg: footer key only (encrypted footer), AAD prefix supplied + // by the reader from the manifest instead of stored in the file. + ::parquet::FileEncryptionProperties::Builder encryption{ + ::arrow::util::SecureString(std::string(kKey))}; + encryption.aad_prefix(kAadPrefix); + encryption.disable_aad_prefix_storage(); + auto properties = ::parquet::WriterProperties::Builder() + .compression(codec) + ->encryption(encryption.build()) + ->build(); + + auto sink = ::arrow::io::BufferOutputStream::Create().ValueOrDie(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( + *MakeTable(), ::arrow::default_memory_pool(), sink, 1024, properties)); + return sink->Finish().ValueOrDie(); +} + +std::shared_ptr<::arrow::Table> Read(const std::shared_ptr<::arrow::Buffer>& buffer, + const std::string& aad_prefix) { + ::parquet::FileDecryptionProperties::Builder decryption; + decryption.footer_key(::arrow::util::SecureString(std::string(kKey))); + decryption.aad_prefix(aad_prefix); + ::parquet::ReaderProperties properties; + properties.file_decryption_properties(decryption.build()); + + auto reader = ::parquet::ParquetFileReader::Open( + std::make_shared<::arrow::io::BufferReader>(buffer), properties); + auto arrow_reader = ::parquet::arrow::FileReader::Make(::arrow::default_memory_pool(), + std::move(reader)) + .ValueOrDie(); + return arrow_reader->ReadTable().ValueOrDie(); +} + +} // namespace + +TEST(ParquetEncryptionTest, ModularEncryptionIsAvailable) { + for (auto codec : + {::parquet::Compression::UNCOMPRESSED, ::parquet::Compression::ZSTD}) { + auto buffer = WriteEncrypted(codec); + // Encrypted footer mode + ASSERT_EQ(std::string(reinterpret_cast(buffer->data()), 4), "PARE"); + EXPECT_TRUE(Read(buffer, kAadPrefix)->Equals(*MakeTable())); + // The AAD prefix binds the file to its manifest entry + EXPECT_THROW(Read(buffer, "another-prefix--"), ::parquet::ParquetException); + } +} + +} // namespace iceberg::parquet