From cb4513d926ac861892aaa2614052c180353c0f05 Mon Sep 17 00:00:00 2001 From: alice-shang Date: Thu, 8 Oct 2026 20:51:29 +0900 Subject: [PATCH 1/4] build: enable Parquet modular encryption and zstd in vendored Arrow Table encryption needs Parquet modular encryption for data and delete files, and AES-GCM for the AGS1 streams of manifests and manifest lists. - Add the ICEBERG_ENCRYPTION option (default ON). When enabled, find OpenSSL before the vendored Arrow is configured (with a Homebrew hint on macOS), build Arrow with PARQUET_REQUIRE_ENCRYPTION, and link the bundle library with OpenSSL::Crypto. - Build the vendored Arrow with zstd, the default Parquet codec of both Java and C++, so tables written by Java can be read. - Record the OpenSSL root in iceberg-config.cmake for consumers. - Install OpenSSL in CI (libssl-dev on Ubuntu, vcpkg on Windows). - Add a smoke test for encrypted Parquet with the settings Iceberg uses (encrypted footer, AAD prefix not stored in the file). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/aws_test.yml | 2 +- .github/workflows/cpp-linter.yml | 2 +- .github/workflows/rc.yml | 2 +- .github/workflows/sanitizer_test.yml | 2 +- .github/workflows/sql_catalog_test.yml | 2 +- .github/workflows/test.yml | 6 +- CMakeLists.txt | 2 + .../IcebergThirdpartyToolchain.cmake | 41 ++++++++ src/iceberg/CMakeLists.txt | 10 ++ src/iceberg/iceberg-config.cmake.in | 5 + src/iceberg/test/CMakeLists.txt | 3 + src/iceberg/test/parquet_encryption_test.cc | 97 +++++++++++++++++++ 12 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 src/iceberg/test/parquet_encryption_test.cc diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 9214027aeb..8f2a1963f9 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -87,7 +87,7 @@ jobs: - name: Install dependencies on Ubuntu if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} shell: bash - run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libjitterentropy3-dev + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev libjitterentropy3-dev - name: Cache vcpkg packages if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }} uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 198af3d678..8e7bc726b6 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -60,7 +60,7 @@ jobs: shell: bash run: | sudo apt-get update - sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev + sudo apt-get install -y libcurl4-openssl-dev libssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index ba494db466..6cbcbeacbb 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -104,7 +104,7 @@ jobs: - name: Install dependencies on Ubuntu if: ${{ startsWith(matrix.os, 'ubuntu') }} shell: bash - run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev - name: Verify run: | diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index 343e661efe..98612c6c38 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -51,7 +51,7 @@ jobs: persist-credentials: false - name: Install dependencies shell: bash - run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 91b5d60e9d..28ddaae32d 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -96,7 +96,7 @@ jobs: if: ${{ startsWith(matrix.runs-on, 'windows') && steps.vcpkg-cache.outputs.cache-hit != 'true' }} shell: pwsh run: | - vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows + vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows openssl:x64-windows - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 352da0d7a4..0a0efdba72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,7 @@ jobs: persist-credentials: false - name: Install dependencies shell: bash - run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev - name: Set up sccache uses: ./.github/actions/setup-sccache with: @@ -102,7 +102,7 @@ jobs: persist-credentials: false - name: Install dependencies shell: bash - run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev - name: Restore sccache cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -182,7 +182,7 @@ jobs: if: ${{ steps.vcpkg-cache.outputs.cache-hit != 'true' }} shell: pwsh run: | - vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows + vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows openssl:x64-windows - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/CMakeLists.txt b/CMakeLists.txt index 90af528721..d4339a8ba5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,8 @@ option(ICEBERG_BUILD_SHARED "Build shared library" OFF) option(ICEBERG_BUILD_TESTS "Build tests" ON) option(ICEBERG_BUILD_BENCHMARKS "Build benchmarks" OFF) option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON) +option(ICEBERG_ENCRYPTION + "Build table encryption support (Parquet modular encryption, AES GCM streams)" ON) option(ICEBERG_BUILD_REST "Build rest catalog client" ON) option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration tests" OFF) option(ICEBERG_BUILD_HIVE "Build hive (HMS) catalog client" OFF) diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index c38d960286..956813f8d2 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -163,7 +163,30 @@ endmacro() # ---------------------------------------------------------------------- # Apache Arrow +# Find OpenSSL for table encryption. Called before Arrow is configured, so that the +# imported OpenSSL targets are visible to the vendored Arrow subdirectories. +macro(resolve_openssl_for_encryption) + if(APPLE + AND NOT DEFINED OPENSSL_ROOT_DIR + AND NOT DEFINED ENV{OPENSSL_ROOT_DIR}) + # Homebrew's OpenSSL is keg-only and not on the default search path. + execute_process(COMMAND brew --prefix openssl@3 + OUTPUT_VARIABLE ICEBERG_BREW_OPENSSL_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(ICEBERG_BREW_OPENSSL_PREFIX AND EXISTS "${ICEBERG_BREW_OPENSSL_PREFIX}") + set(OPENSSL_ROOT_DIR "${ICEBERG_BREW_OPENSSL_PREFIX}") + endif() + endif() + find_package(OpenSSL REQUIRED) + # Remembered for iceberg-config.cmake, so consumers find the same OpenSSL. + get_filename_component(ICEBERG_OPENSSL_ROOT_DIR "${OPENSSL_INCLUDE_DIR}" DIRECTORY) +endmacro() + function(resolve_arrow_dependency) + if(ICEBERG_ENCRYPTION) + resolve_openssl_for_encryption() + endif() + prepare_fetchcontent() # Prevent Arrow from injecting -Werror into CMAKE_CXX_FLAGS_DEBUG via @@ -178,12 +201,18 @@ function(resolve_arrow_dependency) set(ARROW_S3 ${ICEBERG_S3}) set(ARROW_JSON ON) set(ARROW_PARQUET ON) + if(ICEBERG_ENCRYPTION) + # Parquet modular encryption, used for encrypted Iceberg data and delete files. + set(PARQUET_REQUIRE_ENCRYPTION ON) + endif() set(ARROW_ENABLE_THREADING ON) set(ARROW_SIMD_LEVEL "NONE") set(ARROW_RUNTIME_SIMD_LEVEL "NONE") set(ARROW_POSITION_INDEPENDENT_CODE ON) set(ARROW_DEPENDENCY_SOURCE "BUNDLED") set(ARROW_WITH_ZLIB ON) + # zstd is the default Parquet codec of Iceberg (Java and C++). + set(ARROW_WITH_ZSTD ON) if(ICEBERG_S3 AND NOT ICEBERG_AWSSDK_BUNDLED) set(AWSSDK_SOURCE "SYSTEM") endif() @@ -216,6 +245,12 @@ function(resolve_arrow_dependency) fetchcontent_makeavailable(VendoredArrow) + # Arrow's Parquet object library does not inherit the OpenSSL include directory + # when Arrow is vendored; add it for the Parquet encryption sources. + if(ICEBERG_ENCRYPTION AND TARGET parquet_objlib) + target_link_libraries(parquet_objlib PRIVATE OpenSSL::Crypto) + endif() + if(vendoredarrow_SOURCE_DIR) if(NOT TARGET Arrow::arrow_static) add_library(Arrow::arrow_static INTERFACE IMPORTED) @@ -258,6 +293,9 @@ function(resolve_arrow_dependency) # Arrow's exported static target interface may reference system libraries # (e.g. Threads, OpenSSL, CURL, ZLIB) that consumers need to find. list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Threads ZLIB) + if(ICEBERG_ENCRYPTION) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL) + endif() if(ARROW_S3) list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL CURL) endif() @@ -274,6 +312,9 @@ function(resolve_arrow_dependency) set(ARROW_VENDORED ${ARROW_VENDORED} PARENT_SCOPE) + set(ICEBERG_OPENSSL_ROOT_DIR + "${ICEBERG_OPENSSL_ROOT_DIR}" + PARENT_SCOPE) endfunction() # ---------------------------------------------------------------------- diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 8a98274fff..774a5333ce 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -355,6 +355,16 @@ if(ICEBERG_BUILD_BUNDLE) "$,iceberg::avrocpp_s,$,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>>" ) + if(ICEBERG_ENCRYPTION) + foreach(libs + ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS + ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS + ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS + ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS) + list(APPEND ${libs} OpenSSL::Crypto) + endforeach() + endif() + add_iceberg_lib(iceberg_bundle SOURCES ${ICEBERG_BUNDLE_SOURCES} diff --git a/src/iceberg/iceberg-config.cmake.in b/src/iceberg/iceberg-config.cmake.in index 9a7bfbea41..cb9dd727d4 100644 --- a/src/iceberg/iceberg-config.cmake.in +++ b/src/iceberg/iceberg-config.cmake.in @@ -78,6 +78,11 @@ macro(iceberg_find_components components) endforeach() endmacro() +# OpenSSL used at build time (table encryption), e.g. Homebrew's keg-only OpenSSL. +if(NOT DEFINED OPENSSL_ROOT_DIR AND NOT "@ICEBERG_OPENSSL_ROOT_DIR@" STREQUAL "") + set(OPENSSL_ROOT_DIR "@ICEBERG_OPENSSL_ROOT_DIR@") +endif() + # Find system dependencies iceberg_find_dependencies("${ICEBERG_SYSTEM_DEPENDENCIES}") diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 6f6ff76031..bf99773168 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -220,6 +220,9 @@ if(ICEBERG_BUILD_BUNDLE) parquet_metrics_test.cc parquet_schema_test.cc parquet_test.cc) + if(ICEBERG_ENCRYPTION) + target_sources(parquet_test PRIVATE parquet_encryption_test.cc) + endif() add_iceberg_test(scan_test USE_BUNDLE diff --git a/src/iceberg/test/parquet_encryption_test.cc b/src/iceberg/test/parquet_encryption_test.cc new file mode 100644 index 0000000000..10ec64f849 --- /dev/null +++ b/src/iceberg/test/parquet_encryption_test.cc @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace iceberg::parquet { + +namespace { + +const std::string kKey = "0123456789012345"; +const std::string kAadPrefix = "file-aad-prefix-"; + +std::shared_ptr<::arrow::Table> MakeTable() { + auto schema = ::arrow::schema({::arrow::field("id", ::arrow::int64())}); + auto ids = + ::arrow::json::ArrayFromJSONString(::arrow::int64(), "[1, 2, 3]").ValueOrDie(); + return ::arrow::Table::Make(schema, {ids}); +} + +std::shared_ptr<::arrow::Buffer> WriteEncrypted(::parquet::Compression::type codec) { + // Same settings as Iceberg: footer key only (encrypted footer), AAD prefix supplied + // by the reader from the manifest instead of stored in the file. + ::parquet::FileEncryptionProperties::Builder encryption{ + ::arrow::util::SecureString(std::string(kKey))}; + encryption.aad_prefix(kAadPrefix); + encryption.disable_aad_prefix_storage(); + auto properties = ::parquet::WriterProperties::Builder() + .compression(codec) + ->encryption(encryption.build()) + ->build(); + + auto sink = ::arrow::io::BufferOutputStream::Create().ValueOrDie(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( + *MakeTable(), ::arrow::default_memory_pool(), sink, 1024, properties)); + return sink->Finish().ValueOrDie(); +} + +std::shared_ptr<::arrow::Table> Read(const std::shared_ptr<::arrow::Buffer>& buffer, + const std::string& aad_prefix) { + ::parquet::FileDecryptionProperties::Builder decryption; + decryption.footer_key(::arrow::util::SecureString(std::string(kKey))); + decryption.aad_prefix(aad_prefix); + ::parquet::ReaderProperties properties; + properties.file_decryption_properties(decryption.build()); + + auto reader = ::parquet::ParquetFileReader::Open( + std::make_shared<::arrow::io::BufferReader>(buffer), properties); + auto arrow_reader = ::parquet::arrow::FileReader::Make(::arrow::default_memory_pool(), + std::move(reader)) + .ValueOrDie(); + return arrow_reader->ReadTable().ValueOrDie(); +} + +} // namespace + +TEST(ParquetEncryptionTest, ModularEncryptionIsAvailable) { + for (auto codec : + {::parquet::Compression::UNCOMPRESSED, ::parquet::Compression::ZSTD}) { + auto buffer = WriteEncrypted(codec); + // Encrypted footer mode + ASSERT_EQ(std::string(reinterpret_cast(buffer->data()), 4), "PARE"); + EXPECT_TRUE(Read(buffer, kAadPrefix)->Equals(*MakeTable())); + // The AAD prefix binds the file to its manifest entry + EXPECT_THROW(Read(buffer, "another-prefix--"), ::parquet::ParquetException); + } +} + +} // namespace iceberg::parquet From db6f9d304c15d31bb80f6b2503c6d6a848c88b4f Mon Sep 17 00:00:00 2001 From: alice-shang Date: Fri, 9 Oct 2026 10:21:05 +0900 Subject: [PATCH 2/4] build: add the OpenSSL include directory to all vendored Parquet targets On Windows the Parquet encryption sources are compiled in parquet_static rather than parquet_objlib, so they did not find the OpenSSL headers. Co-Authored-By: Claude Opus 5.5 --- cmake_modules/IcebergThirdpartyToolchain.cmake | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 956813f8d2..0692a2d6aa 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -245,10 +245,16 @@ function(resolve_arrow_dependency) fetchcontent_makeavailable(VendoredArrow) - # Arrow's Parquet object library does not inherit the OpenSSL include directory - # when Arrow is vendored; add it for the Parquet encryption sources. - if(ICEBERG_ENCRYPTION AND TARGET parquet_objlib) - target_link_libraries(parquet_objlib PRIVATE OpenSSL::Crypto) + # Arrow's Parquet targets do not inherit the OpenSSL include directory when + # Arrow is vendored; add it for the Parquet encryption sources. Depending on the + # platform and build type, they are compiled in the object library or directly in + # the static/shared library. The bundle library links OpenSSL::Crypto. + if(ICEBERG_ENCRYPTION) + foreach(parquet_target parquet_objlib parquet_static parquet_shared) + if(TARGET ${parquet_target}) + target_include_directories(${parquet_target} PRIVATE ${OPENSSL_INCLUDE_DIR}) + endif() + endforeach() endif() if(vendoredarrow_SOURCE_DIR) From 59b678ae0f466dec4f7b09e1273815bfdc11fc80 Mon Sep 17 00:00:00 2001 From: shangxinli Date: Wed, 14 Oct 2026 10:22:39 -0700 Subject: [PATCH 3/4] build: address review feedback on OpenSSL wiring and encryption default - Register OpenSSL in ICEBERG_SYSTEM_DEPENDENCIES for both vendored and system Arrow, since the bundle exports OpenSSL::Crypto in both modes. - Stop exporting the build machine's OpenSSL root from the installed package so consumers resolve OpenSSL from their own toolchain. - Restrict the Parquet include injection to vendored Arrow targets and skip ALIAS targets, which target_include_directories cannot modify. - Default ICEBERG_ENCRYPTION to OFF and enable it on the Ubuntu and macOS test jobs, so the encryption build, parquet_encryption_test, and the Homebrew keg-only OpenSSL lookup all stay covered. - Document why zstd is enabled in the vendored Arrow build. --- .github/workflows/test.yml | 6 +++ CMakeLists.txt | 2 +- .../IcebergThirdpartyToolchain.cmake | 50 +++++++++++-------- src/iceberg/iceberg-config.cmake.in | 5 -- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a0efdba72..2d233c3081 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,6 +54,9 @@ jobs: cmake_build_type: - Debug - RelWithDebInfo + env: + # ICEBERG_ENCRYPTION defaults to OFF; keep one job building and testing it. + ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_ENCRYPTION=ON" steps: - name: Checkout iceberg-cpp uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -135,6 +138,9 @@ jobs: timeout-minutes: 30 strategy: fail-fast: false + env: + # Covers the Homebrew keg-only OpenSSL lookup, which only runs on macOS. + ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_ENCRYPTION=ON" steps: - name: Checkout iceberg-cpp uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/CMakeLists.txt b/CMakeLists.txt index d4339a8ba5..f7987fa49e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ option(ICEBERG_BUILD_TESTS "Build tests" ON) option(ICEBERG_BUILD_BENCHMARKS "Build benchmarks" OFF) option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON) option(ICEBERG_ENCRYPTION - "Build table encryption support (Parquet modular encryption, AES GCM streams)" ON) + "Build table encryption support (Parquet modular encryption, AES GCM streams)" OFF) option(ICEBERG_BUILD_REST "Build rest catalog client" ON) option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration tests" OFF) option(ICEBERG_BUILD_HIVE "Build hive (HMS) catalog client" OFF) diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 0692a2d6aa..4c545e1b44 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -178,8 +178,6 @@ macro(resolve_openssl_for_encryption) endif() endif() find_package(OpenSSL REQUIRED) - # Remembered for iceberg-config.cmake, so consumers find the same OpenSSL. - get_filename_component(ICEBERG_OPENSSL_ROOT_DIR "${OPENSSL_INCLUDE_DIR}" DIRECTORY) endmacro() function(resolve_arrow_dependency) @@ -211,7 +209,10 @@ function(resolve_arrow_dependency) set(ARROW_POSITION_INDEPENDENT_CODE ON) set(ARROW_DEPENDENCY_SOURCE "BUNDLED") set(ARROW_WITH_ZLIB ON) - # zstd is the default Parquet codec of Iceberg (Java and C++). + # zstd is the default value of the write.parquet.compression-codec table property, + # so a vendored Arrow without it cannot write Iceberg's default Parquet files. + # Not required by modular encryption itself; parquet_encryption_test covers the + # encrypted-plus-zstd combination to keep the two features verified together. set(ARROW_WITH_ZSTD ON) if(ICEBERG_S3 AND NOT ICEBERG_AWSSDK_BUNDLED) set(AWSSDK_SOURCE "SYSTEM") @@ -245,19 +246,26 @@ function(resolve_arrow_dependency) fetchcontent_makeavailable(VendoredArrow) - # Arrow's Parquet targets do not inherit the OpenSSL include directory when - # Arrow is vendored; add it for the Parquet encryption sources. Depending on the - # platform and build type, they are compiled in the object library or directly in - # the static/shared library. The bundle library links OpenSSL::Crypto. - if(ICEBERG_ENCRYPTION) - foreach(parquet_target parquet_objlib parquet_static parquet_shared) - if(TARGET ${parquet_target}) - target_include_directories(${parquet_target} PRIVATE ${OPENSSL_INCLUDE_DIR}) - endif() - endforeach() - endif() - if(vendoredarrow_SOURCE_DIR) + # Arrow's Parquet targets do not inherit the OpenSSL include directory when + # Arrow is vendored; add it for the Parquet encryption sources. Depending on the + # platform and build type, they are compiled in the object library or directly in + # the static/shared library. The bundle library links OpenSSL::Crypto. + # + # Only vendored targets are touched here. With system Arrow the same names can + # be ALIAS targets created by a parent project, and target_include_directories + # cannot modify an ALIAS. + if(ICEBERG_ENCRYPTION) + foreach(parquet_target parquet_objlib parquet_static parquet_shared) + if(TARGET ${parquet_target}) + get_target_property(parquet_target_alias ${parquet_target} ALIASED_TARGET) + if(NOT parquet_target_alias) + target_include_directories(${parquet_target} PRIVATE ${OPENSSL_INCLUDE_DIR}) + endif() + endif() + endforeach() + endif() + if(NOT TARGET Arrow::arrow_static) add_library(Arrow::arrow_static INTERFACE IMPORTED) target_link_libraries(Arrow::arrow_static INTERFACE arrow_static) @@ -299,9 +307,6 @@ function(resolve_arrow_dependency) # Arrow's exported static target interface may reference system libraries # (e.g. Threads, OpenSSL, CURL, ZLIB) that consumers need to find. list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Threads ZLIB) - if(ICEBERG_ENCRYPTION) - list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL) - endif() if(ARROW_S3) list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL CURL) endif() @@ -312,15 +317,18 @@ function(resolve_arrow_dependency) list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Arrow Parquet) endif() + # The bundle exports OpenSSL::Crypto in its link interface for both vendored and + # system Arrow, so consumers must find OpenSSL in both modes. + if(ICEBERG_ENCRYPTION) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL) + endif() + set(ICEBERG_SYSTEM_DEPENDENCIES ${ICEBERG_SYSTEM_DEPENDENCIES} PARENT_SCOPE) set(ARROW_VENDORED ${ARROW_VENDORED} PARENT_SCOPE) - set(ICEBERG_OPENSSL_ROOT_DIR - "${ICEBERG_OPENSSL_ROOT_DIR}" - PARENT_SCOPE) endfunction() # ---------------------------------------------------------------------- diff --git a/src/iceberg/iceberg-config.cmake.in b/src/iceberg/iceberg-config.cmake.in index cb9dd727d4..9a7bfbea41 100644 --- a/src/iceberg/iceberg-config.cmake.in +++ b/src/iceberg/iceberg-config.cmake.in @@ -78,11 +78,6 @@ macro(iceberg_find_components components) endforeach() endmacro() -# OpenSSL used at build time (table encryption), e.g. Homebrew's keg-only OpenSSL. -if(NOT DEFINED OPENSSL_ROOT_DIR AND NOT "@ICEBERG_OPENSSL_ROOT_DIR@" STREQUAL "") - set(OPENSSL_ROOT_DIR "@ICEBERG_OPENSSL_ROOT_DIR@") -endif() - # Find system dependencies iceberg_find_dependencies("${ICEBERG_SYSTEM_DEPENDENCIES}") From 6a437976793b2e17490a648749e80e830e2b27c0 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Thu, 15 Oct 2026 11:38:23 +0800 Subject: [PATCH 4/4] ci: build and test Parquet encryption in the Windows job --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 532a96561a..a85e6f24a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -240,7 +240,7 @@ jobs: - name: Build Iceberg shell: pwsh env: - ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON" + ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON -DICEBERG_ENCRYPTION=ON" run: | $ErrorActionPreference = "Stop" bash -lc '