From dca05eb4183ea8f08bac968cf5001d0e9417da22 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Tue, 13 Oct 2026 21:00:25 +0800 Subject: [PATCH 1/3] feat(global-index): align row filtering and BTree Bloom support --- docs/source/user_guide/global_index.rst | 6 + .../global_index/global_index_io_meta.h | 10 +- .../paimon/global_index/global_index_scan.h | 15 +- include/paimon/global_index/global_indexer.h | 20 +- .../bitmap/bitmap_global_index_writer.cpp | 4 +- .../bitmap/bitmap_global_indexer.cpp | 6 +- .../bitmap/bitmap_global_indexer.h | 3 +- .../bitmap/lazy_filtered_bitmap_reader.cpp | 15 +- .../bitmap/lazy_filtered_bitmap_reader.h | 5 +- .../lazy_filtered_bitmap_reader_test.cpp | 7 +- .../btree/btree_compatibility_test.cpp | 30 ++- .../common/global_index/btree/btree_defs.h | 4 + .../btree_global_index_integration_test.cpp | 192 ++++++++++++++---- .../btree/btree_global_index_reader.cpp | 45 +++- .../btree/btree_global_index_reader.h | 8 +- .../btree/btree_global_index_writer.cpp | 27 ++- .../btree/btree_global_index_writer.h | 4 +- .../btree/btree_global_indexer.cpp | 14 +- .../global_index/btree/btree_global_indexer.h | 3 +- .../btree/lazy_filtered_btree_reader.cpp | 182 ++++++++++++++++- .../btree/lazy_filtered_btree_reader.h | 21 +- .../btree/lazy_filtered_btree_reader_test.cpp | 78 +++++-- .../rangebitmap/range_bitmap_global_index.cpp | 4 +- .../rangebitmap/range_bitmap_global_index.h | 3 +- .../range_bitmap_global_index_test.cpp | 4 +- .../sorted_file_global_index_reader.cpp | 65 ++++-- .../sorted_file_global_index_reader.h | 7 +- .../sorted_file_meta_selector_test.cpp | 47 +++-- .../union_global_index_reader_test.cpp | 9 +- .../wrap/file_index_writer_wrapper.h | 2 +- .../common/lookup/lookup_store_factory.cpp | 14 +- .../common/lookup/lookup_store_factory.h | 8 +- .../lookup/sort/sort_lookup_store_factory.cpp | 5 +- .../lookup/sort/sort_lookup_store_factory.h | 2 +- src/paimon/common/sst/sst_file_io_test.cpp | 16 +- src/paimon/common/sst/sst_file_reader.cpp | 3 +- src/paimon/common/sst/sst_file_writer.cpp | 28 ++- src/paimon/common/sst/sst_file_writer.h | 5 +- src/paimon/common/utils/bloom_filter.cpp | 103 +++++++++- src/paimon/common/utils/bloom_filter.h | 20 ++ src/paimon/common/utils/bloom_filter_test.cpp | 50 ++++- .../global_index/global_index_scan_impl.cpp | 16 +- .../global_index/global_index_write_task.cpp | 2 +- src/paimon/core/mergetree/lookup_levels.cpp | 7 +- .../source/data_evolution_batch_scan_test.cpp | 19 ++ .../source/primary_key_sorted_index_scan.cpp | 5 +- .../primary_key_sorted_index_scan_test.cpp | 4 +- .../full_text/full_text_global_index.cpp | 4 +- .../full_text/full_text_global_index.h | 3 +- .../full_text/full_text_global_index_test.cpp | 24 ++- .../full_text_global_index_writer.cpp | 2 +- .../lucene/lucene_global_index.cpp | 4 +- .../global_index/lucene/lucene_global_index.h | 3 +- .../lucene/lucene_global_index_test.cpp | 3 +- .../lucene/lucene_global_index_writer.cpp | 1 + .../lucene/lucene_global_index_writer.h | 2 +- .../indexer/lumina/lumina_global_index.cpp | 5 +- .../indexer/lumina/lumina_global_index.h | 3 +- .../lumina/lumina_global_index_test.cpp | 7 +- test/inte/global_index_test.cpp | 18 +- .../primary_key_sorted_index_inte_test.cpp | 50 ++++- 61 files changed, 1030 insertions(+), 246 deletions(-) diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 956d28dba..0655f9dcc 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -60,6 +60,12 @@ BTree is an efficient index based on multi-level SST files, supporting rich pred **Special Configuration:** +- **Option**: ``btree-index.bloom-filter.enabled`` + + - **Default**: ``false`` + - **Description**: Enables a Bloom filter for BTree point lookups such as equality and ``IN`` + predicates. This can avoid reading data blocks for missing keys. + - **Option**: ``btree-index.read-buffer-size`` - **Description**: Optional. Specifies the read buffer size for the B-tree index. This setting can be tuned based on query patterns: diff --git a/include/paimon/global_index/global_index_io_meta.h b/include/paimon/global_index/global_index_io_meta.h index 10ace821e..44f7b7de7 100644 --- a/include/paimon/global_index/global_index_io_meta.h +++ b/include/paimon/global_index/global_index_io_meta.h @@ -24,16 +24,22 @@ #include #include "paimon/memory/bytes.h" +#include "paimon/visibility.h" namespace paimon { /// Metadata describing a single file entry in a global index. struct PAIMON_EXPORT GlobalIndexIOMeta { - GlobalIndexIOMeta(const std::string& _file_path, int64_t _file_size, + GlobalIndexIOMeta(const std::string& _file_path, int64_t _file_size, int64_t _row_count, const std::shared_ptr& _metadata) - : file_path(_file_path), file_size(_file_size), metadata(_metadata) {} + : file_path(_file_path), + file_size(_file_size), + row_count(_row_count), + metadata(_metadata) {} std::string file_path; int64_t file_size; + /// Total logical rows processed by the writer, including null or skipped rows. + int64_t row_count; /// Optional binary metadata associated with the file, such as serialized /// secondary index structures or inline index bytes. /// May be null if no additional metadata is available. diff --git a/include/paimon/global_index/global_index_scan.h b/include/paimon/global_index/global_index_scan.h index d8ba799de..288bfbb14 100644 --- a/include/paimon/global_index/global_index_scan.h +++ b/include/paimon/global_index/global_index_scan.h @@ -88,8 +88,9 @@ class PAIMON_EXPORT GlobalIndexScan { /// Creates several `GlobalIndexReader`s for a specific field. /// @param field_name Name of the indexed column. - /// @param row_range_index Optional row range that limits the scan to a sub-range of row ids. - /// If not provided, the entire row range is considered. + /// @param row_range_index Optional row ranges used to skip non-intersecting index shards and + /// provide pruning hints. Returned candidates are not guaranteed to be + /// clipped; callers must intersect them with the requested ranges. /// @return A `Result` that is: /// - Successful with several readers(with global row id) if the indexes exist and load /// correctly; @@ -103,8 +104,9 @@ class PAIMON_EXPORT GlobalIndexScan { /// Creates a `GlobalIndexReader` for a specific field with a specific index type. /// @param field_name Name of the indexed column. /// @param index_type Name of index type. - /// @param row_range_index Optional row range that limits the scan to a sub-range of row ids. - /// If not provided, the entire row range is considered. + /// @param row_range_index Optional row ranges used to skip non-intersecting index shards and + /// provide pruning hints. Returned candidates are not guaranteed to be + /// clipped; callers must intersect them with the requested ranges. /// @return A `Result` that is: /// - Successful with a reader(with global row id) if the index exists and loads /// correctly; @@ -118,8 +120,9 @@ class PAIMON_EXPORT GlobalIndexScan { /// Creates several `GlobalIndexReader`s for a specific field (looked up by id), /// @param field_id Field id of the indexed column. - /// @param row_range_index Optional row range that limits the scan to a sub-range of row ids. - /// If not provided, the entire row range is considered. + /// @param row_range_index Optional row ranges used to skip non-intersecting index shards and + /// provide pruning hints. Returned candidates are not guaranteed to be + /// clipped; callers must intersect them with the requested ranges. /// @return A `Result` that is: /// - Successful with several readers(with global row id) if the indexes exist and load /// correctly; diff --git a/include/paimon/global_index/global_indexer.h b/include/paimon/global_index/global_indexer.h index 5be789aca..d7fddfa49 100644 --- a/include/paimon/global_index/global_indexer.h +++ b/include/paimon/global_index/global_indexer.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include #include @@ -30,6 +31,7 @@ #include "paimon/global_index/io/global_index_file_reader.h" #include "paimon/global_index/io/global_index_file_writer.h" #include "paimon/memory/memory_pool.h" +#include "paimon/utils/range.h" #include "paimon/visibility.h" struct ArrowSchema; @@ -58,17 +60,23 @@ class PAIMON_EXPORT GlobalIndexer { const std::shared_ptr& file_writer, const std::shared_ptr& pool) const = 0; - /// Creates a reader for querying a pre-built global index. + /// Creates a reader for querying a pre-built global index whose relative row IDs cover + /// [0, total_row_count). /// - /// @param arrow_schema Schema of the indexed data; used to interpret predicate literals. - /// @param file_reader I/O handler for reading index artifacts from storage. - /// @param files List of index file metadata entries produced during writing. - /// @param pool Memory pool for temporary allocations; if nullptr, uses default. + /// @param arrow_schema Schema of the indexed data; used to interpret predicate literals. + /// @param file_reader I/O handler for reading index artifacts from storage. + /// @param files List of index file metadata entries produced during writing. + /// @param total_row_count Complete row count for evaluating negative predicates by complement. + /// @param row_ranges Optional local row ranges used only as a pruning hint. + /// Implementations may ignore them; callers must still clip returned + /// candidates to the requested ranges. + /// @param pool Memory pool for temporary allocations; if nullptr, uses default. /// @return A `Result` containing a shared pointer to the created `GlobalIndexReader`, /// or an error if the index cannot be loaded or is incompatible, etc. virtual Result> CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, const std::shared_ptr& pool) const = 0; /// Whether this indexer supports checkpointing an index build. diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.cpp b/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.cpp index f7f5865de..a9c3c2a17 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.cpp +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index_writer.cpp @@ -143,8 +143,8 @@ Result> BitmapGlobalIndexWriter::Finish() { SortedIndexFileMeta index_meta(first_key_bytes, last_key_bytes, !null_rows_.IsEmpty()); std::shared_ptr metadata = index_meta.Serialize(pool_.get()); PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(file_name_)); - return std::vector{ - GlobalIndexIOMeta(file_writer_->ToPath(file_name_), file_size, std::move(metadata))}; + return std::vector{GlobalIndexIOMeta( + file_writer_->ToPath(file_name_), file_size, row_count_, std::move(metadata))}; } Status BitmapGlobalIndexWriter::FlushCurrentBitmap() { diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_indexer.cpp b/src/paimon/common/global_index/bitmap/bitmap_global_indexer.cpp index 06bfdb67f..bf9a2bf97 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_indexer.cpp +++ b/src/paimon/common/global_index/bitmap/bitmap_global_indexer.cpp @@ -98,7 +98,9 @@ Result> BitmapGlobalIndexer::CreateWriter( Result> BitmapGlobalIndexer::CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, const std::shared_ptr& pool) const { + const std::vector& files, int64_t total_row_count, + const std::optional>& /*row_ranges*/, + const std::shared_ptr& pool) const { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, arrow::ImportSchema(arrow_schema)); if (schema->num_fields() != 1) { @@ -113,7 +115,7 @@ Result> BitmapGlobalIndexer::CreateReader( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr reader, LazyFilteredBitmapReader::Create(file_reader, files, schema->field(0)->type(), - fallback_scan_max_size_, pool, executor)); + fallback_scan_max_size_, total_row_count, pool, executor)); return std::shared_ptr(std::move(reader)); } diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_indexer.h b/src/paimon/common/global_index/bitmap/bitmap_global_indexer.h index 4a79e40ec..4bcbe4b7f 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_indexer.h +++ b/src/paimon/common/global_index/bitmap/bitmap_global_indexer.h @@ -45,7 +45,8 @@ class BitmapGlobalIndexer : public GlobalIndexer { Result> CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, const std::shared_ptr& pool) const override; private: diff --git a/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader.cpp b/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader.cpp index 60a81dd3a..505b4eb3d 100644 --- a/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader.cpp +++ b/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader.cpp @@ -29,28 +29,33 @@ namespace paimon { Result> LazyFilteredBitmapReader::Create( const std::shared_ptr& file_reader, const std::vector& files, const std::shared_ptr& key_type, - int64_t fallback_scan_max_size, const std::shared_ptr& pool, - const std::shared_ptr& executor) { + int64_t fallback_scan_max_size, int64_t total_row_count, + const std::shared_ptr& pool, const std::shared_ptr& executor) { if (file_reader == nullptr || key_type == nullptr || pool == nullptr) { return Status::Invalid( "Cannot create LazyFilteredBitmapReader without file reader, key type, and memory " "pool."); } + if (total_row_count < 0) { + return Status::Invalid("Total row count must be non-negative."); + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_serializer, KeySerializer::Create(key_type, pool)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_selector, SortedFileMetaSelector::Create(files, key_serializer)); return std::shared_ptr( new LazyFilteredBitmapReader(file_reader, std::move(file_selector), key_serializer, - fallback_scan_max_size, pool, executor)); + fallback_scan_max_size, total_row_count, pool, executor)); } LazyFilteredBitmapReader::LazyFilteredBitmapReader( const std::shared_ptr& file_reader, std::unique_ptr file_selector, const std::shared_ptr& key_serializer, int64_t fallback_scan_max_size, - const std::shared_ptr& pool, const std::shared_ptr& executor) - : SortedFileGlobalIndexReader(std::move(file_selector), fallback_scan_max_size, executor), + int64_t total_row_count, const std::shared_ptr& pool, + const std::shared_ptr& executor) + : SortedFileGlobalIndexReader(std::move(file_selector), fallback_scan_max_size, total_row_count, + executor), file_reader_(file_reader), key_serializer_(key_serializer), pool_(pool) {} diff --git a/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader.h b/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader.h index b6291003a..811aaaf9b 100644 --- a/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader.h +++ b/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader.h @@ -39,7 +39,8 @@ class LazyFilteredBitmapReader : public SortedFileGlobalIndexReader { const std::shared_ptr& file_reader, const std::vector& files, const std::shared_ptr& key_type, int64_t fallback_scan_max_size, - const std::shared_ptr& pool, const std::shared_ptr& executor); + int64_t total_row_count, const std::shared_ptr& pool, + const std::shared_ptr& executor); Result> VisitVectorSearch( const std::shared_ptr& vector_search) override; @@ -54,7 +55,7 @@ class LazyFilteredBitmapReader : public SortedFileGlobalIndexReader { LazyFilteredBitmapReader(const std::shared_ptr& file_reader, std::unique_ptr file_selector, const std::shared_ptr& key_serializer, - int64_t fallback_scan_max_size, + int64_t fallback_scan_max_size, int64_t total_row_count, const std::shared_ptr& pool, const std::shared_ptr& executor); diff --git a/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader_test.cpp b/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader_test.cpp index 0f444cca3..eff9366f5 100644 --- a/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader_test.cpp +++ b/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader_test.cpp @@ -100,10 +100,15 @@ class LazyFilteredBitmapReaderTest : public ::testing::Test { std::shared_ptr CreateReader( const std::vector& metas, int64_t fallback_scan_max_size, const std::shared_ptr& file_reader = nullptr) { + int64_t total_row_count = 0; + for (const GlobalIndexIOMeta& meta : metas) { + total_row_count += meta.row_count; + } EXPECT_OK_AND_ASSIGN( std::shared_ptr reader, LazyFilteredBitmapReader::Create(file_reader == nullptr ? file_manager_ : file_reader, - metas, arrow::utf8(), fallback_scan_max_size, pool_, + metas, arrow::utf8(), fallback_scan_max_size, + total_row_count, pool_, /*executor=*/nullptr)); return reader; } diff --git a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp index 80c362058..fde4292ed 100644 --- a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp +++ b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp @@ -112,13 +112,13 @@ class BTreeCompatibilityTest : public ::testing::Test { Result> CreateReaderFromFiles( const std::string& bin_path, const std::string& meta_path, - const std::shared_ptr& arrow_type) const { + const std::shared_ptr& arrow_type, int64_t total_row_count) const { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); PAIMON_ASSIGN_OR_RAISE(FileStatus file_status, fs_->GetFileStatus(bin_path)); auto file_size = file_status.GetLen(); - GlobalIndexIOMeta io_meta(bin_path, file_size, meta_bytes); + GlobalIndexIOMeta io_meta(bin_path, file_size, total_row_count, meta_bytes); std::vector metas = {io_meta}; auto schema = arrow::schema({arrow::field("testField", arrow_type)}); @@ -128,7 +128,8 @@ class BTreeCompatibilityTest : public ::testing::Test { auto file_reader = std::make_shared(fs_); std::map options; PAIMON_ASSIGN_OR_RAISE(auto indexer, BTreeGlobalIndexer::Create(options)); - return indexer->CreateReader(c_schema.get(), file_reader, metas, pool_); + return indexer->CreateReader(c_schema.get(), file_reader, metas, total_row_count, + /*row_ranges=*/std::nullopt, pool_); } void RunIntQueries(const std::shared_ptr& reader, @@ -570,7 +571,8 @@ TEST_F(BTreeCompatibilityTest, ReadAndQueryIntData) { ASSERT_EQ(static_cast(records.size()), count); ASSERT_OK_AND_ASSIGN(auto reader, - CreateReaderFromFiles(bin_path, meta_path, arrow::int32())); + CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), + records.size())); RunIntQueries(reader, records); } } @@ -586,7 +588,8 @@ TEST_F(BTreeCompatibilityTest, ReadAndQueryStringData) { ASSERT_EQ(static_cast(records.size()), count); ASSERT_OK_AND_ASSIGN(auto reader, - CreateReaderFromFiles(bin_path, meta_path, arrow::utf8())); + CreateReaderFromFiles(bin_path, meta_path, arrow::utf8(), + records.size())); RunStringQueries(reader, records); } } @@ -601,7 +604,9 @@ TEST_F(BTreeCompatibilityTest, ReadAndQueryFloatData) { auto records = ParseCsvFile(csv_path); ASSERT_EQ(static_cast(records.size()), count); - ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::float32())); + ASSERT_OK_AND_ASSIGN( + auto reader, + CreateReaderFromFiles(bin_path, meta_path, arrow::float32(), records.size())); RunFloatQueries(reader, records); } @@ -615,7 +620,8 @@ TEST_F(BTreeCompatibilityTest, AllNulls) { ASSERT_FALSE(records.empty()); auto count = static_cast(records.size()); - ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32())); + ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), + records.size())); // All rows should be null { @@ -653,7 +659,8 @@ TEST_F(BTreeCompatibilityTest, NoNulls) { ASSERT_FALSE(records.empty()); auto count = static_cast(records.size()); - ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32())); + ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), + records.size())); // No rows should be null { @@ -720,7 +727,8 @@ TEST_F(BTreeCompatibilityTest, DuplicateKeys) { auto records = ParseCsvFile(csv_path); ASSERT_FALSE(records.empty()); - ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32())); + ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), + records.size())); // VisitIsNull { @@ -914,7 +922,9 @@ TEST_F(BTreeCompatibilityTest, RowCountConsistency) { ASSERT_FALSE(records.empty()); auto count = static_cast(records.size()); - ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow_type)); + ASSERT_OK_AND_ASSIGN( + auto reader, + CreateReaderFromFiles(bin_path, meta_path, arrow_type, records.size())); ASSERT_OK_AND_ASSIGN(auto null_result, reader->VisitIsNull()); auto null_ids = CollectRowIds(null_result); diff --git a/src/paimon/common/global_index/btree/btree_defs.h b/src/paimon/common/global_index/btree/btree_defs.h index b83433efe..f0287e966 100644 --- a/src/paimon/common/global_index/btree/btree_defs.h +++ b/src/paimon/common/global_index/btree/btree_defs.h @@ -36,6 +36,9 @@ struct BtreeDefs { /// "btree-index.block-size" - The block size to use for BTreeIndex. /// Default value is 64 KB. static inline const char kBtreeIndexBlockSize[] = "btree-index.block-size"; + /// "btree-index.bloom-filter.enabled" - Whether to enable the Bloom filter for BTree index + /// point lookups. Default value is false. + static inline const char kBtreeIndexBloomFilterEnabled[] = "btree-index.bloom-filter.enabled"; /// "btree-index.cache-size" - The cache size to use for BTreeIndex. /// Default value is 128 MB. static inline const char kBtreeIndexCacheSize[] = "btree-index.cache-size"; @@ -66,5 +69,6 @@ struct BtreeDefs { static inline const char kDefaultBtreeIndexFallbackScanMaxSize[] = "256MB"; static inline const int32_t kDefaultBtreeIndexCompressionLevel = 1; static inline const double kDefaultBtreeIndexHighPriorityPoolRatio = 0.1; + static inline const bool kDefaultBtreeIndexBloomFilterEnabled = false; }; } // namespace paimon diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index 89d98d43a..1332666c6 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -38,6 +38,7 @@ #include "paimon/global_index/io/global_index_file_writer.h" #include "paimon/io/buffered_input_stream.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/predicate/literal.h" #include "paimon/testing/utils/io_exception_helper.h" #include "paimon/testing/utils/testharness.h" @@ -121,6 +122,14 @@ class BTreeGlobalIndexIntegrationTest : public ::testing::Test, << ", expected=" << RoaringBitmap64::From(expected).ToString(); } + static int64_t TotalRowCount(const std::vector& metas) { + int64_t total_row_count = 0; + for (const GlobalIndexIOMeta& meta : metas) { + total_row_count += meta.row_count; + } + return total_row_count; + } + std::shared_ptr pool_; std::shared_ptr compression_factory_; std::unique_ptr test_dir_; @@ -133,8 +142,10 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadIntData) { auto field = arrow::field("int_field", arrow::int32()); auto c_schema = CreateArrowSchema(field); - std::map options = {{BtreeDefs::kBtreeIndexBlockSize, "4096"}, - {BtreeDefs::kBtreeIndexCompression, GetParam()}}; + std::map options = { + {BtreeDefs::kBtreeIndexBlockSize, "4096"}, + {BtreeDefs::kBtreeIndexCompression, GetParam()}, + {BtreeDefs::kBtreeIndexBloomFilterEnabled, "true"}}; ASSERT_OK_AND_ASSIGN(auto indexer, BTreeGlobalIndexer::Create(options)); ASSERT_OK_AND_ASSIGN(auto writer, indexer->CreateWriter("int_field", c_schema.get(), file_writer, pool_)); @@ -163,12 +174,27 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadIntData) { ASSERT_OK(writer->AddBatch(&c_array, std::move(row_ids))); ASSERT_OK_AND_ASSIGN(auto metas, writer->Finish()); ASSERT_EQ(metas.size(), 1); + ASSERT_EQ(metas[0].row_count, array->length()); + ASSERT_OK_AND_ASSIGN(auto index_input, fs_->Open(metas[0].file_path)); + ASSERT_OK(index_input->Seek(metas[0].file_size - BTreeFileFooter::kEncodingLength, + SeekOrigin::FS_SEEK_SET)); + MemorySegment footer_segment = + MemorySegment::AllocateHeapMemory(BTreeFileFooter::kEncodingLength, pool_.get()); + ASSERT_OK_AND_ASSIGN(int64_t footer_bytes_read, + index_input->Read(footer_segment.MutableData(), footer_segment.Size())); + ASSERT_EQ(footer_bytes_read, BTreeFileFooter::kEncodingLength); + MemorySliceInput footer_input = MemorySlice::Wrap(footer_segment).ToInput(); + ASSERT_OK_AND_ASSIGN(auto footer, BTreeFileFooter::Read(&footer_input)); + ASSERT_TRUE(footer->GetBloomFilterHandle().has_value()); + ASSERT_EQ(footer->GetBloomFilterHandle()->ExpectedEntries(), 5); + ASSERT_OK(index_input->Close()); // Now read back auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + indexer->CreateReader(c_schema.get(), file_reader, metas, array->length(), + /*row_ranges=*/std::nullopt, pool_)); // All non-null row ids: {0,1,3,4,6,7,8,9,10} // Null row ids: {2,5,11} @@ -214,6 +240,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadIntData) { Literal literal_3(3); ASSERT_OK_AND_ASSIGN(auto result, reader->VisitNotEqual(literal_3)); CheckResult(result, {0, 1, 3, 4, 7, 8, 9, 10}); + + ASSERT_OK_AND_ASSIGN(result, reader->VisitNotEqual(Literal(FieldType::INT))); + CheckResult(result, {}); } // --- VisitLessThan --- @@ -270,6 +299,11 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadIntData) { ASSERT_OK_AND_ASSIGN(auto result, reader->VisitIn(in_literals)); CheckResult(result, {0, 1, 7}); + // SQL IN ignores NULL literals. + std::vector in_with_null = {Literal(1), Literal(FieldType::INT), Literal(4)}; + ASSERT_OK_AND_ASSIGN(result, reader->VisitIn(in_with_null)); + CheckResult(result, {0, 1, 7}); + // In {99} (not present) -> empty std::vector in_missing = {Literal(99)}; ASSERT_OK_AND_ASSIGN(result, reader->VisitIn(in_missing)); @@ -282,6 +316,36 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadIntData) { std::vector not_in_literals = {Literal(1), Literal(5)}; ASSERT_OK_AND_ASSIGN(auto result, reader->VisitNotIn(not_in_literals)); CheckResult(result, {3, 4, 6, 7}); + + std::vector not_in_with_null = {Literal(1), Literal(FieldType::INT)}; + ASSERT_OK_AND_ASSIGN(result, reader->VisitNotIn(not_in_with_null)); + CheckResult(result, {}); + } + + // Verify that row ranges are converted to a row-id filter and applied to every query result. + std::vector row_ranges = {Range(1, 8), Range(10, 10)}; + c_schema = CreateArrowSchema(field); + ASSERT_OK_AND_ASSIGN( + auto filtered_reader, + indexer->CreateReader(c_schema.get(), file_reader, metas, array->length(), row_ranges, + pool_)); + + { + ASSERT_OK_AND_ASSIGN(auto result, filtered_reader->VisitEqual(Literal(5))); + CheckResult(result, {8, 10}); + } + { + std::vector literals = {Literal(1), Literal(4), Literal(5)}; + ASSERT_OK_AND_ASSIGN(auto result, filtered_reader->VisitIn(literals)); + CheckResult(result, {1, 7, 8, 10}); + } + { + ASSERT_OK_AND_ASSIGN(auto result, filtered_reader->VisitLessThan(Literal(4))); + CheckResult(result, {1, 3, 4, 6}); + } + { + ASSERT_OK_AND_ASSIGN(auto result, filtered_reader->VisitIsNull()); + CheckResult(result, {2, 5}); } } @@ -321,8 +385,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadStringData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,1,3,4,6,7,8}, Null rows: {2,5} @@ -501,8 +566,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteEmptyStringKeyMetadata) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); Literal empty(FieldType::STRING, "", 0); ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(empty)); @@ -542,8 +608,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadBigIntData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -631,8 +698,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadFloatData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -719,8 +787,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadDoubleData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -806,8 +875,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadAllNonNull) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // All rows: {0,1,2,3,4,5}, No null rows @@ -914,8 +984,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadBoolData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,1,3,4,6,7}, Null rows: {2,5} @@ -987,8 +1058,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadTinyIntData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -1066,8 +1138,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadSmallIntData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -1146,8 +1219,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadTimestampCompactData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -1227,8 +1301,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadTimestampNonCompactData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} // micros: 1000000 -> millis=1000, nanos_of_millis=0 @@ -1312,8 +1387,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadDecimalCompactData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -1402,8 +1478,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadDecimalNonCompactData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -1453,8 +1530,10 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadAllNull) { auto field = arrow::field("int_field", arrow::int32()); auto c_schema = CreateArrowSchema(field); - std::map options = {{BtreeDefs::kBtreeIndexBlockSize, "128"}, - {BtreeDefs::kBtreeIndexCompression, GetParam()}}; + std::map options = { + {BtreeDefs::kBtreeIndexBlockSize, "128"}, + {BtreeDefs::kBtreeIndexCompression, GetParam()}, + {BtreeDefs::kBtreeIndexBloomFilterEnabled, "true"}}; ASSERT_OK_AND_ASSIGN(auto indexer, BTreeGlobalIndexer::Create(options)); ASSERT_OK_AND_ASSIGN(auto writer, indexer->CreateWriter("int_field", c_schema.get(), file_writer, pool_)); @@ -1475,10 +1554,26 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadAllNull) { ASSERT_OK_AND_ASSIGN(auto metas, writer->Finish()); ASSERT_EQ(metas.size(), 1); + // Bloom is enabled, but nulls are not indexed as keys, so an all-null index must not write + // an empty Bloom block or advertise a Bloom handle in the footer. + ASSERT_OK_AND_ASSIGN(auto index_input, fs_->Open(metas[0].file_path)); + ASSERT_OK(index_input->Seek(metas[0].file_size - BTreeFileFooter::kEncodingLength, + SeekOrigin::FS_SEEK_SET)); + MemorySegment footer_segment = + MemorySegment::AllocateHeapMemory(BTreeFileFooter::kEncodingLength, pool_.get()); + ASSERT_OK_AND_ASSIGN(int64_t footer_bytes_read, + index_input->Read(footer_segment.MutableData(), footer_segment.Size())); + ASSERT_EQ(footer_bytes_read, BTreeFileFooter::kEncodingLength); + MemorySliceInput footer_input = MemorySlice::Wrap(footer_segment).ToInput(); + ASSERT_OK_AND_ASSIGN(auto footer, BTreeFileFooter::Read(&footer_input)); + ASSERT_FALSE(footer->GetBloomFilterHandle().has_value()); + ASSERT_OK(index_input->Close()); + auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // All rows are null: {0,1,2,3}, No non-null rows @@ -1647,7 +1742,9 @@ TEST_F(BTreeGlobalIndexIntegrationTest, WriteAndReadLargeDataWithSmallBlocks) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); ASSERT_OK_AND_ASSIGN(auto reader, - read_indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + read_indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // check input stream and read buffer size { @@ -1763,10 +1860,12 @@ TEST_P(BTreeGlobalIndexIntegrationTest, CreateReaderWithMultiFieldSchema) { {BtreeDefs::kBtreeIndexCompression, GetParam()}}; ASSERT_OK_AND_ASSIGN(auto indexer, BTreeGlobalIndexer::Create(options)); - GlobalIndexIOMeta meta("fake_path", 100, nullptr); + GlobalIndexIOMeta meta("fake_path", 100, /*row_count=*/0, nullptr); std::vector metas = {meta}; - ASSERT_NOK_WITH_MSG(indexer->CreateReader(c_schema.get(), file_reader, metas, pool_), + ASSERT_NOK_WITH_MSG(indexer->CreateReader(c_schema.get(), file_reader, metas, + /*total_row_count=*/0, + /*row_ranges=*/std::nullopt, pool_), "supposed to have single field"); } @@ -1777,7 +1876,8 @@ TEST_F(BTreeGlobalIndexIntegrationTest, CreateWriterWithMissingField) { ASSERT_TRUE(struct_type); ASSERT_NOK_WITH_MSG( BTreeGlobalIndexWriter::Create("nonexistent_field", struct_type, file_writer, 4096, - compression_factory_, pool_), + compression_factory_, /*bloom_filter_enabled=*/false, + pool_), "not in arrow_array when Create BTreeGlobalIndexWriter"); } @@ -1873,7 +1973,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, InvalidReadOptions) { ASSERT_OK_AND_ASSIGN(auto indexer, BTreeGlobalIndexer::Create(options)); auto file_reader = std::make_shared(fs_, base_path_); - ASSERT_NOK_WITH_MSG(indexer->CreateReader(c_schema.get(), file_reader, /*files=*/{}, pool_), + ASSERT_NOK_WITH_MSG(indexer->CreateReader(c_schema.get(), file_reader, /*files=*/{}, + /*total_row_count=*/0, + /*row_ranges=*/std::nullopt, pool_), "In BTreeGlobalIndexer::CreateReader: option btree-index.read-buffer-size " "is 4GB, exceed INT_MAX or less than 0"); } @@ -1919,7 +2021,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, TestIOException) { // read auto file_reader = std::make_shared(local_fs, local_base); c_schema = CreateArrowSchema(field); - auto reader_result = indexer->CreateReader(c_schema.get(), file_reader, metas, pool_); + auto reader_result = indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_); CHECK_HOOK_STATUS(reader_result.status(), i); auto reader = std::move(reader_result).value(); @@ -1985,8 +2089,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadMultiFilesWithMetaSelector) // SortedFileMetaSelector) auto file_reader = std::make_shared(fs_, base_path_); auto c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, all_metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, all_metas, + TotalRowCount(all_metas), + /*row_ranges=*/std::nullopt, pool_)); // --- VisitEqual: key=12 -> only file1 is selected by meta selector -> row 5 { @@ -2127,8 +2232,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, TestRangeQuery) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, + TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,1,2,3,4,6,7,8}, Null rows: {5} diff --git a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp index 522f1058a..f42d13f85 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_reader.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_reader.cpp @@ -32,7 +32,8 @@ Result> BTreeGlobalIndexReader::Create( const std::shared_ptr& sst_file_reader, RoaringBitmap64&& null_bitmap, const std::optional& min_key_slice, const std::optional& max_key_slice, - const std::shared_ptr& key_serializer) { + const std::shared_ptr& key_serializer, + std::optional row_id_filter) { if (key_serializer == nullptr) { return Status::Invalid("Cannot create BTreeGlobalIndexReader without a key serializer."); } @@ -46,17 +47,19 @@ Result> BTreeGlobalIndexReader::Create( } return std::shared_ptr(new BTreeGlobalIndexReader( sst_file_reader, std::move(null_bitmap), std::move(min_key), std::move(max_key), - min_key_slice, max_key_slice, key_serializer)); + min_key_slice, max_key_slice, key_serializer, std::move(row_id_filter))); } BTreeGlobalIndexReader::BTreeGlobalIndexReader( const std::shared_ptr& sst_file_reader, RoaringBitmap64&& null_bitmap, std::optional min_key, std::optional max_key, std::optional min_key_slice, std::optional max_key_slice, - const std::shared_ptr& key_serializer) + const std::shared_ptr& key_serializer, + std::optional row_id_filter) : sst_file_reader_(sst_file_reader), key_serializer_(key_serializer), null_bitmap_(std::move(null_bitmap)), + row_id_filter_(std::move(row_id_filter)), min_key_(std::move(min_key)), max_key_(std::move(max_key)), min_key_slice_(std::move(min_key_slice)), @@ -73,7 +76,10 @@ Result> BTreeGlobalIndexReader::VisitIsNotNul Result> BTreeGlobalIndexReader::VisitIsNull() { return std::make_shared( [reader = shared_from_this()]() -> Result { - return reader->null_bitmap_; + if (!reader->row_id_filter_) { + return reader->null_bitmap_; + } + return RoaringBitmap64::And(reader->null_bitmap_, reader->row_id_filter_.value()); }); } @@ -220,8 +226,7 @@ Result> BTreeGlobalIndexReader::VisitEqual( const Literal& literal) { return std::make_shared( [reader = shared_from_this(), literal = literal]() -> Result { - return reader->RangeQuery(literal, literal, /*from_inclusive=*/true, - /*to_inclusive=*/true); + return reader->PointQuery(literal); }); } @@ -240,15 +245,33 @@ Result> BTreeGlobalIndexReader::VisitIn( [reader = shared_from_this(), literals = literals]() -> Result { RoaringBitmap64 result; for (const auto& literal : literals) { - PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 literal_result, - reader->RangeQuery(literal, literal, /*from_inclusive=*/true, - /*to_inclusive=*/true)); + // SQL IN never matches NULL. Skip it rather than trying to serialize a null key. + if (literal.IsNull()) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 literal_result, reader->PointQuery(literal)); result |= literal_result; } return result; }); } +Result BTreeGlobalIndexReader::PointQuery(const Literal& literal) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_bytes, key_serializer_->Serialize(literal)); + PAIMON_RETURN_NOT_OK(key_serializer_->ValidateSerializedKey(MemorySlice::Wrap(key_bytes))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_ids, sst_file_reader_->Lookup(key_bytes)); + if (row_ids == nullptr) { + return RoaringBitmap64(); + } + std::vector deserialized_row_ids; + PAIMON_RETURN_NOT_OK(DeserializeRowIds(MemorySlice::Wrap(row_ids), &deserialized_row_ids)); + RoaringBitmap64 result; + if (!deserialized_row_ids.empty()) { + result.AddMany(deserialized_row_ids.size(), deserialized_row_ids.data()); + } + return result; +} + Result> BTreeGlobalIndexReader::VisitNotIn( const std::vector& literals) { return std::make_shared( @@ -433,7 +456,9 @@ Status BTreeGlobalIndexReader::DeserializeRowIds(const MemorySlice& slice, out->reserve(out->size() + static_cast(num_row_ids)); for (int32_t i = 0; i < num_row_ids; i++) { PAIMON_ASSIGN_OR_RAISE(int64_t row_id, input.ReadVarLenLong()); - out->push_back(row_id); + if (!row_id_filter_ || row_id_filter_->Contains(row_id)) { + out->push_back(row_id); + } } return Status::OK(); } diff --git a/src/paimon/common/global_index/btree/btree_global_index_reader.h b/src/paimon/common/global_index/btree/btree_global_index_reader.h index 1e8508b45..c1d7731f8 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_reader.h +++ b/src/paimon/common/global_index/btree/btree_global_index_reader.h @@ -44,7 +44,8 @@ class BTreeGlobalIndexReader : public GlobalIndexReader, const std::shared_ptr& sst_file_reader, RoaringBitmap64&& null_bitmap, const std::optional& min_key_slice, const std::optional& max_key_slice, - const std::shared_ptr& key_serializer); + const std::shared_ptr& key_serializer, + std::optional row_id_filter); Result> VisitIsNotNull() override; @@ -95,11 +96,13 @@ class BTreeGlobalIndexReader : public GlobalIndexReader, RoaringBitmap64&& null_bitmap, std::optional min_key, std::optional max_key, std::optional min_key_slice, std::optional max_key_slice, - const std::shared_ptr& key_serializer); + const std::shared_ptr& key_serializer, + std::optional row_id_filter); Result RangeQuery(const std::optional& from, const std::optional& to, bool from_inclusive, bool to_inclusive); + Result PointQuery(const Literal& literal); Status DeserializeRowIds(const MemorySlice& slice, std::vector* out) const; @@ -108,6 +111,7 @@ class BTreeGlobalIndexReader : public GlobalIndexReader, std::shared_ptr sst_file_reader_; std::shared_ptr key_serializer_; RoaringBitmap64 null_bitmap_; + std::optional row_id_filter_; std::optional min_key_; std::optional max_key_; /// Cached serialized min/max key slices to avoid repeated serialization in RangeQuery. diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp index fd30728d0..a7c9ddacc 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp @@ -19,6 +19,7 @@ #include "paimon/common/global_index/btree/btree_global_index_writer.h" +#include #include #include "arrow/c/bridge.h" @@ -32,28 +33,40 @@ #include "paimon/common/memory/memory_slice_output.h" #include "paimon/common/predicate/literal_converter.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/bloom_filter.h" #include "paimon/common/utils/checked_cast.h" #include "paimon/common/utils/crc32c.h" #include "paimon/common/utils/preconditions.h" #include "paimon/memory/bytes.h" namespace paimon { +namespace { + +constexpr double kBloomFilterFpp = 0.05; + +} // namespace + Result> BTreeGlobalIndexWriter::Create( const std::string& field_name, const std::shared_ptr& arrow_type, const std::shared_ptr& file_writer, int32_t block_size, const std::shared_ptr& compression_factory, - const std::shared_ptr& pool) { + bool bloom_filter_enabled, const std::shared_ptr& pool) { auto key_field = arrow_type->GetFieldByName(field_name); PAIMON_RETURN_NOT_OK(Preconditions::CheckNotNull( key_field, fmt::format("field {} not in arrow_array when Create BTreeGlobalIndexWriter", field_name))); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_serializer, KeySerializer::Create(key_field->type(), pool)); + std::shared_ptr bloom_filter_builder; + if (bloom_filter_enabled) { + PAIMON_ASSIGN_OR_RAISE(bloom_filter_builder, + BloomFilter::DynamicBuilder(kBloomFilterFpp, pool)); + } PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, file_writer->NewFileName(BtreeDefs::kIdentifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr output_stream, file_writer->NewOutputStream(index_file_name)); - auto sst_file_writer = std::make_unique(output_stream, /*bloom_filter=*/nullptr, - block_size, compression_factory, pool); + auto sst_file_writer = std::make_unique( + output_stream, bloom_filter_builder, block_size, compression_factory, pool); return std::shared_ptr(new BTreeGlobalIndexWriter( field_name, arrow_type, key_serializer, file_writer, index_file_name, output_stream, std::move(sst_file_writer), pool)); @@ -96,6 +109,10 @@ Status BTreeGlobalIndexWriter::AddBatch(::ArrowArray* arrow_array, LiteralConverter::ConvertLiteralsFromArray(*value_array, /*own_data=*/true)); for (size_t i = 0; i < literals.size(); ++i) { + if (row_count_ == std::numeric_limits::max()) { + return Status::Invalid("BTree global index row count exceeds INT64_MAX."); + } + ++row_count_; int64_t row_id = relative_row_ids[i]; const auto& literal = literals[i]; if (literal.IsNull()) { @@ -170,8 +187,6 @@ Result> BTreeGlobalIndexWriter::Finish() { // Write null bitmap first PAIMON_ASSIGN_OR_RAISE(std::optional null_bitmap_handle, WriteNullBitmap(output_stream_)); - // write bloom filter (currently is always null, but we could add it for equal - // and in condition.) PAIMON_ASSIGN_OR_RAISE(std::optional bloom_filter_handle, sst_writer_->WriteBloomFilter()); // Write index block @@ -206,7 +221,7 @@ Result> BTreeGlobalIndexWriter::Finish() { // Create GlobalIndexIOMeta std::string file_path = file_writer_->ToPath(index_file_name_); PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name_)); - GlobalIndexIOMeta io_meta(file_path, file_size, meta_bytes); + GlobalIndexIOMeta io_meta(file_path, file_size, row_count_, meta_bytes); return std::vector{io_meta}; } diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.h b/src/paimon/common/global_index/btree/btree_global_index_writer.h index 1078c0b4b..6140fba57 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_writer.h +++ b/src/paimon/common/global_index/btree/btree_global_index_writer.h @@ -21,6 +21,7 @@ #include #include +#include #include "arrow/api.h" #include "paimon/common/global_index/btree/btree_file_footer.h" @@ -64,7 +65,7 @@ class BTreeGlobalIndexWriter : public GlobalIndexWriter { const std::string& field_name, const std::shared_ptr& arrow_type, const std::shared_ptr& file_writer, int32_t block_size, const std::shared_ptr& compression_factory, - const std::shared_ptr& pool); + bool bloom_filter_enabled, const std::shared_ptr& pool); ~BTreeGlobalIndexWriter() override = default; @@ -102,6 +103,7 @@ class BTreeGlobalIndexWriter : public GlobalIndexWriter { std::optional last_key_; // Null bitmap tracking + int64_t row_count_ = 0; RoaringBitmap64 null_bitmap_; std::vector current_row_ids_; }; diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index e9b5ea4cc..847d7f7c2 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -67,6 +67,7 @@ Result> BTreeGlobalIndexer::Create( } Result>> BTreeGlobalIndexer::GetExtraFieldNames() const { + // TODO: Support multi-field composite BTree indexes consistently with Java. return std::optional>(std::nullopt); } @@ -100,17 +101,23 @@ Result> BTreeGlobalIndexer::CreateWriter( int32_t compress_level, OptionsUtils::GetValueFromMap(options_, BtreeDefs::kBtreeIndexCompressionLevel, BtreeDefs::kDefaultBtreeIndexCompressionLevel)); + PAIMON_ASSIGN_OR_RAISE( + bool bloom_filter_enabled, + OptionsUtils::GetValueFromMap(options_, BtreeDefs::kBtreeIndexBloomFilterEnabled, + BtreeDefs::kDefaultBtreeIndexBloomFilterEnabled)); CompressOptions compress_options{compress_str, compress_level}; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr compression_factory, BlockCompressionFactory::Create(compress_options)); return BTreeGlobalIndexWriter::Create(field_name, struct_type, file_writer, static_cast(block_size), compression_factory, - pool); + bloom_filter_enabled, pool); } Result> BTreeGlobalIndexer::CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, const std::shared_ptr& pool) const { + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, + const std::shared_ptr& pool) const { // Get field type from arrow schema PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, arrow::ImportSchema(arrow_schema)); @@ -140,7 +147,8 @@ Result> BTreeGlobalIndexer::CreateReader( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr reader, LazyFilteredBTreeReader::Create(read_buffer_size, files, key_type, file_reader, - cache_manager_, fallback_scan_max_size_, pool, executor)); + cache_manager_, fallback_scan_max_size_, total_row_count, + row_ranges, pool, executor)); return std::shared_ptr(std::move(reader)); } diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.h b/src/paimon/common/global_index/btree/btree_global_indexer.h index 57fe5ce2b..40be14970 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.h +++ b/src/paimon/common/global_index/btree/btree_global_indexer.h @@ -66,7 +66,8 @@ class BTreeGlobalIndexer : public GlobalIndexer { Result> CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, const std::shared_ptr& pool) const override; private: diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp index 06e163905..907faf213 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp @@ -31,24 +31,101 @@ #include "paimon/common/sst/block_cache.h" #include "paimon/common/sst/sst_file_reader.h" #include "paimon/common/utils/crc32c.h" +#include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/io/buffered_input_stream.h" #include "paimon/utils/roaring_bitmap64.h" namespace paimon { +namespace { + +using FullRangeBounds = std::optional>; + +Result GetFullRangeBounds( + const std::vector& files, int64_t total_row_count, + const std::shared_ptr& key_serializer, MemoryPool* pool) { + if (total_row_count <= 0 || files.empty()) { + return std::optional>(); + } + int64_t remaining = total_row_count; + std::optional min_key; + std::optional max_key; + for (const GlobalIndexIOMeta& file : files) { + // A file's min/max does not identify its row IDs. Only skip the entire reader when its + // scalar entries cover the complete local row-ID domain without gaps. + if (file.row_count < 0 || file.row_count > remaining) { + return std::optional>(); + } + remaining -= file.row_count; + if (file.metadata == nullptr) { + return std::optional>(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr meta, + SortedIndexFileMeta::Deserialize(file.metadata, pool)); + if (meta->HasNulls() || meta->FirstKey() == nullptr || meta->LastKey() == nullptr) { + return std::optional>(); + } + PAIMON_ASSIGN_OR_RAISE(Literal first, + key_serializer->Deserialize(MemorySlice::Wrap(meta->FirstKey()))); + PAIMON_ASSIGN_OR_RAISE(Literal last, + key_serializer->Deserialize(MemorySlice::Wrap(meta->LastKey()))); + if (!min_key) { + min_key = first; + } else { + PAIMON_ASSIGN_OR_RAISE(int32_t compare, first.CompareTo(min_key.value())); + if (compare < 0) { + min_key = first; + } + } + if (!max_key) { + max_key = last; + } else { + PAIMON_ASSIGN_OR_RAISE(int32_t compare, last.CompareTo(max_key.value())); + if (compare > 0) { + max_key = last; + } + } + } + if (remaining != 0) { + return std::optional>(); + } + return std::optional>( + std::make_pair(std::move(min_key.value()), std::move(max_key.value()))); +} + +std::optional CreateRowIdFilter( + const std::optional>& row_ranges) { + if (!row_ranges) { + return std::nullopt; + } + RoaringBitmap64 filter; + for (const Range& range : row_ranges.value()) { + filter.AddRange(range.from, range.to + 1); + } + return filter; +} + +} // namespace Result> LazyFilteredBTreeReader::Create( std::optional read_buffer_size, const std::vector& files, const std::shared_ptr& key_type, const std::shared_ptr& file_reader, const std::shared_ptr& cache_manager, int64_t fallback_scan_max_size, + int64_t total_row_count, const std::optional>& row_ranges, const std::shared_ptr& pool, const std::shared_ptr& executor) { + if (total_row_count < 0) { + return Status::Invalid("Total row count must be non-negative."); + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key_serializer, KeySerializer::Create(key_type, pool)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_selector, SortedFileMetaSelector::Create(files, key_serializer)); + PAIMON_ASSIGN_OR_RAISE(FullRangeBounds full_range_bounds, + GetFullRangeBounds(files, total_row_count, key_serializer, pool.get())); return std::shared_ptr(new LazyFilteredBTreeReader( read_buffer_size, std::move(file_selector), key_serializer, file_reader, cache_manager, - fallback_scan_max_size, pool, executor)); + fallback_scan_max_size, total_row_count, std::move(full_range_bounds), + CreateRowIdFilter(row_ranges), pool, executor)); } LazyFilteredBTreeReader::LazyFilteredBTreeReader( @@ -56,13 +133,108 @@ LazyFilteredBTreeReader::LazyFilteredBTreeReader( const std::shared_ptr& key_serializer, const std::shared_ptr& file_reader, const std::shared_ptr& cache_manager, int64_t fallback_scan_max_size, - const std::shared_ptr& pool, const std::shared_ptr& executor) - : SortedFileGlobalIndexReader(std::move(file_selector), fallback_scan_max_size, executor), + int64_t total_row_count, std::optional> full_range_bounds, + std::optional row_id_filter, const std::shared_ptr& pool, + const std::shared_ptr& executor) + : SortedFileGlobalIndexReader(std::move(file_selector), fallback_scan_max_size, total_row_count, + executor), read_buffer_size_(read_buffer_size), pool_(pool), key_serializer_(key_serializer), file_reader_(file_reader), - cache_manager_(cache_manager) {} + cache_manager_(cache_manager), + total_row_count_(total_row_count), + full_range_bounds_(std::move(full_range_bounds)), + row_id_filter_(std::move(row_id_filter)) {} + +Result> LazyFilteredBTreeReader::VisitEqual( + const Literal& literal) { + PAIMON_ASSIGN_OR_RAISE(bool all_match, + AllKeysMatch([&literal](const Literal& key) -> Result { + if (literal.IsNull()) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(int32_t compare, key.CompareTo(literal)); + return compare == 0; + })); + return all_match ? FullResult() : SortedFileGlobalIndexReader::VisitEqual(literal); +} + +Result> LazyFilteredBTreeReader::VisitLessThan( + const Literal& literal) { + PAIMON_ASSIGN_OR_RAISE(bool all_match, + AllKeysMatch([&literal](const Literal& key) -> Result { + if (literal.IsNull()) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(int32_t compare, key.CompareTo(literal)); + return compare < 0; + })); + return all_match ? FullResult() : SortedFileGlobalIndexReader::VisitLessThan(literal); +} + +Result> LazyFilteredBTreeReader::VisitLessOrEqual( + const Literal& literal) { + PAIMON_ASSIGN_OR_RAISE(bool all_match, + AllKeysMatch([&literal](const Literal& key) -> Result { + if (literal.IsNull()) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(int32_t compare, key.CompareTo(literal)); + return compare <= 0; + })); + return all_match ? FullResult() : SortedFileGlobalIndexReader::VisitLessOrEqual(literal); +} + +Result> LazyFilteredBTreeReader::VisitGreaterThan( + const Literal& literal) { + PAIMON_ASSIGN_OR_RAISE(bool all_match, + AllKeysMatch([&literal](const Literal& key) -> Result { + if (literal.IsNull()) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(int32_t compare, key.CompareTo(literal)); + return compare > 0; + })); + return all_match ? FullResult() : SortedFileGlobalIndexReader::VisitGreaterThan(literal); +} + +Result> LazyFilteredBTreeReader::VisitGreaterOrEqual( + const Literal& literal) { + PAIMON_ASSIGN_OR_RAISE(bool all_match, + AllKeysMatch([&literal](const Literal& key) -> Result { + if (literal.IsNull()) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(int32_t compare, key.CompareTo(literal)); + return compare >= 0; + })); + return all_match ? FullResult() : SortedFileGlobalIndexReader::VisitGreaterOrEqual(literal); +} + +Result> LazyFilteredBTreeReader::FullResult() const { + if (total_row_count_ <= 0) { + return BitmapGlobalIndexResult::FromRanges({}); + } + if (row_id_filter_) { + RoaringBitmap64 filter = row_id_filter_.value(); + return std::make_shared( + [filter = std::move(filter)]() -> Result { return filter; }); + } + return BitmapGlobalIndexResult::FromRanges({Range(0, total_row_count_ - 1)}); +} + +Result LazyFilteredBTreeReader::AllKeysMatch( + const std::function(const Literal&)>& predicate) const { + if (!full_range_bounds_) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(bool min_matches, predicate(full_range_bounds_->first)); + if (!min_matches) { + return false; + } + return predicate(full_range_bounds_->second); +} Result> LazyFilteredBTreeReader::VisitVectorSearch( const std::shared_ptr& vector_search) { @@ -123,7 +295,7 @@ Result> LazyFilteredBTreeReader::OpenReader( comparator, block_cache, pool_)); return BTreeGlobalIndexReader::Create(sst_file_reader, std::move(null_bitmap), min_key_slice, - max_key_slice, key_serializer_); + max_key_slice, key_serializer_, row_id_filter_); } Result LazyFilteredBTreeReader::ReadNullBitmap( diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h index ffc8bb3cb..4009b72f5 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.h @@ -19,9 +19,11 @@ #pragma once +#include #include #include #include +#include #include #include "arrow/api.h" @@ -31,6 +33,7 @@ #include "paimon/common/sst/block_cache.h" #include "paimon/common/sst/block_handle.h" #include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/utils/range.h" #include "paimon/utils/roaring_bitmap64.h" namespace paimon { @@ -41,8 +44,15 @@ class LazyFilteredBTreeReader : public SortedFileGlobalIndexReader { const std::shared_ptr& key_type, const std::shared_ptr& file_reader, const std::shared_ptr& cache_manager, int64_t fallback_scan_max_size, + int64_t total_row_count, const std::optional>& row_ranges, const std::shared_ptr& pool, const std::shared_ptr& executor); + Result> VisitEqual(const Literal& literal) override; + Result> VisitLessThan(const Literal& literal) override; + Result> VisitLessOrEqual(const Literal& literal) override; + Result> VisitGreaterThan(const Literal& literal) override; + Result> VisitGreaterOrEqual(const Literal& literal) override; + Result> VisitVectorSearch( const std::shared_ptr& vector_search) override; @@ -59,9 +69,15 @@ class LazyFilteredBTreeReader : public SortedFileGlobalIndexReader { const std::shared_ptr& key_serializer, const std::shared_ptr& file_reader, const std::shared_ptr& cache_manager, - int64_t fallback_scan_max_size, const std::shared_ptr& pool, + int64_t fallback_scan_max_size, int64_t total_row_count, + std::optional> full_range_bounds, + std::optional row_id_filter, + const std::shared_ptr& pool, const std::shared_ptr& executor); + Result> FullResult() const; + Result AllKeysMatch(const std::function(const Literal&)>& predicate) const; + Result> OpenReader(const GlobalIndexIOMeta& meta) override; Result ReadNullBitmap(const std::shared_ptr& cache, const std::optional& block_handle); @@ -72,6 +88,9 @@ class LazyFilteredBTreeReader : public SortedFileGlobalIndexReader { std::shared_ptr key_serializer_; std::shared_ptr file_reader_; std::shared_ptr cache_manager_; + int64_t total_row_count_; + std::optional> full_range_bounds_; + std::optional row_id_filter_; }; } // namespace paimon diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp index 68fd1fba1..80d3a9a58 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp @@ -139,16 +139,35 @@ class LazyFilteredBTreeReaderTest : public ::testing::Test { std::shared_ptr CreateReader( const std::shared_ptr& executor = nullptr, int64_t fallback_scan_max_size = std::numeric_limits::max()) const { + return CreateReader(/*row_ranges=*/std::nullopt, executor, fallback_scan_max_size); + } + + std::shared_ptr CreateReader( + const std::optional>& row_ranges, + const std::shared_ptr& executor, int64_t fallback_scan_max_size) const { auto file_reader = std::make_shared(fs_, base_path_); auto cache_manager = std::make_shared(1024 * 1024, 0.5); EXPECT_OK_AND_ASSIGN( std::shared_ptr reader, LazyFilteredBTreeReader::Create( /*read_buffer_size=*/std::nullopt, all_metas_, arrow::int32(), file_reader, - cache_manager, fallback_scan_max_size, pool_, executor)); + cache_manager, fallback_scan_max_size, TotalRowCount(), + row_ranges, pool_, executor)); return reader; } + void ClearFiles() { + all_metas_.clear(); + } + + int64_t TotalRowCount() const { + int64_t total_row_count = 0; + for (const GlobalIndexIOMeta& meta : all_metas_) { + total_row_count += meta.row_count; + } + return total_row_count; + } + void CheckResult(const std::shared_ptr& result, const std::vector& expected) const { ASSERT_TRUE(result); @@ -205,6 +224,38 @@ TEST_F(LazyFilteredBTreeReaderTest, TestVisitEqualNoMatch) { CheckEmpty(result); } +TEST_F(LazyFilteredBTreeReaderTest, TestFullMatch) { + ClearFiles(); + WriteSingleFile(R"([[5],[5],[5],[5],[5]])", {0, 1, 2, 3, 4}); + + // Disabling fallback scans ensures that range predicates return a result only through the + // full-match shortcut. + std::shared_ptr reader = + CreateReader(/*executor=*/nullptr, /*fallback_scan_max_size=*/0); + ASSERT_OK_AND_ASSIGN(std::shared_ptr equal, + reader->VisitEqual(Literal(5))); + CheckResult(equal, {0, 1, 2, 3, 4}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr less_than, + reader->VisitLessThan(Literal(6))); + CheckResult(less_than, {0, 1, 2, 3, 4}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr less_or_equal, + reader->VisitLessOrEqual(Literal(5))); + CheckResult(less_or_equal, {0, 1, 2, 3, 4}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr greater_than, + reader->VisitGreaterThan(Literal(4))); + CheckResult(greater_than, {0, 1, 2, 3, 4}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr greater_or_equal, + reader->VisitGreaterOrEqual(Literal(5))); + CheckResult(greater_or_equal, {0, 1, 2, 3, 4}); + + std::vector row_ranges = {Range(1, 3)}; + std::shared_ptr filtered_reader = + CreateReader(row_ranges, /*executor=*/nullptr, /*fallback_scan_max_size=*/0); + ASSERT_OK_AND_ASSIGN(std::shared_ptr filtered, + filtered_reader->VisitLessThan(Literal(6))); + CheckResult(filtered, {1, 2, 3}); +} + // --- VisitNotEqual --- TEST_F(LazyFilteredBTreeReaderTest, TestVisitNotEqual) { @@ -320,17 +371,8 @@ TEST_F(LazyFilteredBTreeReaderTest, RejectsMismatchedLiteralEncoding) { auto reader = CreateReader(); Literal int8_literal(static_cast(1)); - ASSERT_OK_AND_ASSIGN(std::shared_ptr not_equal, - reader->VisitNotEqual(int8_literal)); - auto not_equal_bitmap = std::dynamic_pointer_cast(not_equal); - ASSERT_TRUE(not_equal_bitmap != nullptr); - ASSERT_NOK(not_equal_bitmap->GetBitmap()); - - ASSERT_OK_AND_ASSIGN(std::shared_ptr not_in, - reader->VisitNotIn({int8_literal})); - auto not_in_bitmap = std::dynamic_pointer_cast(not_in); - ASSERT_TRUE(not_in_bitmap != nullptr); - ASSERT_NOK(not_in_bitmap->GetBitmap()); + ASSERT_NOK(reader->VisitNotEqual(int8_literal)); + ASSERT_NOK(reader->VisitNotIn({int8_literal})); } // --- VisitIsNull --- @@ -401,7 +443,8 @@ TEST_F(LazyFilteredBTreeReaderTest, TestEmptyFilesList) { ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, LazyFilteredBTreeReader::Create( /*read_buffer_size=*/std::nullopt, empty_metas, arrow::int32(), - file_reader, cache_manager, std::numeric_limits::max(), pool_, + file_reader, cache_manager, std::numeric_limits::max(), + /*total_row_count=*/0, /*row_ranges=*/std::nullopt, pool_, /*executor=*/nullptr)); // Any query on empty files should return empty bitmap @@ -496,7 +539,9 @@ TEST_F(LazyFilteredBTreeReaderTest, TestParallelEmptyFilesList) { std::shared_ptr reader, LazyFilteredBTreeReader::Create(/*read_buffer_size=*/std::nullopt, empty_metas, arrow::int32(), file_reader, cache_manager, - std::numeric_limits::max(), pool_, executor)); + std::numeric_limits::max(), + /*total_row_count=*/0, /*row_ranges=*/std::nullopt, pool_, + executor)); Literal literal_1(1); ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(literal_1)); CheckEmpty(result); @@ -506,10 +551,11 @@ TEST_F(LazyFilteredBTreeReaderTest, RejectsMalformedFileMetadata) { auto file_reader = std::make_shared(fs_, base_path_); auto cache_manager = std::make_shared(1024 * 1024, 0.5); std::vector invalid_metas = { - GlobalIndexIOMeta("invalid", 1, /*metadata=*/nullptr)}; + GlobalIndexIOMeta("invalid", 1, /*row_count=*/1, /*metadata=*/nullptr)}; ASSERT_NOK(LazyFilteredBTreeReader::Create( /*read_buffer_size=*/std::nullopt, invalid_metas, arrow::int32(), file_reader, - cache_manager, std::numeric_limits::max(), pool_, /*executor=*/nullptr)); + cache_manager, std::numeric_limits::max(), /*total_row_count=*/1, + /*row_ranges=*/std::nullopt, pool_, /*executor=*/nullptr)); } } // namespace paimon::test diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp index 71f9f4bc1..5c0953a48 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp @@ -38,7 +38,9 @@ Result> RangeBitmapGlobalIndex::CreateWriter( Result> RangeBitmapGlobalIndex::CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, const std::shared_ptr& pool) const { + const std::vector& files, int64_t /*total_row_count*/, + const std::optional>& /*row_ranges*/, + const std::shared_ptr& pool) const { if (files.size() != 1) { return Status::Invalid( "invalid GlobalIndexIOMeta for RangeBitmapGlobalIndex, exist multiple metas"); diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h index 3e60c4287..b516be830 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h @@ -43,7 +43,8 @@ class RangeBitmapGlobalIndex : public GlobalIndexer { Result> CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, const std::shared_ptr& pool) const override; private: diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp index c198fbf70..bbba051a1 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp @@ -84,6 +84,7 @@ class RangeBitmapGlobalIndexTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(auto result_metas, global_writer->Finish()); // check meta EXPECT_EQ(result_metas.size(), 1); + EXPECT_EQ(result_metas[0].row_count, array->length()); auto file_name = PathUtil::GetName(result_metas[0].file_path); EXPECT_TRUE(StringUtils::StartsWith(file_name, "range-bitmap-global-index-")); EXPECT_TRUE(StringUtils::EndsWith(file_name, ".index")); @@ -114,7 +115,8 @@ class RangeBitmapGlobalIndexTest : public ::testing::Test { fs_, path_factory, /*checkpoint_path_factory=*/nullptr); EXPECT_OK_AND_ASSIGN( auto global_index_reader, - global_index->CreateReader(CreateArrowSchema(type).get(), file_reader, {meta}, pool_)); + global_index->CreateReader(CreateArrowSchema(type).get(), file_reader, {meta}, + meta.row_count, /*row_ranges=*/std::nullopt, pool_)); EXPECT_TRUE(global_index_reader); return global_index_reader; } diff --git a/src/paimon/common/global_index/sorted_file_global_index_reader.cpp b/src/paimon/common/global_index/sorted_file_global_index_reader.cpp index 04e665bce..6552132af 100644 --- a/src/paimon/common/global_index/sorted_file_global_index_reader.cpp +++ b/src/paimon/common/global_index/sorted_file_global_index_reader.cpp @@ -19,6 +19,7 @@ #include "paimon/common/global_index/sorted_file_global_index_reader.h" +#include #include #include #include @@ -40,15 +41,20 @@ std::shared_ptr EmptyResult() { SortedFileGlobalIndexReader::SortedFileGlobalIndexReader( std::unique_ptr file_selector, int64_t fallback_scan_max_size, - std::shared_ptr executor) + int64_t total_row_count, std::shared_ptr executor) : file_selector_(std::move(file_selector)), fallback_scan_max_size_(fallback_scan_max_size), - executor_(std::move(executor)) {} + executor_(std::move(executor)), + total_row_count_(total_row_count) { + assert(total_row_count_ >= 0); +} Result> SortedFileGlobalIndexReader::VisitIsNotNull() { - return VisitParallel( - [this]() { return file_selector_->VisitIsNotNull(); }, - [](const std::shared_ptr& reader) { return reader->VisitIsNotNull(); }); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr null_result, VisitIsNull()); + if (null_result == nullptr) { + return null_result; + } + return Complement({null_result}); } Result> SortedFileGlobalIndexReader::VisitIsNull() { @@ -67,10 +73,15 @@ Result> SortedFileGlobalIndexReader::VisitEqu Result> SortedFileGlobalIndexReader::VisitNotEqual( const Literal& literal) { - return VisitParallel([this, &literal]() { return file_selector_->VisitNotEqual(literal); }, - [&literal](const std::shared_ptr& reader) { - return reader->VisitNotEqual(literal); - }); + if (literal.IsNull()) { + return EmptyResult(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr null_result, VisitIsNull()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr equal_result, VisitEqual(literal)); + if (null_result == nullptr || equal_result == nullptr) { + return std::shared_ptr(nullptr); + } + return Complement({null_result, equal_result}); } Result> SortedFileGlobalIndexReader::VisitLessThan( @@ -131,10 +142,17 @@ Result> SortedFileGlobalIndexReader::VisitIn( Result> SortedFileGlobalIndexReader::VisitNotIn( const std::vector& literals) { - return VisitParallel([this, &literals]() { return file_selector_->VisitNotIn(literals); }, - [&literals](const std::shared_ptr& reader) { - return reader->VisitNotIn(literals); - }); + for (const Literal& literal : literals) { + if (literal.IsNull()) { + return EmptyResult(); + } + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr null_result, VisitIsNull()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr in_result, VisitIn(literals)); + if (null_result == nullptr || in_result == nullptr) { + return std::shared_ptr(nullptr); + } + return Complement({null_result, in_result}); } Result> SortedFileGlobalIndexReader::VisitStartsWith( @@ -277,6 +295,27 @@ Result> SortedFileGlobalIndexReader::GetOrCre return reader; } +Result> SortedFileGlobalIndexReader::Complement( + const std::vector>& excluded_results) const { + int64_t total_row_count = total_row_count_; + auto supplier = [total_row_count, excluded_results]() -> Result { + RoaringBitmap64 result; + result.AddRange(/*min=*/0, /*max=*/total_row_count); + for (const std::shared_ptr& excluded_result : excluded_results) { + std::shared_ptr bitmap_result = + std::dynamic_pointer_cast(excluded_result); + if (bitmap_result == nullptr) { + return Status::Invalid( + "SortedFileGlobalIndexReader expects bitmap index results."); + } + PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* excluded, bitmap_result->GetBitmap()); + result -= *excluded; + } + return result; + }; + return std::make_shared(std::move(supplier)); +} + bool SortedFileGlobalIndexReader::FallbackScanEnabled( const std::vector& files) const { if (fallback_scan_max_size_ <= 0) { diff --git a/src/paimon/common/global_index/sorted_file_global_index_reader.h b/src/paimon/common/global_index/sorted_file_global_index_reader.h index fd4a33b4f..ce28987ff 100644 --- a/src/paimon/common/global_index/sorted_file_global_index_reader.h +++ b/src/paimon/common/global_index/sorted_file_global_index_reader.h @@ -58,7 +58,9 @@ class SortedFileGlobalIndexReader : public GlobalIndexReader { protected: SortedFileGlobalIndexReader(std::unique_ptr file_selector, - int64_t fallback_scan_max_size, std::shared_ptr executor); + int64_t fallback_scan_max_size, + int64_t total_row_count, + std::shared_ptr executor); virtual Result> OpenReader( const GlobalIndexIOMeta& meta) = 0; @@ -74,6 +76,8 @@ class SortedFileGlobalIndexReader : public GlobalIndexReader { ReaderAction visitor); Result> VisitSelectedFiles( const std::vector& files, ReaderAction visitor); + Result> Complement( + const std::vector>& excluded_results) const; Result> GetOrCreateReader(const GlobalIndexIOMeta& meta); bool FallbackScanEnabled(const std::vector& files) const; @@ -82,6 +86,7 @@ class SortedFileGlobalIndexReader : public GlobalIndexReader { int64_t fallback_scan_max_size_; std::map> reader_cache_; std::shared_ptr executor_; + int64_t total_row_count_; }; } // namespace paimon diff --git a/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp b/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp index 9a6f482eb..054535ac2 100644 --- a/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp +++ b/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp @@ -57,12 +57,12 @@ class SortedFileMetaSelectorTest : public ::testing::Test { // file6: only-nulls file (no keys, has_nulls=true) auto meta6 = std::make_shared(nullptr, nullptr, true); files_ = { - GlobalIndexIOMeta("file1", 1, meta1->Serialize(pool_.get())), - GlobalIndexIOMeta("file2", 1, meta2->Serialize(pool_.get())), - GlobalIndexIOMeta("file3", 1, meta3->Serialize(pool_.get())), - GlobalIndexIOMeta("file4", 1, meta4->Serialize(pool_.get())), - GlobalIndexIOMeta("file5", 1, meta5->Serialize(pool_.get())), - GlobalIndexIOMeta("file6", 1, meta6->Serialize(pool_.get())), + GlobalIndexIOMeta("file1", 1, /*row_count=*/1, meta1->Serialize(pool_.get())), + GlobalIndexIOMeta("file2", 1, /*row_count=*/1, meta2->Serialize(pool_.get())), + GlobalIndexIOMeta("file3", 1, /*row_count=*/1, meta3->Serialize(pool_.get())), + GlobalIndexIOMeta("file4", 1, /*row_count=*/1, meta4->Serialize(pool_.get())), + GlobalIndexIOMeta("file5", 1, /*row_count=*/1, meta5->Serialize(pool_.get())), + GlobalIndexIOMeta("file6", 1, /*row_count=*/1, meta6->Serialize(pool_.get())), }; } @@ -244,10 +244,11 @@ TEST_F(SortedFileMetaSelectorTest, TestVisitStartsWith) { auto only_nulls_meta = std::make_shared(nullptr, nullptr, /*has_nulls=*/true); std::vector files = { - GlobalIndexIOMeta("first", 1, first_meta->Serialize(pool.get())), - GlobalIndexIOMeta("second", 1, second_meta->Serialize(pool.get())), - GlobalIndexIOMeta("third", 1, third_meta->Serialize(pool.get())), - GlobalIndexIOMeta("only_nulls", 1, only_nulls_meta->Serialize(pool.get())), + GlobalIndexIOMeta("first", 1, /*row_count=*/1, first_meta->Serialize(pool.get())), + GlobalIndexIOMeta("second", 1, /*row_count=*/1, second_meta->Serialize(pool.get())), + GlobalIndexIOMeta("third", 1, /*row_count=*/1, third_meta->Serialize(pool.get())), + GlobalIndexIOMeta("only_nulls", 1, /*row_count=*/1, + only_nulls_meta->Serialize(pool.get())), }; ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, SortedFileMetaSelector::Create(files, key_serializer)); @@ -328,9 +329,9 @@ TEST_F(SortedFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) { serialize("zzz.com", 7), false); auto null_meta = std::make_shared(nullptr, nullptr, true); std::vector files = { - GlobalIndexIOMeta("file_empty", 1, empty_meta->Serialize(pool.get())), - GlobalIndexIOMeta("file_normal", 1, normal_meta->Serialize(pool.get())), - GlobalIndexIOMeta("file_nulls", 1, null_meta->Serialize(pool.get())), + GlobalIndexIOMeta("file_empty", 1, /*row_count=*/1, empty_meta->Serialize(pool.get())), + GlobalIndexIOMeta("file_normal", 1, /*row_count=*/1, normal_meta->Serialize(pool.get())), + GlobalIndexIOMeta("file_nulls", 1, /*row_count=*/1, null_meta->Serialize(pool.get())), }; ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, @@ -356,36 +357,42 @@ TEST_F(SortedFileMetaSelectorTest, RejectsMalformedFileMetadata) { std::shared_ptr pool = GetDefaultPool(); ASSERT_OK_AND_ASSIGN(std::shared_ptr key_serializer, KeySerializer::Create(arrow::int32(), pool)); - std::vector files = {GlobalIndexIOMeta("missing", 1, /*metadata=*/nullptr)}; + std::vector files = { + GlobalIndexIOMeta("missing", 1, /*row_count=*/1, /*metadata=*/nullptr)}; ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto truncated = std::make_shared(std::string(4, '\0'), pool.get()); - files = {GlobalIndexIOMeta("truncated", 1, truncated)}; + files = {GlobalIndexIOMeta("truncated", 1, /*row_count=*/1, truncated)}; ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto short_key = std::make_shared(std::string(1, '\0'), pool.get()); auto invalid_key_meta = std::make_shared(short_key, short_key, false); - files = {GlobalIndexIOMeta("invalid-key", 1, invalid_key_meta->Serialize(pool.get()))}; + files = {GlobalIndexIOMeta("invalid-key", 1, /*row_count=*/1, + invalid_key_meta->Serialize(pool.get()))}; ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto reversed_meta = std::make_shared(SerializeInt(10), SerializeInt(1), false); - files = {GlobalIndexIOMeta("reversed-range", 1, reversed_meta->Serialize(pool.get()))}; + files = {GlobalIndexIOMeta("reversed-range", 1, /*row_count=*/1, + reversed_meta->Serialize(pool.get()))}; ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto only_first_meta = std::make_shared(SerializeInt(1), /*last_key=*/nullptr, false); - files = {GlobalIndexIOMeta("only-first-key", 1, only_first_meta->Serialize(pool.get()))}; + files = {GlobalIndexIOMeta("only-first-key", 1, /*row_count=*/1, + only_first_meta->Serialize(pool.get()))}; ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto only_last_meta = std::make_shared(/*first_key=*/nullptr, SerializeInt(1), false); - files = {GlobalIndexIOMeta("only-last-key", 1, only_last_meta->Serialize(pool.get()))}; + files = {GlobalIndexIOMeta("only-last-key", 1, /*row_count=*/1, + only_last_meta->Serialize(pool.get()))}; ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); auto empty_nonnull_meta = std::make_shared(/*first_key=*/nullptr, /*last_key=*/nullptr, false); - files = {GlobalIndexIOMeta("empty-nonnull", 1, empty_nonnull_meta->Serialize(pool.get()))}; + files = {GlobalIndexIOMeta("empty-nonnull", 1, /*row_count=*/1, + empty_nonnull_meta->Serialize(pool.get()))}; ASSERT_NOK(SortedFileMetaSelector::Create(files, key_serializer)); } diff --git a/src/paimon/common/global_index/union_global_index_reader_test.cpp b/src/paimon/common/global_index/union_global_index_reader_test.cpp index 5399073b4..6d44f27f1 100644 --- a/src/paimon/common/global_index/union_global_index_reader_test.cpp +++ b/src/paimon/common/global_index/union_global_index_reader_test.cpp @@ -43,7 +43,8 @@ class FakeSortedFileGlobalIndexReader : public SortedFileGlobalIndexReader { std::map>&& readers, std::shared_ptr executor) : SortedFileGlobalIndexReader(/*file_selector=*/nullptr, - /*fallback_scan_max_size=*/0, std::move(executor)), + /*fallback_scan_max_size=*/0, + /*total_row_count=*/0, std::move(executor)), readers_(std::move(readers)) {} Result> VisitSelectedFilesForTest( @@ -276,8 +277,10 @@ TEST_F(UnionGlobalIndexReaderTest, TestSortedFileSelectedFilesSkipNullptr) { {"unsupported", unsupported_reader}, {"supported", supported_reader}}; FakeSortedFileGlobalIndexReader sorted_reader(std::move(readers), CreateDefaultExecutor()); std::vector files = { - GlobalIndexIOMeta("unsupported", /*file_size=*/1, /*metadata=*/nullptr), - GlobalIndexIOMeta("supported", /*file_size=*/1, /*metadata=*/nullptr)}; + GlobalIndexIOMeta("unsupported", /*file_size=*/1, /*row_count=*/0, + /*metadata=*/nullptr), + GlobalIndexIOMeta("supported", /*file_size=*/1, /*row_count=*/0, + /*metadata=*/nullptr)}; ASSERT_OK_AND_ASSIGN(auto result, sorted_reader.VisitSelectedFilesForTest(files)); CheckResult(result, {1, 2}); diff --git a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h index a85b2dc45..fc2347d6f 100644 --- a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h +++ b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h @@ -69,7 +69,7 @@ class FileIndexWriterWrapper : public GlobalIndexWriter { PAIMON_RETURN_NOT_OK(out->Flush()); PAIMON_RETURN_NOT_OK(out->Close()); GlobalIndexIOMeta meta(file_manager_->ToPath(file_name), /*file_size=*/bytes->size(), - /*metadata=*/nullptr); + /*row_count=*/count_, /*metadata=*/nullptr); return std::vector({meta}); } diff --git a/src/paimon/common/lookup/lookup_store_factory.cpp b/src/paimon/common/lookup/lookup_store_factory.cpp index d6cb8d400..1f0ad8f84 100644 --- a/src/paimon/common/lookup/lookup_store_factory.cpp +++ b/src/paimon/common/lookup/lookup_store_factory.cpp @@ -30,17 +30,11 @@ Result> LookupStoreFactory::Create( std::move(comparator), cache_manager, options.GetCachePageSize(), compression_factory); } -Result> LookupStoreFactory::BfGenerator(int64_t row_count, - const CoreOptions& options, - MemoryPool* pool) { +Result> LookupStoreFactory::BfGenerator( + int64_t row_count, const CoreOptions& options, MemoryPool* pool) { if (row_count <= 0 || !options.LookupCacheBloomFilterEnabled()) { - return std::shared_ptr(); + return std::shared_ptr(); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bloom_filter, - BloomFilter::Create(row_count, options.GetLookupCacheBloomFilterFpp())); - MemorySegment memory_segment = - MemorySegment::AllocateHeapMemory(bloom_filter->ByteLength(), pool); - PAIMON_RETURN_NOT_OK(bloom_filter->SetMemorySegment(memory_segment)); - return bloom_filter; + return BloomFilter::FixedBuilder(row_count, options.GetLookupCacheBloomFilterFpp(), pool); } } // namespace paimon diff --git a/src/paimon/common/lookup/lookup_store_factory.h b/src/paimon/common/lookup/lookup_store_factory.h index 222dc7b50..e43c640de 100644 --- a/src/paimon/common/lookup/lookup_store_factory.h +++ b/src/paimon/common/lookup/lookup_store_factory.h @@ -54,7 +54,7 @@ class LookupStoreFactory { virtual Result> CreateWriter( const std::shared_ptr& fs, const std::string& file_path, - const std::shared_ptr& bloom_filter, + const std::shared_ptr& bloom_filter_builder, const std::shared_ptr& pool) const = 0; virtual Result> CreateReader( @@ -65,8 +65,8 @@ class LookupStoreFactory { MemorySlice::SliceComparator comparator, const std::shared_ptr& cache_manager, const CoreOptions& options); - static Result> BfGenerator(int64_t row_count, - const CoreOptions& options, - MemoryPool* pool); + static Result> BfGenerator(int64_t row_count, + const CoreOptions& options, + MemoryPool* pool); }; } // namespace paimon diff --git a/src/paimon/common/lookup/sort/sort_lookup_store_factory.cpp b/src/paimon/common/lookup/sort/sort_lookup_store_factory.cpp index e376dffa7..a2c851d0e 100644 --- a/src/paimon/common/lookup/sort/sort_lookup_store_factory.cpp +++ b/src/paimon/common/lookup/sort/sort_lookup_store_factory.cpp @@ -24,13 +24,14 @@ namespace paimon { Result> SortLookupStoreFactory::CreateWriter( const std::shared_ptr& fs, const std::string& file_path, - const std::shared_ptr& bloom_filter, + const std::shared_ptr& bloom_filter_builder, const std::shared_ptr& pool) const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, fs->Create(file_path, /*overwrite=*/false)); return std::make_unique( out, - std::make_shared(out, bloom_filter, block_size_, compression_factory_, pool), + std::make_shared(out, bloom_filter_builder, block_size_, + compression_factory_, pool), pool); } diff --git a/src/paimon/common/lookup/sort/sort_lookup_store_factory.h b/src/paimon/common/lookup/sort/sort_lookup_store_factory.h index 3aa193ce3..272deec35 100644 --- a/src/paimon/common/lookup/sort/sort_lookup_store_factory.h +++ b/src/paimon/common/lookup/sort/sort_lookup_store_factory.h @@ -79,7 +79,7 @@ class SortLookupStoreFactory : public LookupStoreFactory { Result> CreateWriter( const std::shared_ptr& fs, const std::string& file_path, - const std::shared_ptr& bloom_filter, + const std::shared_ptr& bloom_filter_builder, const std::shared_ptr& pool) const override; Result> CreateReader( diff --git a/src/paimon/common/sst/sst_file_io_test.cpp b/src/paimon/common/sst/sst_file_io_test.cpp index 54d4b1b13..180be3ae3 100644 --- a/src/paimon/common/sst/sst_file_io_test.cpp +++ b/src/paimon/common/sst/sst_file_io_test.cpp @@ -93,10 +93,10 @@ TEST_P(SstFileIOTest, TestSimple) { fs_->Create(index_path, /*overwrite=*/false)); // write data - ASSERT_OK_AND_ASSIGN(std::shared_ptr bf, BloomFilter::Create(30, 0.01)); - auto seg_for_bf = MemorySegment::AllocateHeapMemory(bf->ByteLength(), pool_.get()); - ASSERT_OK(bf->SetMemorySegment(seg_for_bf)); - auto writer = std::make_shared(out, bf, 50, factory, pool_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr bloom_filter_builder, + BloomFilter::FixedBuilder(30, 0.01, pool_.get())); + auto writer = + std::make_shared(out, bloom_filter_builder, 50, factory, pool_); std::set value_hash; // k1-k5 for (size_t i = 1; i <= 5; i++) { @@ -234,10 +234,10 @@ TEST_F(SstFileIOTest, TestIOException) { CHECK_HOOK_STATUS(out_result.status(), i); std::shared_ptr out = std::move(out_result).value(); - ASSERT_OK_AND_ASSIGN(std::shared_ptr bf, BloomFilter::Create(30, 0.01)); - MemorySegment seg_for_bf = MemorySegment::AllocateHeapMemory(bf->ByteLength(), pool_.get()); - ASSERT_OK(bf->SetMemorySegment(seg_for_bf)); - auto writer = std::make_shared(out, bf, 50, factory, pool_); + ASSERT_OK_AND_ASSIGN(std::shared_ptr bloom_filter_builder, + BloomFilter::FixedBuilder(30, 0.01, pool_.get())); + auto writer = + std::make_shared(out, bloom_filter_builder, 50, factory, pool_); bool write_failed = false; for (size_t j = 1; j <= 5; j++) { diff --git a/src/paimon/common/sst/sst_file_reader.cpp b/src/paimon/common/sst/sst_file_reader.cpp index 315bc199a..dea87f393 100644 --- a/src/paimon/common/sst/sst_file_reader.cpp +++ b/src/paimon/common/sst/sst_file_reader.cpp @@ -29,7 +29,8 @@ Result> SstFileReader::Create( const std::optional& bloom_filter_handle, MemorySlice::SliceComparator comparator, const std::shared_ptr& block_cache, const std::shared_ptr& pool) { - // read bloom filter directly now + // TODO: Align with Java FileBasedBloomFilter by probing the cache first and loading the Bloom + // filter on demand during point lookups instead of reading it eagerly when creating the reader. std::shared_ptr bloom_filter = nullptr; if (bloom_filter_handle.has_value() && (bloom_filter_handle->ExpectedEntries() || bloom_filter_handle->Size() || diff --git a/src/paimon/common/sst/sst_file_writer.cpp b/src/paimon/common/sst/sst_file_writer.cpp index f2b3b2dee..214db6666 100644 --- a/src/paimon/common/sst/sst_file_writer.cpp +++ b/src/paimon/common/sst/sst_file_writer.cpp @@ -24,10 +24,14 @@ namespace paimon { SstFileWriter::SstFileWriter(const std::shared_ptr& out, - const std::shared_ptr& bloom_filter, int32_t block_size, + const std::shared_ptr& bloom_filter_builder, + int32_t block_size, const std::shared_ptr& factory, const std::shared_ptr& pool) - : pool_(pool), out_(out), bloom_filter_(bloom_filter), block_size_(block_size) { + : pool_(pool), + out_(out), + bloom_filter_builder_(bloom_filter_builder), + block_size_(block_size) { // block_size * 1.1 exceeds INT32_MAX for block_size above ~1.9GB; saturate instead of // relying on the undefined double->int32_t conversion. data_block_writer_ = @@ -44,12 +48,9 @@ Status SstFileWriter::Write(std::shared_ptr&& key, std::shared_ptr if (data_block_writer_->Memory() > block_size_) { PAIMON_RETURN_NOT_OK(Flush()); } - if (bloom_filter_) { - // Double-check that bloom_filter_ is valid - if (!bloom_filter_->GetBitSet()) { - return Status::Invalid("Bloom filter bit set is null"); - } - PAIMON_RETURN_NOT_OK(bloom_filter_->AddHash(MurmurHashUtils::HashBytes(key))); + if (bloom_filter_builder_) { + PAIMON_RETURN_NOT_OK( + bloom_filter_builder_->AddHash(MurmurHashUtils::HashBytes(key))); } return Status::OK(); } @@ -72,13 +73,18 @@ Result SstFileWriter::WriteIndexBlock() { } Result> SstFileWriter::WriteBloomFilter() { - if (!bloom_filter_) { + if (!bloom_filter_builder_) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bloom_filter, + bloom_filter_builder_->Build()); + if (!bloom_filter) { return std::optional(); } - auto bf_slice = bloom_filter_->GetBitSet()->ToSlice(); + auto bf_slice = bloom_filter->GetBitSet()->ToSlice(); auto data = bf_slice.ReadStringView(); PAIMON_ASSIGN_OR_RAISE(int64_t bloom_filter_pos, out_->GetPos()); - BloomFilterHandle handle(bloom_filter_pos, data.size(), bloom_filter_->ExpectedEntries()); + BloomFilterHandle handle(bloom_filter_pos, data.size(), bloom_filter->ExpectedEntries()); PAIMON_RETURN_NOT_OK(WriteBytes(data.data(), data.size())); diff --git a/src/paimon/common/sst/sst_file_writer.h b/src/paimon/common/sst/sst_file_writer.h index 15ffd1a75..d84a4b51c 100644 --- a/src/paimon/common/sst/sst_file_writer.h +++ b/src/paimon/common/sst/sst_file_writer.h @@ -44,7 +44,8 @@ class MemoryPool; class PAIMON_EXPORT SstFileWriter { public: SstFileWriter(const std::shared_ptr& out, - const std::shared_ptr& bloom_filter, int32_t block_size, + const std::shared_ptr& bloom_filter_builder, + int32_t block_size, const std::shared_ptr& factory, const std::shared_ptr& pool); @@ -77,7 +78,7 @@ class PAIMON_EXPORT SstFileWriter { std::shared_ptr pool_; std::shared_ptr out_; - std::shared_ptr bloom_filter_; + std::shared_ptr bloom_filter_builder_; BlockCompressionType compression_type_; std::shared_ptr compressor_; diff --git a/src/paimon/common/utils/bloom_filter.cpp b/src/paimon/common/utils/bloom_filter.cpp index ba9fa4b44..121b5bd4f 100644 --- a/src/paimon/common/utils/bloom_filter.cpp +++ b/src/paimon/common/utils/bloom_filter.cpp @@ -22,8 +22,84 @@ #include #include #include +#include + +#include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" namespace paimon { +namespace { + +Status ValidateFpp(double fpp) { + if (!std::isfinite(fpp) || fpp <= 0.0 || fpp >= 1.0) { + return Status::Invalid("fpp must be greater than 0 and less than 1 for bloom filter"); + } + return Status::OK(); +} + +class FixedBloomFilterBuilder : public BloomFilter::Builder { + public: + explicit FixedBloomFilterBuilder(const std::shared_ptr& bloom_filter) + : bloom_filter_(bloom_filter) {} + + Status AddHash(int32_t hash) override { + return bloom_filter_->AddHash(hash); + } + + Result> Build() override { + return bloom_filter_; + } + + private: + std::shared_ptr bloom_filter_; +}; + +class DynamicBloomFilterBuilder : public BloomFilter::Builder { + public: + DynamicBloomFilterBuilder(double fpp, const std::shared_ptr& pool) + : fpp_(fpp), pool_(pool) { + hashes_.reserve(kInitialHashCapacity); + } + + Status AddHash(int32_t hash) override { + if (!built_) { + hashes_.push_back(hash); + } + return Status::OK(); + } + + Result> Build() override { + if (built_) { + return bloom_filter_; + } + if (hashes_.empty()) { + built_ = true; + return std::shared_ptr(); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr builder, + BloomFilter::FixedBuilder(static_cast(hashes_.size()), fpp_, pool_.get())); + for (int32_t hash : hashes_) { + PAIMON_RETURN_NOT_OK(builder->AddHash(hash)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bloom_filter, builder->Build()); + bloom_filter_ = std::move(bloom_filter); + built_ = true; + std::vector().swap(hashes_); + return bloom_filter_; + } + + private: + static constexpr int32_t kInitialHashCapacity = 1024; + + double fpp_; + std::shared_ptr pool_; + std::vector hashes_; + std::shared_ptr bloom_filter_; + bool built_ = false; +}; + +} // namespace int32_t BloomFilter::OptimalNumOfBits(int64_t expected_entries, double fpp) { if (expected_entries <= 0 || fpp <= 0.0 || fpp >= 1.0) { @@ -48,14 +124,35 @@ Result> BloomFilter::Create(int64_t expected_entrie if (expected_entries <= 0) { return Status::Invalid("expected entries must be greater than 0 for bloom filter"); } - if (!std::isfinite(fpp) || fpp <= 0.0 || fpp >= 1.0) { - return Status::Invalid("fpp must be greater than 0 and less than 1 for bloom filter"); - } + PAIMON_RETURN_NOT_OK(ValidateFpp(fpp)); auto bytes = static_cast(ceil(BloomFilter::OptimalNumOfBits(expected_entries, fpp) / 8.0)); return std::make_shared(expected_entries, bytes); } +Result> BloomFilter::FixedBuilder( + int64_t expected_entries, double fpp, MemoryPool* pool) { + if (pool == nullptr) { + return Status::Invalid("Memory pool must not be null for bloom filter builder"); + } + PAIMON_RETURN_NOT_OK(ValidateFpp(fpp)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bloom_filter, + BloomFilter::Create(expected_entries, fpp)); + MemorySegment memory = MemorySegment::AllocateHeapMemory(bloom_filter->ByteLength(), pool); + PAIMON_RETURN_NOT_OK(bloom_filter->SetMemorySegment(std::move(memory))); + bloom_filter->Reset(); + return std::shared_ptr(new FixedBloomFilterBuilder(bloom_filter)); +} + +Result> BloomFilter::DynamicBuilder( + double fpp, const std::shared_ptr& pool) { + PAIMON_RETURN_NOT_OK(ValidateFpp(fpp)); + if (pool == nullptr) { + return Status::Invalid("Memory pool must not be null for bloom filter builder"); + } + return std::shared_ptr(new DynamicBloomFilterBuilder(fpp, pool)); +} + BloomFilter::BloomFilter(int64_t expected_entries, int32_t byte_length) : expected_entries_(expected_entries) { num_hash_functions_ = OptimalNumOfHashFunctions( diff --git a/src/paimon/common/utils/bloom_filter.h b/src/paimon/common/utils/bloom_filter.h index 61e27af59..0077b232f 100644 --- a/src/paimon/common/utils/bloom_filter.h +++ b/src/paimon/common/utils/bloom_filter.h @@ -27,13 +27,33 @@ namespace paimon { +class MemoryPool; + /// Bloom filter based on MemorySegment. class PAIMON_EXPORT BloomFilter { public: + class PAIMON_EXPORT Builder { + public: + virtual ~Builder() = default; + + virtual Status AddHash(int32_t hash) = 0; + + virtual Result> Build() = 0; + }; + static int32_t OptimalNumOfBits(int64_t expected_entries, double fpp); static int32_t OptimalNumOfHashFunctions(int64_t expected_entries, int64_t bit_size); static Result> Create(int64_t expected_entries, double fpp); + /// Creates a builder backed by a Bloom filter sized from the expected entry count. + static Result> FixedBuilder(int64_t expected_entries, double fpp, + MemoryPool* pool); + + /// Creates a builder which retains hashes and sizes the Bloom filter from the actual entry + /// count when Build is called. + static Result> DynamicBuilder( + double fpp, const std::shared_ptr& pool); + public: BloomFilter(int64_t expected_entries, int32_t byte_length); diff --git a/src/paimon/common/utils/bloom_filter_test.cpp b/src/paimon/common/utils/bloom_filter_test.cpp index 0832fdf5c..9d595e23a 100644 --- a/src/paimon/common/utils/bloom_filter_test.cpp +++ b/src/paimon/common/utils/bloom_filter_test.cpp @@ -32,7 +32,7 @@ namespace paimon::test { -TEST(BloomFilterTest, TestOneSegmentBuilder) { +TEST(BloomFilterTest, TestBloomFilterWithOneSegment) { int32_t items = 100; auto pool = GetDefaultPool(); ASSERT_OK_AND_ASSIGN(std::shared_ptr bloom_filter, @@ -83,6 +83,54 @@ TEST(BloomFilterTest, TestInvalidExpectedEntriesAndFpp) { "fpp must be greater than 0 and less than 1"); } +TEST(BloomFilterTest, TestFixedBuilder) { + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr builder, + BloomFilter::FixedBuilder(/*expected_entries=*/3, /*fpp=*/0.05, + pool.get())); + ASSERT_OK(builder->AddHash(11)); + ASSERT_OK(builder->AddHash(22)); + ASSERT_OK(builder->AddHash(33)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr bloom_filter, builder->Build()); + ASSERT_EQ(bloom_filter->ExpectedEntries(), 3); + ASSERT_TRUE(bloom_filter->TestHash(11)); + ASSERT_TRUE(bloom_filter->TestHash(22)); + ASSERT_TRUE(bloom_filter->TestHash(33)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr same_bloom_filter, builder->Build()); + ASSERT_EQ(same_bloom_filter, bloom_filter); +} + +TEST(BloomFilterTest, TestDynamicBuilder) { + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr builder, + BloomFilter::DynamicBuilder(/*fpp=*/0.05, pool)); + ASSERT_OK(builder->AddHash(11)); + ASSERT_OK(builder->AddHash(22)); + ASSERT_OK(builder->AddHash(33)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr bloom_filter, builder->Build()); + ASSERT_EQ(bloom_filter->ExpectedEntries(), 3); + ASSERT_TRUE(bloom_filter->TestHash(11)); + ASSERT_TRUE(bloom_filter->TestHash(22)); + ASSERT_TRUE(bloom_filter->TestHash(33)); + + // Match Java DynamicBloomFilterBuilder: additions after Build are ignored and Build is + // idempotent. + ASSERT_OK(builder->AddHash(44)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr same_bloom_filter, builder->Build()); + ASSERT_EQ(same_bloom_filter, bloom_filter); +} + +TEST(BloomFilterTest, TestEmptyDynamicBuilder) { + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr builder, + BloomFilter::DynamicBuilder(/*fpp=*/0.05, pool)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr bloom_filter, builder->Build()); + ASSERT_EQ(bloom_filter, nullptr); +} + TEST(BloomFilterTest, TestBloomNumBits) { ASSERT_EQ(0, BloomFilter::OptimalNumOfBits(0, 0)); ASSERT_EQ(0, BloomFilter::OptimalNumOfBits(0, 1)); diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index c38c50371..7e1cfff18 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -189,6 +189,17 @@ Result>> GlobalIndexScanImpl::Cre if (row_range_index && !row_range_index->Intersects(range.from, range.to)) { continue; } + std::optional> local_row_ranges; + if (row_range_index) { + std::vector intersected_ranges = + row_range_index->IntersectedRanges(range.from, range.to); + local_row_ranges.emplace(); + local_row_ranges->reserve(intersected_ranges.size()); + for (const Range& intersected_range : intersected_ranges) { + local_row_ranges->emplace_back(intersected_range.from - range.from, + intersected_range.to - range.from); + } + } auto arrow_field = DataField::ConvertDataFieldToArrowField(field); auto arrow_schema = arrow::schema({arrow_field}); @@ -198,7 +209,8 @@ Result>> GlobalIndexScanImpl::Cre ScopeGuard guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr index_reader, - indexer->CreateReader(&c_arrow_schema, index_file_manager_, index_io_metas, pool_)); + indexer->CreateReader(&c_arrow_schema, index_file_manager_, index_io_metas, + range.Count(), local_row_ranges, pool_)); union_readers.push_back( std::make_shared(std::move(index_reader), range.from)); } @@ -225,7 +237,7 @@ GlobalIndexIOMeta GlobalIndexScanImpl::ToGlobalIndexIOMeta( const std::shared_ptr& index_meta) const { assert(index_meta->GetGlobalIndexMeta()); const auto& global_index_meta = index_meta->GetGlobalIndexMeta().value(); - return {index_file_manager_->ToPath(index_meta), index_meta->FileSize(), + return {index_file_manager_->ToPath(index_meta), index_meta->FileSize(), index_meta->RowCount(), global_index_meta.index_meta}; } diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index 43132783f..d5bd13e99 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -314,7 +314,7 @@ Result> ToCommitMessage( external_path = path.ToString(); } index_file_metas.push_back(std::make_shared( - index_type, PathUtil::GetName(io_meta.file_path), io_meta.file_size, range.Count(), + index_type, PathUtil::GetName(io_meta.file_path), io_meta.file_size, io_meta.row_count, /*dv_ranges=*/std::nullopt, external_path, GlobalIndexMeta(range.from, range.to, field_id, extra_field_ids, io_meta.metadata))); } diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index 512f28728..cc1cb53a5 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -347,11 +347,12 @@ Status LookupLevels::CreateSstFileFromDataFile(const std::shared_ptrId(), serializer_factory_->Version())); // Prepare writer to write lookup file - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bloom_filter, - LookupStoreFactory::BfGenerator(file->row_count, options_, pool_.get())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr bloom_filter_builder, + LookupStoreFactory::BfGenerator(file->row_count, options_, pool_.get())); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr kv_writer, - lookup_store_factory_->CreateWriter(fs_, kv_file_path, bloom_filter, pool_)); + lookup_store_factory_->CreateWriter(fs_, kv_file_path, bloom_filter_builder, pool_)); ScopeGuard write_guard([&]() -> void { [[maybe_unused]] auto status = kv_writer->Close(); diff --git a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp index 5fbd5f093..7d3279afd 100644 --- a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp +++ b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp @@ -103,6 +103,25 @@ TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsExcludesRowIdsInFileRang ASSERT_EQ(indexed_split->Scores(), std::vector({0.5f, 0.9f})); } +TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsClipsNegativePredicateComplement) { + // A negative index predicate may produce a complement over the complete row-id domain. + // The final data scan must still intersect that result with each selected data file. + std::shared_ptr data_plan = NewDataPlan({NewAppendFile("file-1", 10l, 10l)}); + ASSERT_OK_AND_ASSIGN( + RowRangeIndex row_range_index, + RowRangeIndex::Create({Range(0, 14), Range(16, 30)})); // complement of row 15 + + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, + DataEvolutionBatchScan::WrapToIndexedSplits(data_plan, row_range_index, + /*id_to_score=*/{})); + + ASSERT_EQ(plan->Splits().size(), 1); + auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]); + ASSERT_NE(indexed_split, nullptr); + ASSERT_EQ(indexed_split->RowRanges(), + std::vector({Range(10, 14), Range(16, 19)})); +} + TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsWithoutIntersection) { std::shared_ptr data_plan = NewDataPlan({NewAppendFile("file-1", 100l, 10l)}); ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 50)})); diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp index 8d6466100..bd6cc4d28 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan.cpp @@ -554,14 +554,15 @@ PrimaryKeySortedIndexScan::ReaderFactory PrimaryKeySortedIndexScan::MakeReaderFa } std::vector io_metas; io_metas.emplace_back(path_factory->ToPath(payload), payload->FileSize(), - payload_meta.value().index_meta); + payload->RowCount(), payload_meta.value().index_meta); PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(definition.FieldId())); auto arrow_field = DataField::ConvertDataFieldToArrowField(field); auto arrow_schema = arrow::schema({arrow_field}); ArrowSchema c_arrow_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); ScopeGuard guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); - return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, payload->RowCount(), + /*row_ranges=*/std::nullopt, pool); }; } diff --git a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp index 99f2655ca..5c28048a8 100644 --- a/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp +++ b/src/paimon/core/table/source/primary_key_sorted_index_scan_test.cpp @@ -306,6 +306,7 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { const std::shared_ptr& payload = group.Payload(); std::vector io_metas; io_metas.emplace_back(base_path + "/" + payload->FileName(), payload->FileSize(), + payload->RowCount(), payload->GetGlobalIndexMeta().value().index_meta); PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(definition.FieldId())); auto arrow_field = DataField::ConvertDataFieldToArrowField(field); @@ -313,7 +314,8 @@ class PrimaryKeySortedIndexScanTest : public ::testing::Test { ArrowSchema c_arrow_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); auto file_reader = std::make_shared(fs); - return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, pool); + return indexer->CreateReader(&c_arrow_schema, file_reader, io_metas, + payload->RowCount(), /*row_ranges=*/std::nullopt, pool); }; } diff --git a/src/paimon/global_index/full_text/full_text_global_index.cpp b/src/paimon/global_index/full_text/full_text_global_index.cpp index 93c3dd804..e4e8b1973 100644 --- a/src/paimon/global_index/full_text/full_text_global_index.cpp +++ b/src/paimon/global_index/full_text/full_text_global_index.cpp @@ -73,7 +73,9 @@ Result> FullTextGlobalIndex::CreateWriter( Result> FullTextGlobalIndex::CreateReader( ::ArrowSchema* c_arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, const std::shared_ptr& pool) const { + const std::vector& files, int64_t /*total_row_count*/, + const std::optional>& /*row_ranges*/, + const std::shared_ptr& pool) const { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, arrow::ImportSchema(c_arrow_schema)); if (files.size() != 1) { diff --git a/src/paimon/global_index/full_text/full_text_global_index.h b/src/paimon/global_index/full_text/full_text_global_index.h index 2bc120159..08a280e87 100644 --- a/src/paimon/global_index/full_text/full_text_global_index.h +++ b/src/paimon/global_index/full_text/full_text_global_index.h @@ -46,7 +46,8 @@ class FullTextGlobalIndex : public GlobalIndexer { Result> CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, const std::shared_ptr& pool) const override; private: diff --git a/src/paimon/global_index/full_text/full_text_global_index_test.cpp b/src/paimon/global_index/full_text/full_text_global_index_test.cpp index 110b9d650..3af19be05 100644 --- a/src/paimon/global_index/full_text/full_text_global_index_test.cpp +++ b/src/paimon/global_index/full_text/full_text_global_index_test.cpp @@ -252,7 +252,12 @@ class FullTextGlobalIndexTest : public ::testing::Test { auto schema = arrow::schema({arrow::field("f0", field_type)}); auto c_schema = std::make_unique<::ArrowSchema>(); EXPECT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); - return global_index.CreateReader(c_schema.get(), CreateFileManager(), metas, pool_); + int64_t total_row_count = 0; + for (const GlobalIndexIOMeta& meta : metas) { + total_row_count += meta.row_count; + } + return global_index.CreateReader(c_schema.get(), CreateFileManager(), metas, + total_row_count, /*row_ranges=*/std::nullopt, pool_); } static std::shared_ptr MakeSearch( @@ -375,6 +380,7 @@ TEST_F(FullTextGlobalIndexTest, TestWriteAndSearch) { ASSERT_OK_AND_ASSIGN(std::vector metas, writer->Finish()); ASSERT_EQ(metas.size(), 1u); const GlobalIndexIOMeta& meta = metas[0]; + ASSERT_EQ(meta.row_count, array->length()); std::string file_name = PathUtil::GetName(meta.file_path); ASSERT_TRUE(StringUtils::StartsWith(file_name, "full-text-global-index-")) << file_name; @@ -415,6 +421,7 @@ TEST_F(FullTextGlobalIndexTest, TestWriterWithOnlyNullRows) { ASSERT_OK_AND_ASSIGN(std::vector metas, WriteIndex(/*options=*/{}, MakeArray(R"([[null], [null]])"))); ASSERT_EQ(metas.size(), 1u); + ASSERT_EQ(metas[0].row_count, 2); ASSERT_EQ(std::string(metas[0].metadata->data(), metas[0].metadata->size()), "{}"); ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, CreateReader(metas)); @@ -568,7 +575,7 @@ TEST_F(FullTextGlobalIndexTest, TestNestedBooleanAndBoost) { } TEST_F(FullTextGlobalIndexTest, TestInvalidRead) { - GlobalIndexIOMeta meta("full-text-global-index-1.index", 1, nullptr); + GlobalIndexIOMeta meta("full-text-global-index-1.index", 1, /*row_count=*/1, nullptr); ASSERT_NOK_WITH_MSG(CreateReader({}), "exactly one index file"); ASSERT_NOK_WITH_MSG(CreateReader({meta, meta}), "exactly one index file"); ASSERT_NOK_WITH_MSG(CreateReader({meta}, arrow::int32()), "only supports string fields"); @@ -603,7 +610,8 @@ TEST_F(FullTextGlobalIndexTest, TestInvalidRead) { TEST_F(FullTextGlobalIndexTest, TestInvalidIndexFile) { std::string file_path = PathUtil::JoinPath(index_dir_->Str(), "invalid.index"); ASSERT_OK(fs_->WriteFile(file_path, "this is not a full-text index", /*overwrite=*/false)); - std::vector metas = {GlobalIndexIOMeta(file_path, 29, nullptr)}; + std::vector metas = { + GlobalIndexIOMeta(file_path, 29, /*row_count=*/1, nullptr)}; ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, CreateReader(metas)); ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"))), "bad magic"); @@ -611,7 +619,8 @@ TEST_F(FullTextGlobalIndexTest, TestInvalidIndexFile) { ASSERT_OK(fs_->ReadFile(FixturePath("default"), &archive)); std::string truncated_path = PathUtil::JoinPath(index_dir_->Str(), "truncated.index"); ASSERT_OK(fs_->WriteFile(truncated_path, archive.substr(0, 32), /*overwrite=*/false)); - ASSERT_OK_AND_ASSIGN(reader, CreateReader({GlobalIndexIOMeta(truncated_path, 32, nullptr)})); + ASSERT_OK_AND_ASSIGN( + reader, CreateReader({GlobalIndexIOMeta(truncated_path, 32, /*row_count=*/5, nullptr)})); Status status = reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"))).status(); ASSERT_TRUE(status.IsIOError()) << status.ToString(); ASSERT_NOK_WITH_MSG(status, "failed to open full-text index " + truncated_path); @@ -625,7 +634,8 @@ TEST_F(FullTextGlobalIndexTest, TestInvalidIndexFile) { ASSERT_OK_AND_ASSIGN( reader, CreateReader({GlobalIndexIOMeta( - other_version_path, static_cast(other_version_archive.size()), nullptr)})); + other_version_path, static_cast(other_version_archive.size()), + /*row_count=*/5, nullptr)})); ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"))), "unsupported Tantivy index version tantivy v0.25.0"); } @@ -735,11 +745,13 @@ TEST_F(FullTextGlobalIndexTest, TestCrossReadFixtures) { for (const auto& fixture : GoldenFixtures()) { std::string fixture_path = FixturePath(fixture.name); ASSERT_OK_AND_ASSIGN(FileStatus file_status, fs_->GetFileStatus(fixture_path)); + std::shared_ptr fixture_array = MakeArray(fixture.rows); CheckGoldenSearches(fixture, GlobalIndexIOMeta(fixture_path, file_status.GetLen(), + fixture_array->length(), /*metadata=*/nullptr)); ASSERT_OK_AND_ASSIGN(std::vector metas, - WriteIndex(fixture.options, MakeArray(fixture.rows))); + WriteIndex(fixture.options, fixture_array)); ASSERT_EQ(metas.size(), 1u) << fixture.name; ASSERT_OK_AND_ASSIGN(std::unique_ptr written_header, ReadHeader(metas[0].file_path)); diff --git a/src/paimon/global_index/full_text/full_text_global_index_writer.cpp b/src/paimon/global_index/full_text/full_text_global_index_writer.cpp index 4cf6dad21..375fc6f8d 100644 --- a/src/paimon/global_index/full_text/full_text_global_index_writer.cpp +++ b/src/paimon/global_index/full_text/full_text_global_index_writer.cpp @@ -220,7 +220,7 @@ Result FullTextGlobalIndexWriter::WriteIndex( std::string options_json; PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); auto metadata = std::make_shared(options_json, pool_.get()); - return GlobalIndexIOMeta(file_writer_->ToPath(file_name), file_size, metadata); + return GlobalIndexIOMeta(file_writer_->ToPath(file_name), file_size, row_count_, metadata); } } // namespace paimon::full_text diff --git a/src/paimon/global_index/lucene/lucene_global_index.cpp b/src/paimon/global_index/lucene/lucene_global_index.cpp index 03715bccb..064441ef3 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index.cpp @@ -62,7 +62,9 @@ Result> LuceneGlobalIndex::CreateWriter( Result> LuceneGlobalIndex::CreateReader( ::ArrowSchema* c_arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, const std::shared_ptr& pool) const { + const std::vector& files, int64_t /*total_row_count*/, + const std::optional>& /*row_ranges*/, + const std::shared_ptr& pool) const { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, arrow::ImportSchema(c_arrow_schema)); if (files.size() != 1) { diff --git a/src/paimon/global_index/lucene/lucene_global_index.h b/src/paimon/global_index/lucene/lucene_global_index.h index 68857c795..254eec326 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.h +++ b/src/paimon/global_index/lucene/lucene_global_index.h @@ -43,7 +43,8 @@ class LuceneGlobalIndex : public GlobalIndexer { Result> CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, const std::shared_ptr& pool) const override; private: diff --git a/src/paimon/global_index/lucene/lucene_global_index_test.cpp b/src/paimon/global_index/lucene/lucene_global_index_test.cpp index 40885e2a6..2be7aa02d 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_test.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_test.cpp @@ -98,6 +98,7 @@ class LuceneGlobalIndexTest : public ::testing::Test, // check meta EXPECT_EQ(result_metas.size(), 1); + EXPECT_EQ(result_metas[0].row_count, array->length()); auto file_name = PathUtil::GetName(result_metas[0].file_path); EXPECT_TRUE(StringUtils::StartsWith(file_name, "lucene-fts-global-index-")); EXPECT_TRUE(StringUtils::EndsWith(file_name, ".index")); @@ -120,7 +121,7 @@ class LuceneGlobalIndexTest : public ::testing::Test, auto file_reader = std::make_shared( fs_, path_factory, /*checkpoint_path_factory=*/nullptr); return global_index->CreateReader(CreateArrowSchema(data_type).get(), file_reader, {meta}, - pool_); + meta.row_count, /*row_ranges=*/std::nullopt, pool_); } static std::shared_ptr MakeSearch( diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp index 14a665848..133ddf23d 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -239,6 +239,7 @@ Result> LuceneGlobalIndexWriter::Finish() { PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); auto meta_bytes = std::make_shared(options_json, pool_.get()); GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size, + /*row_count=*/row_id_, /*metadata=*/meta_bytes); return std::vector({meta}); } diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.h b/src/paimon/global_index/lucene/lucene_global_index_writer.h index 380618539..e11be99cd 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.h +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.h @@ -66,7 +66,7 @@ class LuceneGlobalIndexWriter : public GlobalIndexWriter { private: std::shared_ptr pool_; - int32_t row_id_ = 0; + int64_t row_id_ = 0; std::string field_name_; std::shared_ptr arrow_type_; LuceneWriteContext write_context_; diff --git a/src/paimon/indexer/lumina/lumina_global_index.cpp b/src/paimon/indexer/lumina/lumina_global_index.cpp index 5a7bf66c6..beac36d68 100644 --- a/src/paimon/indexer/lumina/lumina_global_index.cpp +++ b/src/paimon/indexer/lumina/lumina_global_index.cpp @@ -104,7 +104,9 @@ Result LuminaIndexReader::GetIndexInfo( Result> LuminaGlobalIndex::CreateReader( ::ArrowSchema* c_arrow_schema, const std::shared_ptr& file_manager, - const std::vector& files, const std::shared_ptr& pool) const { + const std::vector& files, int64_t /*total_row_count*/, + const std::optional>& /*row_ranges*/, + const std::shared_ptr& pool) const { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, arrow::ImportSchema(c_arrow_schema)); if (files.size() != 1) { @@ -236,6 +238,7 @@ Result> LuminaIndexWriter::Finish() { PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(lumina_options_, &options_json)); auto meta_bytes = std::make_shared(options_json, pool_->GetPaimonPool().get()); GlobalIndexIOMeta meta(file_manager_->ToPath(index_file_name), file_size, + /*row_count=*/count_, /*metadata=*/meta_bytes); if (checkpoint_file_manager_) { Status status = checkpoint_file_manager_->DeleteCheckpoint(); diff --git a/src/paimon/indexer/lumina/lumina_global_index.h b/src/paimon/indexer/lumina/lumina_global_index.h index 0a9186fef..20293d50d 100644 --- a/src/paimon/indexer/lumina/lumina_global_index.h +++ b/src/paimon/indexer/lumina/lumina_global_index.h @@ -86,7 +86,8 @@ class LuminaGlobalIndex : public GlobalIndexer { Result> CreateReader( ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, - const std::vector& files, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, const std::shared_ptr& pool) const override; private: diff --git a/src/paimon/indexer/lumina/lumina_global_index_test.cpp b/src/paimon/indexer/lumina/lumina_global_index_test.cpp index e734d48c3..563f5c607 100644 --- a/src/paimon/indexer/lumina/lumina_global_index_test.cpp +++ b/src/paimon/indexer/lumina/lumina_global_index_test.cpp @@ -188,6 +188,7 @@ class LuminaGlobalIndexTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(auto result_metas, global_writer->Finish()); // check meta EXPECT_EQ(result_metas.size(), 1); + EXPECT_EQ(result_metas[0].row_count, array->length()); auto file_name = PathUtil::GetName(result_metas[0].file_path); EXPECT_TRUE(StringUtils::StartsWith(file_name, "lumina-global-index-")); EXPECT_TRUE(StringUtils::EndsWith(file_name, ".index")); @@ -228,7 +229,7 @@ class LuminaGlobalIndexTest : public ::testing::Test { auto file_reader = std::make_shared( fs_, path_factory, /*checkpoint_path_factory=*/nullptr); return global_index->CreateReader(CreateArrowSchema(data_type).get(), file_reader, {meta}, - pool_); + meta.row_count, /*row_ranges=*/std::nullopt, pool_); } std::shared_ptr CreateRandomVector(int32_t element_size, @@ -1225,7 +1226,9 @@ TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { fs_, path_factory, /*checkpoint_path_factory=*/nullptr); ASSERT_NOK_WITH_MSG(global_index->CreateReader(CreateArrowSchema(data_type_).get(), - file_reader, {meta, meta}, pool_), + file_reader, {meta, meta}, + meta.row_count, + /*row_ranges=*/std::nullopt, pool_), "lumina index only has one index file per shard"); } { diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index e6c1abc59..7f7931ed4 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -3319,22 +3319,30 @@ TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndexWithPartition) { ASSERT_EQ(result2->ToString(), "{}"); } - // Scan with row_range_index filtering: only range [5,7] (partition f1=20) + // Scan with row_range_index filtering: only global row 6 in range [5,7]. This verifies + // that the global range is translated to the file-local range [1,1] before creating the + // BTree reader. { ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, /*partitions=*/std::nullopt, /*options=*/{}, fs_, /*executor=*/nullptr, pool_)); - ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(5, 7)})); + ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(6, 6)})); ASSERT_OK_AND_ASSIGN(auto index_readers, global_index_scan->CreateReaders("f0", row_range_index)); ASSERT_EQ(index_readers.size(), 1u); - // "Tony" in range [5,7]: local id {2} in range [5,7] -> global {7} + // "Lucy" is local row 1 in the index range [5,7] -> global row 6. ASSERT_OK_AND_ASSIGN(auto result, - index_readers[0]->VisitEqual(Literal(FieldType::STRING, "Tony", 4))); + index_readers[0]->VisitEqual(Literal(FieldType::STRING, "Lucy", 4))); ASSERT_TRUE(result); - ASSERT_EQ(result->ToString(), "{7}"); + ASSERT_EQ(result->ToString(), "{6}"); + + // "Tony" is in the selected index file but outside the requested row range. + ASSERT_OK_AND_ASSIGN( + result, index_readers[0]->VisitEqual(Literal(FieldType::STRING, "Tony", 4))); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{}"); } // Full pipeline with evaluator: Scan(predicate) -> read data diff --git a/test/inte/primary_key_sorted_index_inte_test.cpp b/test/inte/primary_key_sorted_index_inte_test.cpp index 1d63f6380..73060b9db 100644 --- a/test/inte/primary_key_sorted_index_inte_test.cpp +++ b/test/inte/primary_key_sorted_index_inte_test.cpp @@ -119,11 +119,11 @@ std::vector ExpectedResidualRowsAtSnapshot4() { return rows; } -std::vector ExpectedPartitionRowsAtSnapshot2() { +std::vector ExpectedPartitionScoreZeroRowsAtSnapshot2() { std::vector rows; for (int32_t pt = 1; pt <= 2; pt++) { - for (int32_t id = 1; id <= 100; id++) { - rows.emplace_back(pt, id, id % 10, id % 2 == 0 ? "keep" : "drop"); + for (int32_t id = 10; id <= 100; id += 10) { + rows.emplace_back(pt, id, 0, "keep"); } } return rows; @@ -174,6 +174,11 @@ class PrimaryKeySortedIndexInteTest : public ::testing::Test, Literal(score)); } + std::shared_ptr ScoreNotEqual(bool partitioned, int32_t score) const { + return PredicateBuilder::NotEqual(partitioned ? 2 : 1, "score", FieldType::INT, + Literal(score)); + } + std::shared_ptr ScoreGreaterOrEqual(bool partitioned, int32_t score) const { return PredicateBuilder::GreaterOrEqual(partitioned ? 2 : 1, "score", FieldType::INT, Literal(score)); @@ -373,6 +378,37 @@ TEST_P(PrimaryKeySortedIndexInteTest, MultiSourceOrdinalsBecomeBoundedFileLocalR ASSERT_EQ(ExpectedScoreZeroRows(/*snapshot_id=*/2), ranges_only_rows); } +TEST_P(PrimaryKeySortedIndexInteTest, NegativePredicateUsesCompleteIndexRowCount) { + std::shared_ptr predicate = ScoreNotEqual(/*partitioned=*/false, 0); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr indexed_plan, + Scan(/*partitioned=*/false, predicate, /*snapshot_id=*/2, /*index_enabled=*/true, + /*partition_filters=*/{}, /*branch=*/"", /*file_system=*/nullptr)); + const auto [indexed_count, data_count] = CountSplitKinds(indexed_plan); + ASSERT_GT(indexed_count, 0); + ASSERT_EQ(data_count, 0); + + ASSERT_OK_AND_ASSIGN( + std::vector indexed_rows, + Read(/*partitioned=*/false, predicate, indexed_plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); + ASSERT_EQ(indexed_rows.size(), 1800u); + for (const Row& row : indexed_rows) { + ASSERT_NE(std::get<2>(row), 0); + } + + // Compare with a non-indexed scan so the assertion also covers the exact complement domain. + ASSERT_OK_AND_ASSIGN( + std::shared_ptr data_plan, + Scan(/*partitioned=*/false, predicate, /*snapshot_id=*/2, /*index_enabled=*/false, + /*partition_filters=*/{}, /*branch=*/"", /*file_system=*/nullptr)); + ASSERT_OK_AND_ASSIGN( + std::vector data_rows, + Read(/*partitioned=*/false, predicate, data_plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); + ASSERT_EQ(indexed_rows, data_rows); +} + TEST_P(PrimaryKeySortedIndexInteTest, DeletionVectorResidualAndBooleanFallback) { std::shared_ptr score = ScoreEqual(/*partitioned=*/false, 0); std::shared_ptr id_upper_bound = IdLessOrEqual(/*partitioned=*/false, /*id=*/1500); @@ -417,7 +453,9 @@ TEST_P(PrimaryKeySortedIndexInteTest, DeletionVectorResidualAndBooleanFallback) } TEST_P(PrimaryKeySortedIndexInteTest, PartitionsBucketsAndIndexPaths) { - std::shared_ptr predicate = ScoreGreaterOrEqual(/*partitioned=*/true, /*score=*/0); + // Avoid the metadata-only full-match path so this test still verifies that the selected + // partition and bucket index files are opened through the configured file system. + std::shared_ptr predicate = ScoreEqual(/*partitioned=*/true, /*score=*/0); auto tracking_file_system = std::make_shared(); ASSERT_OK_AND_ASSIGN( std::shared_ptr plan, @@ -444,13 +482,13 @@ TEST_P(PrimaryKeySortedIndexInteTest, PartitionsBucketsAndIndexPaths) { } ASSERT_EQ((std::set{1, 2}), partitions); ASSERT_EQ((std::set{0, 1}), buckets); - ASSERT_EQ(200, selected_rows); + ASSERT_EQ(20, selected_rows); ASSERT_FALSE(tracking_file_system->OpenedIndexPaths().empty()); ASSERT_OK_AND_ASSIGN(std::vector rows, Read(/*partitioned=*/true, predicate, plan->Splits(), /*options=*/{}, tracking_file_system)); - ASSERT_EQ(ExpectedPartitionRowsAtSnapshot2(), rows); + ASSERT_EQ(ExpectedPartitionScoreZeroRowsAtSnapshot2(), rows); } TEST_P(PrimaryKeySortedIndexInteTest, FallbackTableReadRoutesMainAndFallbackSplits) { From 3c285d6cbd65d5cf37d05596bf99477ea1a1b115 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 14 Oct 2026 12:12:40 +0800 Subject: [PATCH 2/3] fix clang tidy --- .../lazy_filtered_bitmap_reader_test.cpp | 11 +- .../btree/btree_compatibility_test.cpp | 28 ++--- .../btree_global_index_integration_test.cpp | 114 +++++++++--------- .../btree/btree_global_index_writer.cpp | 4 +- .../btree/btree_global_indexer.cpp | 2 +- .../btree/lazy_filtered_btree_reader.cpp | 7 +- .../btree/lazy_filtered_btree_reader_test.cpp | 26 ++-- .../sorted_file_global_index_reader.cpp | 3 +- .../sorted_file_global_index_reader.h | 3 +- .../sorted_file_meta_selector_test.cpp | 3 +- src/paimon/common/sst/sst_file_io_test.cpp | 3 +- src/paimon/common/sst/sst_file_reader.cpp | 5 +- src/paimon/common/sst/sst_file_writer.cpp | 8 +- src/paimon/common/sst/sst_file_writer.h | 3 +- src/paimon/common/utils/bloom_filter.cpp | 5 +- src/paimon/common/utils/bloom_filter.h | 6 +- src/paimon/common/utils/bloom_filter_test.cpp | 6 +- src/paimon/core/mergetree/lookup_levels.cpp | 5 +- .../source/data_evolution_batch_scan_test.cpp | 3 +- .../full_text/full_text_global_index_test.cpp | 13 +- .../lumina/lumina_global_index_test.cpp | 10 +- test/inte/global_index_test.cpp | 4 +- .../primary_key_sorted_index_inte_test.cpp | 14 +-- 23 files changed, 132 insertions(+), 154 deletions(-) diff --git a/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader_test.cpp b/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader_test.cpp index eff9366f5..b72c72871 100644 --- a/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader_test.cpp +++ b/src/paimon/common/global_index/bitmap/lazy_filtered_bitmap_reader_test.cpp @@ -104,12 +104,11 @@ class LazyFilteredBitmapReaderTest : public ::testing::Test { for (const GlobalIndexIOMeta& meta : metas) { total_row_count += meta.row_count; } - EXPECT_OK_AND_ASSIGN( - std::shared_ptr reader, - LazyFilteredBitmapReader::Create(file_reader == nullptr ? file_manager_ : file_reader, - metas, arrow::utf8(), fallback_scan_max_size, - total_row_count, pool_, - /*executor=*/nullptr)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr reader, + LazyFilteredBitmapReader::Create( + file_reader == nullptr ? file_manager_ : file_reader, metas, + arrow::utf8(), fallback_scan_max_size, total_row_count, pool_, + /*executor=*/nullptr)); return reader; } diff --git a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp index fde4292ed..c0a6787c5 100644 --- a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp +++ b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp @@ -570,9 +570,8 @@ TEST_F(BTreeCompatibilityTest, ReadAndQueryIntData) { auto records = ParseCsvFile(csv_path); ASSERT_EQ(static_cast(records.size()), count); - ASSERT_OK_AND_ASSIGN(auto reader, - CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), - records.size())); + ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), + records.size())); RunIntQueries(reader, records); } } @@ -587,9 +586,8 @@ TEST_F(BTreeCompatibilityTest, ReadAndQueryStringData) { auto records = ParseCsvFile(csv_path); ASSERT_EQ(static_cast(records.size()), count); - ASSERT_OK_AND_ASSIGN(auto reader, - CreateReaderFromFiles(bin_path, meta_path, arrow::utf8(), - records.size())); + ASSERT_OK_AND_ASSIGN( + auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::utf8(), records.size())); RunStringQueries(reader, records); } } @@ -605,8 +603,7 @@ TEST_F(BTreeCompatibilityTest, ReadAndQueryFloatData) { ASSERT_EQ(static_cast(records.size()), count); ASSERT_OK_AND_ASSIGN( - auto reader, - CreateReaderFromFiles(bin_path, meta_path, arrow::float32(), records.size())); + auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::float32(), records.size())); RunFloatQueries(reader, records); } @@ -620,8 +617,8 @@ TEST_F(BTreeCompatibilityTest, AllNulls) { ASSERT_FALSE(records.empty()); auto count = static_cast(records.size()); - ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), - records.size())); + ASSERT_OK_AND_ASSIGN( + auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), records.size())); // All rows should be null { @@ -659,8 +656,8 @@ TEST_F(BTreeCompatibilityTest, NoNulls) { ASSERT_FALSE(records.empty()); auto count = static_cast(records.size()); - ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), - records.size())); + ASSERT_OK_AND_ASSIGN( + auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), records.size())); // No rows should be null { @@ -727,8 +724,8 @@ TEST_F(BTreeCompatibilityTest, DuplicateKeys) { auto records = ParseCsvFile(csv_path); ASSERT_FALSE(records.empty()); - ASSERT_OK_AND_ASSIGN(auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), - records.size())); + ASSERT_OK_AND_ASSIGN( + auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow::int32(), records.size())); // VisitIsNull { @@ -923,8 +920,7 @@ TEST_F(BTreeCompatibilityTest, RowCountConsistency) { auto count = static_cast(records.size()); ASSERT_OK_AND_ASSIGN( - auto reader, - CreateReaderFromFiles(bin_path, meta_path, arrow_type, records.size())); + auto reader, CreateReaderFromFiles(bin_path, meta_path, arrow_type, records.size())); ASSERT_OK_AND_ASSIGN(auto null_result, reader->VisitIsNull()); auto null_ids = CollectRowIds(null_result); diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index 1332666c6..20db545ba 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -325,10 +325,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadIntData) { // Verify that row ranges are converted to a row-id filter and applied to every query result. std::vector row_ranges = {Range(1, 8), Range(10, 10)}; c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN( - auto filtered_reader, - indexer->CreateReader(c_schema.get(), file_reader, metas, array->length(), row_ranges, - pool_)); + ASSERT_OK_AND_ASSIGN(auto filtered_reader, + indexer->CreateReader(c_schema.get(), file_reader, metas, array->length(), + row_ranges, pool_)); { ASSERT_OK_AND_ASSIGN(auto result, filtered_reader->VisitEqual(Literal(5))); @@ -385,9 +384,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadStringData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,1,3,4,6,7,8}, Null rows: {2,5} @@ -566,9 +565,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteEmptyStringKeyMetadata) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); Literal empty(FieldType::STRING, "", 0); ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(empty)); @@ -608,9 +607,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadBigIntData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -698,9 +697,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadFloatData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -787,9 +786,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadDoubleData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -875,9 +874,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadAllNonNull) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // All rows: {0,1,2,3,4,5}, No null rows @@ -984,9 +983,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadBoolData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,1,3,4,6,7}, Null rows: {2,5} @@ -1058,9 +1057,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadTinyIntData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -1138,9 +1137,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadSmallIntData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -1219,9 +1218,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadTimestampCompactData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -1301,9 +1300,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadTimestampNonCompactData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} // micros: 1000000 -> millis=1000, nanos_of_millis=0 @@ -1387,9 +1386,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadDecimalCompactData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -1478,9 +1477,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadDecimalNonCompactData) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,2,3,4,6}, Null rows: {1,5} @@ -1571,9 +1570,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadAllNull) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // All rows are null: {0,1,2,3}, No non-null rows @@ -1741,10 +1740,10 @@ TEST_F(BTreeGlobalIndexIntegrationTest, WriteAndReadLargeDataWithSmallBlocks) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, - read_indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, + read_indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // check input stream and read buffer size { @@ -1876,8 +1875,7 @@ TEST_F(BTreeGlobalIndexIntegrationTest, CreateWriterWithMissingField) { ASSERT_TRUE(struct_type); ASSERT_NOK_WITH_MSG( BTreeGlobalIndexWriter::Create("nonexistent_field", struct_type, file_writer, 4096, - compression_factory_, /*bloom_filter_enabled=*/false, - pool_), + compression_factory_, /*bloom_filter_enabled=*/false, pool_), "not in arrow_array when Create BTreeGlobalIndexWriter"); } @@ -2021,9 +2019,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, TestIOException) { // read auto file_reader = std::make_shared(local_fs, local_base); c_schema = CreateArrowSchema(field); - auto reader_result = indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_); + auto reader_result = + indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_); CHECK_HOOK_STATUS(reader_result.status(), i); auto reader = std::move(reader_result).value(); @@ -2232,9 +2230,9 @@ TEST_P(BTreeGlobalIndexIntegrationTest, TestRangeQuery) { auto file_reader = std::make_shared(fs_, base_path_); c_schema = CreateArrowSchema(field); - ASSERT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, - TotalRowCount(metas), - /*row_ranges=*/std::nullopt, pool_)); + ASSERT_OK_AND_ASSIGN( + auto reader, indexer->CreateReader(c_schema.get(), file_reader, metas, TotalRowCount(metas), + /*row_ranges=*/std::nullopt, pool_)); // Non-null rows: {0,1,2,3,4,6,7,8}, Null rows: {5} diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp index a7c9ddacc..614bd366e 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp @@ -65,8 +65,8 @@ Result> BTreeGlobalIndexWriter::Create( file_writer->NewFileName(BtreeDefs::kIdentifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr output_stream, file_writer->NewOutputStream(index_file_name)); - auto sst_file_writer = std::make_unique( - output_stream, bloom_filter_builder, block_size, compression_factory, pool); + auto sst_file_writer = std::make_unique(output_stream, bloom_filter_builder, + block_size, compression_factory, pool); return std::shared_ptr(new BTreeGlobalIndexWriter( field_name, arrow_type, key_serializer, file_writer, index_file_name, output_stream, std::move(sst_file_writer), pool)); diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index 847d7f7c2..db81a9c9c 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -67,7 +67,7 @@ Result> BTreeGlobalIndexer::Create( } Result>> BTreeGlobalIndexer::GetExtraFieldNames() const { - // TODO: Support multi-field composite BTree indexes consistently with Java. + // TODO(xinyu.lxy): Support multi-field composite BTree indexes consistently with Java. return std::optional>(std::nullopt); } diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp index 907faf213..136564c15 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader.cpp @@ -40,9 +40,10 @@ namespace { using FullRangeBounds = std::optional>; -Result GetFullRangeBounds( - const std::vector& files, int64_t total_row_count, - const std::shared_ptr& key_serializer, MemoryPool* pool) { +Result GetFullRangeBounds(const std::vector& files, + int64_t total_row_count, + const std::shared_ptr& key_serializer, + MemoryPool* pool) { if (total_row_count <= 0 || files.empty()) { return std::optional>(); } diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp index 80d3a9a58..cfc8404ed 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp @@ -147,12 +147,11 @@ class LazyFilteredBTreeReaderTest : public ::testing::Test { const std::shared_ptr& executor, int64_t fallback_scan_max_size) const { auto file_reader = std::make_shared(fs_, base_path_); auto cache_manager = std::make_shared(1024 * 1024, 0.5); - EXPECT_OK_AND_ASSIGN( - std::shared_ptr reader, - LazyFilteredBTreeReader::Create( - /*read_buffer_size=*/std::nullopt, all_metas_, arrow::int32(), file_reader, - cache_manager, fallback_scan_max_size, TotalRowCount(), - row_ranges, pool_, executor)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr reader, + LazyFilteredBTreeReader::Create( + /*read_buffer_size=*/std::nullopt, all_metas_, arrow::int32(), + file_reader, cache_manager, fallback_scan_max_size, + TotalRowCount(), row_ranges, pool_, executor)); return reader; } @@ -232,8 +231,7 @@ TEST_F(LazyFilteredBTreeReaderTest, TestFullMatch) { // full-match shortcut. std::shared_ptr reader = CreateReader(/*executor=*/nullptr, /*fallback_scan_max_size=*/0); - ASSERT_OK_AND_ASSIGN(std::shared_ptr equal, - reader->VisitEqual(Literal(5))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr equal, reader->VisitEqual(Literal(5))); CheckResult(equal, {0, 1, 2, 3, 4}); ASSERT_OK_AND_ASSIGN(std::shared_ptr less_than, reader->VisitLessThan(Literal(6))); @@ -535,13 +533,11 @@ TEST_F(LazyFilteredBTreeReaderTest, TestParallelEmptyFilesList) { std::vector empty_metas; auto file_reader = std::make_shared(fs_, base_path_); auto cache_manager = std::make_shared(1024 * 1024, 0.5); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr reader, - LazyFilteredBTreeReader::Create(/*read_buffer_size=*/std::nullopt, empty_metas, - arrow::int32(), file_reader, cache_manager, - std::numeric_limits::max(), - /*total_row_count=*/0, /*row_ranges=*/std::nullopt, pool_, - executor)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + LazyFilteredBTreeReader::Create( + /*read_buffer_size=*/std::nullopt, empty_metas, arrow::int32(), + file_reader, cache_manager, std::numeric_limits::max(), + /*total_row_count=*/0, /*row_ranges=*/std::nullopt, pool_, executor)); Literal literal_1(1); ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(literal_1)); CheckEmpty(result); diff --git a/src/paimon/common/global_index/sorted_file_global_index_reader.cpp b/src/paimon/common/global_index/sorted_file_global_index_reader.cpp index 6552132af..d647e7eb1 100644 --- a/src/paimon/common/global_index/sorted_file_global_index_reader.cpp +++ b/src/paimon/common/global_index/sorted_file_global_index_reader.cpp @@ -305,8 +305,7 @@ Result> SortedFileGlobalIndexReader::Compleme std::shared_ptr bitmap_result = std::dynamic_pointer_cast(excluded_result); if (bitmap_result == nullptr) { - return Status::Invalid( - "SortedFileGlobalIndexReader expects bitmap index results."); + return Status::Invalid("SortedFileGlobalIndexReader expects bitmap index results."); } PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* excluded, bitmap_result->GetBitmap()); result -= *excluded; diff --git a/src/paimon/common/global_index/sorted_file_global_index_reader.h b/src/paimon/common/global_index/sorted_file_global_index_reader.h index ce28987ff..2411d16e0 100644 --- a/src/paimon/common/global_index/sorted_file_global_index_reader.h +++ b/src/paimon/common/global_index/sorted_file_global_index_reader.h @@ -58,8 +58,7 @@ class SortedFileGlobalIndexReader : public GlobalIndexReader { protected: SortedFileGlobalIndexReader(std::unique_ptr file_selector, - int64_t fallback_scan_max_size, - int64_t total_row_count, + int64_t fallback_scan_max_size, int64_t total_row_count, std::shared_ptr executor); virtual Result> OpenReader( diff --git a/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp b/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp index 054535ac2..0d8cc8c56 100644 --- a/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp +++ b/src/paimon/common/global_index/sorted_file_meta_selector_test.cpp @@ -247,8 +247,7 @@ TEST_F(SortedFileMetaSelectorTest, TestVisitStartsWith) { GlobalIndexIOMeta("first", 1, /*row_count=*/1, first_meta->Serialize(pool.get())), GlobalIndexIOMeta("second", 1, /*row_count=*/1, second_meta->Serialize(pool.get())), GlobalIndexIOMeta("third", 1, /*row_count=*/1, third_meta->Serialize(pool.get())), - GlobalIndexIOMeta("only_nulls", 1, /*row_count=*/1, - only_nulls_meta->Serialize(pool.get())), + GlobalIndexIOMeta("only_nulls", 1, /*row_count=*/1, only_nulls_meta->Serialize(pool.get())), }; ASSERT_OK_AND_ASSIGN(std::unique_ptr selector, SortedFileMetaSelector::Create(files, key_serializer)); diff --git a/src/paimon/common/sst/sst_file_io_test.cpp b/src/paimon/common/sst/sst_file_io_test.cpp index 180be3ae3..29e25065c 100644 --- a/src/paimon/common/sst/sst_file_io_test.cpp +++ b/src/paimon/common/sst/sst_file_io_test.cpp @@ -95,8 +95,7 @@ TEST_P(SstFileIOTest, TestSimple) { // write data ASSERT_OK_AND_ASSIGN(std::shared_ptr bloom_filter_builder, BloomFilter::FixedBuilder(30, 0.01, pool_.get())); - auto writer = - std::make_shared(out, bloom_filter_builder, 50, factory, pool_); + auto writer = std::make_shared(out, bloom_filter_builder, 50, factory, pool_); std::set value_hash; // k1-k5 for (size_t i = 1; i <= 5; i++) { diff --git a/src/paimon/common/sst/sst_file_reader.cpp b/src/paimon/common/sst/sst_file_reader.cpp index dea87f393..029ca27ce 100644 --- a/src/paimon/common/sst/sst_file_reader.cpp +++ b/src/paimon/common/sst/sst_file_reader.cpp @@ -29,8 +29,9 @@ Result> SstFileReader::Create( const std::optional& bloom_filter_handle, MemorySlice::SliceComparator comparator, const std::shared_ptr& block_cache, const std::shared_ptr& pool) { - // TODO: Align with Java FileBasedBloomFilter by probing the cache first and loading the Bloom - // filter on demand during point lookups instead of reading it eagerly when creating the reader. + // TODO(xinyu.lxy): Align with Java FileBasedBloomFilter by probing the cache first and loading + // the Bloom filter on demand during point lookups instead of reading it eagerly when creating + // the reader. std::shared_ptr bloom_filter = nullptr; if (bloom_filter_handle.has_value() && (bloom_filter_handle->ExpectedEntries() || bloom_filter_handle->Size() || diff --git a/src/paimon/common/sst/sst_file_writer.cpp b/src/paimon/common/sst/sst_file_writer.cpp index 214db6666..030ebd24d 100644 --- a/src/paimon/common/sst/sst_file_writer.cpp +++ b/src/paimon/common/sst/sst_file_writer.cpp @@ -28,10 +28,7 @@ SstFileWriter::SstFileWriter(const std::shared_ptr& out, int32_t block_size, const std::shared_ptr& factory, const std::shared_ptr& pool) - : pool_(pool), - out_(out), - bloom_filter_builder_(bloom_filter_builder), - block_size_(block_size) { + : pool_(pool), out_(out), bloom_filter_builder_(bloom_filter_builder), block_size_(block_size) { // block_size * 1.1 exceeds INT32_MAX for block_size above ~1.9GB; saturate instead of // relying on the undefined double->int32_t conversion. data_block_writer_ = @@ -49,8 +46,7 @@ Status SstFileWriter::Write(std::shared_ptr&& key, std::shared_ptr PAIMON_RETURN_NOT_OK(Flush()); } if (bloom_filter_builder_) { - PAIMON_RETURN_NOT_OK( - bloom_filter_builder_->AddHash(MurmurHashUtils::HashBytes(key))); + PAIMON_RETURN_NOT_OK(bloom_filter_builder_->AddHash(MurmurHashUtils::HashBytes(key))); } return Status::OK(); } diff --git a/src/paimon/common/sst/sst_file_writer.h b/src/paimon/common/sst/sst_file_writer.h index d84a4b51c..8a5cd1d54 100644 --- a/src/paimon/common/sst/sst_file_writer.h +++ b/src/paimon/common/sst/sst_file_writer.h @@ -45,8 +45,7 @@ class PAIMON_EXPORT SstFileWriter { public: SstFileWriter(const std::shared_ptr& out, const std::shared_ptr& bloom_filter_builder, - int32_t block_size, - const std::shared_ptr& factory, + int32_t block_size, const std::shared_ptr& factory, const std::shared_ptr& pool); ~SstFileWriter() = default; diff --git a/src/paimon/common/utils/bloom_filter.cpp b/src/paimon/common/utils/bloom_filter.cpp index 121b5bd4f..57975bd4f 100644 --- a/src/paimon/common/utils/bloom_filter.cpp +++ b/src/paimon/common/utils/bloom_filter.cpp @@ -130,8 +130,9 @@ Result> BloomFilter::Create(int64_t expected_entrie return std::make_shared(expected_entries, bytes); } -Result> BloomFilter::FixedBuilder( - int64_t expected_entries, double fpp, MemoryPool* pool) { +Result> BloomFilter::FixedBuilder(int64_t expected_entries, + double fpp, + MemoryPool* pool) { if (pool == nullptr) { return Status::Invalid("Memory pool must not be null for bloom filter builder"); } diff --git a/src/paimon/common/utils/bloom_filter.h b/src/paimon/common/utils/bloom_filter.h index 0077b232f..993a3c4d2 100644 --- a/src/paimon/common/utils/bloom_filter.h +++ b/src/paimon/common/utils/bloom_filter.h @@ -47,12 +47,12 @@ class PAIMON_EXPORT BloomFilter { /// Creates a builder backed by a Bloom filter sized from the expected entry count. static Result> FixedBuilder(int64_t expected_entries, double fpp, - MemoryPool* pool); + MemoryPool* pool); /// Creates a builder which retains hashes and sizes the Bloom filter from the actual entry /// count when Build is called. - static Result> DynamicBuilder( - double fpp, const std::shared_ptr& pool); + static Result> DynamicBuilder(double fpp, + const std::shared_ptr& pool); public: BloomFilter(int64_t expected_entries, int32_t byte_length); diff --git a/src/paimon/common/utils/bloom_filter_test.cpp b/src/paimon/common/utils/bloom_filter_test.cpp index 9d595e23a..0a48168a6 100644 --- a/src/paimon/common/utils/bloom_filter_test.cpp +++ b/src/paimon/common/utils/bloom_filter_test.cpp @@ -85,9 +85,9 @@ TEST(BloomFilterTest, TestInvalidExpectedEntriesAndFpp) { TEST(BloomFilterTest, TestFixedBuilder) { auto pool = GetDefaultPool(); - ASSERT_OK_AND_ASSIGN(std::shared_ptr builder, - BloomFilter::FixedBuilder(/*expected_entries=*/3, /*fpp=*/0.05, - pool.get())); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr builder, + BloomFilter::FixedBuilder(/*expected_entries=*/3, /*fpp=*/0.05, pool.get())); ASSERT_OK(builder->AddHash(11)); ASSERT_OK(builder->AddHash(22)); ASSERT_OK(builder->AddHash(33)); diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index cc1cb53a5..7d7aecedf 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -347,9 +347,8 @@ Status LookupLevels::CreateSstFileFromDataFile(const std::shared_ptrId(), serializer_factory_->Version())); // Prepare writer to write lookup file - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr bloom_filter_builder, - LookupStoreFactory::BfGenerator(file->row_count, options_, pool_.get())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr bloom_filter_builder, + LookupStoreFactory::BfGenerator(file->row_count, options_, pool_.get())); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr kv_writer, lookup_store_factory_->CreateWriter(fs_, kv_file_path, bloom_filter_builder, pool_)); diff --git a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp index 7d3279afd..c153e8b17 100644 --- a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp +++ b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp @@ -118,8 +118,7 @@ TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsClipsNegativePredicateCo ASSERT_EQ(plan->Splits().size(), 1); auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]); ASSERT_NE(indexed_split, nullptr); - ASSERT_EQ(indexed_split->RowRanges(), - std::vector({Range(10, 14), Range(16, 19)})); + ASSERT_EQ(indexed_split->RowRanges(), std::vector({Range(10, 14), Range(16, 19)})); } TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsWithoutIntersection) { diff --git a/src/paimon/global_index/full_text/full_text_global_index_test.cpp b/src/paimon/global_index/full_text/full_text_global_index_test.cpp index 3af19be05..15da34ef1 100644 --- a/src/paimon/global_index/full_text/full_text_global_index_test.cpp +++ b/src/paimon/global_index/full_text/full_text_global_index_test.cpp @@ -632,10 +632,9 @@ TEST_F(FullTextGlobalIndexTest, TestInvalidIndexFile) { std::string other_version_path = PathUtil::JoinPath(index_dir_->Str(), "other_version.index"); ASSERT_OK(fs_->WriteFile(other_version_path, other_version_archive, /*overwrite=*/false)); ASSERT_OK_AND_ASSIGN( - reader, - CreateReader({GlobalIndexIOMeta( - other_version_path, static_cast(other_version_archive.size()), - /*row_count=*/5, nullptr)})); + reader, CreateReader({GlobalIndexIOMeta(other_version_path, + static_cast(other_version_archive.size()), + /*row_count=*/5, nullptr)})); ASSERT_NOK_WITH_MSG(reader->VisitFullTextSearch(MakeSearch(MatchQuery("paimon"))), "unsupported Tantivy index version tantivy v0.25.0"); } @@ -746,9 +745,9 @@ TEST_F(FullTextGlobalIndexTest, TestCrossReadFixtures) { std::string fixture_path = FixturePath(fixture.name); ASSERT_OK_AND_ASSIGN(FileStatus file_status, fs_->GetFileStatus(fixture_path)); std::shared_ptr fixture_array = MakeArray(fixture.rows); - CheckGoldenSearches(fixture, GlobalIndexIOMeta(fixture_path, file_status.GetLen(), - fixture_array->length(), - /*metadata=*/nullptr)); + CheckGoldenSearches( + fixture, GlobalIndexIOMeta(fixture_path, file_status.GetLen(), fixture_array->length(), + /*metadata=*/nullptr)); ASSERT_OK_AND_ASSIGN(std::vector metas, WriteIndex(fixture.options, fixture_array)); diff --git a/src/paimon/indexer/lumina/lumina_global_index_test.cpp b/src/paimon/indexer/lumina/lumina_global_index_test.cpp index 563f5c607..e3451c44b 100644 --- a/src/paimon/indexer/lumina/lumina_global_index_test.cpp +++ b/src/paimon/indexer/lumina/lumina_global_index_test.cpp @@ -1225,11 +1225,11 @@ TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { auto file_reader = std::make_shared( fs_, path_factory, /*checkpoint_path_factory=*/nullptr); - ASSERT_NOK_WITH_MSG(global_index->CreateReader(CreateArrowSchema(data_type_).get(), - file_reader, {meta, meta}, - meta.row_count, - /*row_ranges=*/std::nullopt, pool_), - "lumina index only has one index file per shard"); + ASSERT_NOK_WITH_MSG( + global_index->CreateReader(CreateArrowSchema(data_type_).get(), file_reader, + {meta, meta}, meta.row_count, + /*row_ranges=*/std::nullopt, pool_), + "lumina index only has one index file per shard"); } { auto data_type = arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 7f7931ed4..89ea06bac 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -3339,8 +3339,8 @@ TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndexWithPartition) { ASSERT_EQ(result->ToString(), "{6}"); // "Tony" is in the selected index file but outside the requested row range. - ASSERT_OK_AND_ASSIGN( - result, index_readers[0]->VisitEqual(Literal(FieldType::STRING, "Tony", 4))); + ASSERT_OK_AND_ASSIGN(result, + index_readers[0]->VisitEqual(Literal(FieldType::STRING, "Tony", 4))); ASSERT_TRUE(result); ASSERT_EQ(result->ToString(), "{}"); } diff --git a/test/inte/primary_key_sorted_index_inte_test.cpp b/test/inte/primary_key_sorted_index_inte_test.cpp index 73060b9db..f2811a699 100644 --- a/test/inte/primary_key_sorted_index_inte_test.cpp +++ b/test/inte/primary_key_sorted_index_inte_test.cpp @@ -388,10 +388,9 @@ TEST_P(PrimaryKeySortedIndexInteTest, NegativePredicateUsesCompleteIndexRowCount ASSERT_GT(indexed_count, 0); ASSERT_EQ(data_count, 0); - ASSERT_OK_AND_ASSIGN( - std::vector indexed_rows, - Read(/*partitioned=*/false, predicate, indexed_plan->Splits(), /*options=*/{}, - /*file_system=*/nullptr)); + ASSERT_OK_AND_ASSIGN(std::vector indexed_rows, Read(/*partitioned=*/false, predicate, + indexed_plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); ASSERT_EQ(indexed_rows.size(), 1800u); for (const Row& row : indexed_rows) { ASSERT_NE(std::get<2>(row), 0); @@ -402,10 +401,9 @@ TEST_P(PrimaryKeySortedIndexInteTest, NegativePredicateUsesCompleteIndexRowCount std::shared_ptr data_plan, Scan(/*partitioned=*/false, predicate, /*snapshot_id=*/2, /*index_enabled=*/false, /*partition_filters=*/{}, /*branch=*/"", /*file_system=*/nullptr)); - ASSERT_OK_AND_ASSIGN( - std::vector data_rows, - Read(/*partitioned=*/false, predicate, data_plan->Splits(), /*options=*/{}, - /*file_system=*/nullptr)); + ASSERT_OK_AND_ASSIGN(std::vector data_rows, + Read(/*partitioned=*/false, predicate, data_plan->Splits(), /*options=*/{}, + /*file_system=*/nullptr)); ASSERT_EQ(indexed_rows, data_rows); } From 3d0b379d336bc4c87f5eb69f9942a88b8ef74162 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 14 Oct 2026 14:44:16 +0800 Subject: [PATCH 3/3] fix rt concurrent test --- src/paimon/fs/local/local_file.cpp | 4 ++++ src/paimon/fs/local/local_file_test.cpp | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/src/paimon/fs/local/local_file.cpp b/src/paimon/fs/local/local_file.cpp index a3f960af8..b2f5c15c3 100644 --- a/src/paimon/fs/local/local_file.cpp +++ b/src/paimon/fs/local/local_file.cpp @@ -164,6 +164,10 @@ Result LocalFile::GetFileStatus() const { struct stat buf; if (stat(path_.c_str(), &buf) < 0) { int32_t cur_errno = errno; + if (cur_errno == ENOENT) { + return Status::NotExist(fmt::format("get file '{}' status failed, ec: {}", path_, + std::strerror(cur_errno))); + } return Status::IOError( fmt::format("get file '{}' status failed, ec: {}", path_, std::strerror(cur_errno))); } diff --git a/src/paimon/fs/local/local_file_test.cpp b/src/paimon/fs/local/local_file_test.cpp index 25b9f6db9..f172780b2 100644 --- a/src/paimon/fs/local/local_file_test.cpp +++ b/src/paimon/fs/local/local_file_test.cpp @@ -32,6 +32,16 @@ TEST(LocalFileTest, TestSchemeCaseInsensitive) { ASSERT_NOK(LocalFile::Create("s3:/tmp")); } +TEST(LocalFileTest, TestGetFileStatusForMissingFile) { + auto test_root_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr file, + LocalFile::Create(test_root_dir->Str() + "/missing-file")); + + Result status = file->GetFileStatus(); + ASSERT_TRUE(status.status().IsNotExist()) << status.status().ToString(); +} + TEST(LocalFileTest, TestReadWriteEmptyContent) { auto test_root_dir = UniqueTestDirectory::Create(); ASSERT_TRUE(test_root_dir);