Skip to content

test(parquet): allocate raw Arrow reader buffers from the default pool #386

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lxy-9602 merged 1 commit into from
Sep 24, 2026
Merged

test(parquet): allocate raw Arrow reader buffers from the default pool #386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2056,11 +2056,12 @@ TEST_F(ParquetFileBatchReaderTest, TestDictionaryPassthroughSkipsFallbackToPlain
// its presence cannot be the signal; the data pages are what say the column went plain. Both
// are asserted so the test fails loudly if the fixture stops producing a mixed chunk rather
// than quietly passing for the wrong reason.
auto metadata_file = arrow::io::ReadableFile::Open(file_path, pool_.get());
auto metadata_file = arrow::io::ReadableFile::Open(file_path, arrow::default_memory_pool());
ASSERT_TRUE(metadata_file.ok());
std::unique_ptr<::parquet::arrow::FileReader> metadata_reader;
ASSERT_TRUE(
::parquet::arrow::OpenFile(metadata_file.ValueOrDie(), pool_.get(), &metadata_reader).ok());
ASSERT_TRUE(::parquet::arrow::OpenFile(metadata_file.ValueOrDie(), arrow::default_memory_pool(),
&metadata_reader)
.ok());
std::unique_ptr<::parquet::ColumnChunkMetaData> column_chunk =
metadata_reader->parquet_reader()->metadata()->RowGroup(0)->ColumnChunk(0);
ASSERT_TRUE(column_chunk->has_dictionary_page());
Expand Down Expand Up @@ -2151,11 +2152,12 @@ TEST_F(ParquetFileBatchReaderTest, TestDictionaryPassthroughRequiresEveryRowGrou

// Pin the fixture: without this the read assertion below would also pass on a file whose
// first row group was never dictionary-encoded in the first place.
auto metadata_file = arrow::io::ReadableFile::Open(file_path, pool_.get());
auto metadata_file = arrow::io::ReadableFile::Open(file_path, arrow::default_memory_pool());
ASSERT_TRUE(metadata_file.ok());
std::unique_ptr<::parquet::arrow::FileReader> metadata_reader;
ASSERT_TRUE(
::parquet::arrow::OpenFile(metadata_file.ValueOrDie(), pool_.get(), &metadata_reader).ok());
ASSERT_TRUE(::parquet::arrow::OpenFile(metadata_file.ValueOrDie(), arrow::default_memory_pool(),
&metadata_reader)
.ok());
std::shared_ptr<::parquet::FileMetaData> metadata =
metadata_reader->parquet_reader()->metadata();
ASSERT_EQ(2, metadata->num_row_groups());
Expand Down
44 changes: 26 additions & 18 deletions src/paimon/format/parquet/parquet_format_writer_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,10 +142,13 @@ class ParquetFormatWriterTest : public ::testing::Test {

void CheckResult(const std::string& file_path, int32_t row_count,
int32_t row_group_count) const {
auto file = arrow::io::ReadableFile::Open(file_path, arrow_pool_.get());
ASSERT_TRUE(file.ok());
// Use the process-wide pool: pre-buffered reads may release their buffers on Arrow IO
// threads after this fixture's pool is destroyed.
auto file = arrow::io::ReadableFile::Open(file_path, arrow::default_memory_pool());
ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &reader);
auto status =
::parquet::arrow::OpenFile(file.ValueOrDie(), arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata = reader->parquet_reader()->metadata().get();
const ::parquet::SchemaDescriptor* schema = metadata->schema();
Expand Down Expand Up @@ -266,10 +269,11 @@ class ParquetFormatWriterTest : public ::testing::Test {

void CheckEncodedResult(const std::string& file_path, int32_t row_count,
bool null_in_dictionary) const {
auto file = arrow::io::ReadableFile::Open(file_path, arrow_pool_.get());
ASSERT_TRUE(file.ok());
auto file = arrow::io::ReadableFile::Open(file_path, arrow::default_memory_pool());
ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &reader);
auto status =
::parquet::arrow::OpenFile(file.ValueOrDie(), arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata = reader->parquet_reader()->metadata().get();
ASSERT_EQ(metadata->num_rows(), row_count);
Expand Down Expand Up @@ -636,10 +640,11 @@ TEST_F(ParquetFormatWriterTest, TestWriteDictionaryChangingAcrossBatches) {
ASSERT_OK(out->Flush());
ASSERT_OK(out->Close());

auto file = arrow::io::ReadableFile::Open(file_path, arrow_pool_.get());
ASSERT_TRUE(file.ok());
auto file = arrow::io::ReadableFile::Open(file_path, arrow::default_memory_pool());
ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &reader);
auto status =
::parquet::arrow::OpenFile(file.ValueOrDie(), arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata = reader->parquet_reader()->metadata().get();
ASSERT_EQ(3 * kBatchRows, metadata->num_rows());
Expand Down Expand Up @@ -708,10 +713,11 @@ TEST_F(ParquetFormatWriterTest, TestWriteDictionaryWithNullRows) {
ASSERT_OK(out->Flush());
ASSERT_OK(out->Close());

auto file = arrow::io::ReadableFile::Open(file_path, arrow_pool_.get());
ASSERT_TRUE(file.ok());
auto file = arrow::io::ReadableFile::Open(file_path, arrow::default_memory_pool());
ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &reader);
auto status =
::parquet::arrow::OpenFile(file.ValueOrDie(), arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata = reader->parquet_reader()->metadata().get();
ASSERT_EQ(6, metadata->num_rows());
Expand Down Expand Up @@ -766,10 +772,11 @@ TEST_F(ParquetFormatWriterTest, TestWriteDictionaryWithDuplicateValues) {
ASSERT_OK(out->Flush());
ASSERT_OK(out->Close());

auto file = arrow::io::ReadableFile::Open(file_path, arrow_pool_.get());
ASSERT_TRUE(file.ok());
auto file = arrow::io::ReadableFile::Open(file_path, arrow::default_memory_pool());
ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &reader);
auto status =
::parquet::arrow::OpenFile(file.ValueOrDie(), arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata = reader->parquet_reader()->metadata().get();
ASSERT_EQ(4, metadata->num_rows());
Expand Down Expand Up @@ -889,10 +896,11 @@ TEST_F(ParquetFormatWriterTest, TestWriteDictionaryOfBinaryColumn) {
ASSERT_OK(out->Flush());
ASSERT_OK(out->Close());

auto file = arrow::io::ReadableFile::Open(file_path, arrow_pool_.get());
ASSERT_TRUE(file.ok());
auto file = arrow::io::ReadableFile::Open(file_path, arrow::default_memory_pool());
ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &reader);
auto status =
::parquet::arrow::OpenFile(file.ValueOrDie(), arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::FileMetaData* metadata = reader->parquet_reader()->metadata().get();
ASSERT_EQ(4, metadata->num_rows());
Expand Down
25 changes: 14 additions & 11 deletions src/paimon/format/parquet/variant_parquet_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -545,10 +545,11 @@ TEST_F(VariantParquetTest, PhysicalLayoutMatchesJava) {
// reader is required because these parquet-level properties (repetition, physical types,
// field ids, the absence of a logical-type annotation) are not visible in the Arrow schema
// surfaced by the paimon reader.
auto file = arrow::io::ReadableFile::Open(file_path_, arrow_pool_.get());
ASSERT_TRUE(file.ok());
auto file = arrow::io::ReadableFile::Open(file_path_, arrow::default_memory_pool());
ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &reader);
auto status =
::parquet::arrow::OpenFile(file.ValueOrDie(), arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const ::parquet::SchemaDescriptor* schema = reader->parquet_reader()->metadata()->schema();
ASSERT_EQ(schema->num_columns(), 3);
Expand Down Expand Up @@ -591,12 +592,14 @@ TEST_F(VariantParquetTest, WriteAndReadRoundTrip) {

{
// Sanity-check the raw file through the plain parquet-arrow reader: the struct child
// arrays must align with the logical rows.
auto file = arrow::io::ReadableFile::Open(file_path_, arrow_pool_.get());
ASSERT_TRUE(file.ok());
// arrays must align with the logical rows. Use the process-wide pool: pre-buffered reads
// may release their buffers on Arrow IO threads after this fixture's pool is destroyed.
auto file = arrow::io::ReadableFile::Open(file_path_, arrow::default_memory_pool());
ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> raw_reader;
ASSERT_TRUE(
::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &raw_reader).ok());
::parquet::arrow::OpenFile(file.ValueOrDie(), arrow::default_memory_pool(), &raw_reader)
.ok());
std::shared_ptr<arrow::Table> table;
ASSERT_TRUE(raw_reader->ReadTable(&table).ok());
auto raw_variant = checked_pointer_cast<arrow::StructArray>(table->column(1)->chunk(0));
Expand Down Expand Up @@ -712,11 +715,11 @@ TEST_F(VariantParquetTest, ShreddedWriteAndReadRoundTrip) {
}

{
auto file = arrow::io::ReadableFile::Open(file_path_, arrow_pool_.get());
ASSERT_TRUE(file.ok());
auto file = arrow::io::ReadableFile::Open(file_path_, arrow::default_memory_pool());
ASSERT_TRUE(file.ok()) << file.status().ToString();
std::unique_ptr<::parquet::arrow::FileReader> reader;
auto status =
::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &reader);
auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(),
arrow::default_memory_pool(), &reader);
ASSERT_TRUE(status.ok()) << status.ToString();
const auto* root = reader->parquet_reader()->metadata()->schema()->group_node();
ASSERT_EQ(root->field_count(), 2);
Expand Down
Loading