diff --git a/paimon-python/README.md b/paimon-python/README.md index 9907c43bc722..4bcdd29995ad 100644 --- a/paimon-python/README.md +++ b/paimon-python/README.md @@ -210,7 +210,7 @@ view or retained snapshot metadata. Current main-branch REST table sources with `query-auth.enabled=true` also use core when the catalog returns no row filter or column mask. Core requests authorization for every source column, even for a key-only DELETE, and refreshes it for each MERGE. A restricted or -denied source fails before target actions, as with Python's `plan_all_files`. +denied source fails before target actions, as with Python's `plan_all`. Python and native writers use Java's escaped partition directories for data, changelog, Blob and row sidecar files, including external locations. Both @@ -561,7 +561,7 @@ live REST authorization and snapshot-file, deletion-vector and merge-group validation. Transported metadata never supplies a live grant. If restrictions changed incompatibly, the captured plan retains Python execution. Continuous streaming delegates planning and resume state to Rust core. -`TableScan.plan_all_files()` includes L0 and every data-evolution column file, +`TableScan.plan_all()` returns a split plan including L0 and every data-evolution column file, retaining scan filters and limits and requiring unrestricted query authorization. Writer restore, overwrite and index construction use this entrypoint and delegate planning to Rust core when Native planning is enabled. diff --git a/paimon-python/conftest.py b/paimon-python/conftest.py index 394fcda7e345..72ca7d474e4d 100644 --- a/paimon-python/conftest.py +++ b/paimon-python/conftest.py @@ -107,7 +107,7 @@ def tracked_plan(self, *args, **kwargs): TableScan._try_native_plan = tracked_plan - from pypaimon.read.streaming_table_scan import StreamTableScan + from pypaimon.execution.native.read.streaming_table_scan import StreamTableScan original_stream_plan = StreamTableScan.plan diff --git a/paimon-python/pypaimon/execution/README.md b/paimon-python/pypaimon/execution/README.md index 1ab8e23ed1e9..2743b89010d4 100644 --- a/paimon-python/pypaimon/execution/README.md +++ b/paimon-python/pypaimon/execution/README.md @@ -98,8 +98,18 @@ Neither backend imports the other. Native adapters do not construct or retain Python executors. Python implementations do not load the optional Native SDK just because it is installed. Planning, reading, writing and committing retain independent options; shared plans and commit messages connect those stages. -Explicit access to an existing Python scanner/router/committer property selects -that implementation. Ordinary configuration and metadata inspection do not. +The public `TableScan.file_scanner`, `AsyncStreamingTableScan` and +`StreamTableScan` are backend-neutral facades, including direct construction. +Accessing file configuration does not construct a Python scanner. File plans +require unrestricted whole-table authorization because they expose raw file +metadata. Internal writer metadata restoration uses the selected executor's +private scanner and retains its write authorization boundary. + +Manifest-source and follow-up-scanner callbacks belong to Python execution +internals. Public scanner configuration selects the existing snapshot, delta +or changelog strategies without accepting custom scanner callbacks. Explicit +Python router/committer properties retain their implementation-specific APIs. +Ordinary configuration and metadata inspection do not select that backend. Deletion-vector codecs, file loading and row filtering belong to Python execution, corresponding to Rust Core's deletion-vector implementation. @@ -200,7 +210,7 @@ Message conversion happens during preparation; publication uses the selected executor's prepared messages. Batch/stream binding details stay in the Native adapter, and overwrite details stay in each backend. -Both batch scanners expose `plan(auth_result=None)`, `plan_all_files()` and +Both batch scanners expose `plan(auth_result=None)`, `plan_all()` and `scan_with_stats(auth_result=None)`. Both readers expose `read_rows(splits)` and `read_batches(splits, schema, parallelism=None, blob_parallelism=1, ordered=False)`. Output materialization and Python parallel-read helpers are diff --git a/paimon-python/pypaimon/execution/native/read/file_scan.py b/paimon-python/pypaimon/execution/native/read/file_scan.py new file mode 100644 index 000000000000..e17d9b872495 --- /dev/null +++ b/paimon-python/pypaimon/execution/native/read/file_scan.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lossless metadata adaptation for Core file scans.""" + +from pypaimon.manifest.schema.manifest_entry import ManifestEntry +from pypaimon.table.row.generic_row import GenericRow +from pypaimon.utils.file_store_path_factory import canonical_data_file_path +from pypaimon.write.commit_message_serializer import deserialize_commit_message + + +def from_native_file_plan(table, plan): + entries = [] + for entry in plan.files(): + # Reuse the existing v14 file-metadata bridge, including historical + # stats, external paths, BLOB descriptors and row-tracking metadata. + message = deserialize_commit_message( + entry._to_commit_message().serialize(), table.partition_keys_fields, + table.trimmed_primary_keys_fields) + file = message.new_files[0] + file.file_path = (file.physical_path() if file.external_path else canonical_data_file_path( + table, message.partition, entry.bucket, file.file_name)) + entries.append(ManifestEntry( + entry.kind, GenericRow(list(message.partition), table.partition_keys_fields), + entry.bucket, entry.total_buckets, file)) + return entries, plan.snapshot_id() diff --git a/paimon-python/pypaimon/execution/native/read/plan.py b/paimon-python/pypaimon/execution/native/read/plan.py index 489c02928d0b..8c3e0887b382 100644 --- a/paimon-python/pypaimon/execution/native/read/plan.py +++ b/paimon-python/pypaimon/execution/native/read/plan.py @@ -22,14 +22,13 @@ from typing import List, Optional, Tuple from pypaimon.common.identifier import UNKNOWN_DATABASE -from pypaimon.common.options.core_options import CoreOptions, ChangelogProducer +from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.predicate import Predicate from pypaimon.execution.native import runtime from pypaimon.execution.native import table as native_table from pypaimon.execution.native.read import builder as native_builder from pypaimon.read.plan import Plan from pypaimon.read.query_auth_split import QueryAuthSplit -from pypaimon.read.scan_stats import ScanStats from pypaimon.read.scan_request import ScanRequest from pypaimon.read import split_serializer @@ -56,6 +55,7 @@ def native_plan( shard: Optional[Tuple[int, int]] = None, partition_predicate: Optional[Predicate] = None, global_index_result=None, + file_filters: Optional[Tuple[bool, bool]] = None, all_files: bool = False) -> Plan: """Plan with pypaimon_rust, preserving snapshot metadata. @@ -84,6 +84,8 @@ def native_plan( scan = builder.new_incremental_scan(*incremental_range) else: scan = builder.new_incremental_scan(*incremental_range, incremental_mode) + if file_filters is not None: + scan = scan.with_file_filters(*file_filters) if global_index_result is not None and not table.options.query_auth_enabled: # A PK result already selected a snapshot and physical files. Java's # PrimaryKeyBatchScan returns those selections before ordinary planning. @@ -120,7 +122,7 @@ def native_plan( scan = scan.with_chunk_shuffle(str(seed), chunk_size) if shard is not None: scan = scan.with_shard(*shard) - rust_plan = scan.plan_all_files() if all_files else scan.plan() + rust_plan = scan.plan_all() if all_files else scan.plan() return _from_native_plan(table, rust_plan) @@ -168,9 +170,28 @@ def plan(self, auth_result=None): # Core requests and captures query authorization with its own plan. return self._plan() - def plan_all_files(self): + def plan_all(self): return self._plan(all_files=True) + def plan_files(self, skip_level0, only_read_real_buckets): + """Plan complete file metadata before query result transformations.""" + try: + request = self._scan._scan_request() + builder = native_builder._configure_native_read_builder( + native_builder._native_read_builder(self._scan.table), + request.predicate, None, None) + if request.partition_filter is not None: + builder = builder.with_partition_filter(native_builder._predicate_to_native( + request.partition_key_predicate)) + plan = builder.new_scan().with_file_filters( + skip_level0, only_read_real_buckets).plan_files() + from pypaimon.execution.native.read.file_scan import from_native_file_plan + return from_native_file_plan(self._scan.table, plan) + except Exception as error: + runtime._raise_if_native_fork_safety_error(error) + logger.warning("Native file planning failed, falling back to Python: %s", error) + return None + def scan_with_stats(self, auth_result=None): # Preserve explicit facade hooks while Core records counters as part # of the ordinary planning pass. @@ -272,12 +293,8 @@ def _plan(self, all_files=False) -> Optional[Plan]: has_distribution = (request.idx_of_this_subtask is not None or request.start_pos_of_this_subtask is not None) extra_options = {} - if scan.table.options.options.contains(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP): - if request.incremental_snapshot_range is None: - return Plan([], _scan_stats=ScanStats.empty_native(request.read_limit)) - extra_options['incremental_range'] = request.incremental_snapshot_range - if scan.table.options.changelog_producer() != ChangelogProducer.NONE: - extra_options['incremental_mode'] = 'changelog' + # Core resolves timestamp options after authorization, including + # empty windows. Never preselect snapshots with Python here. chunk_shuffle = request.chunk_shuffle if chunk_shuffle is not None: extra_options['chunk_shuffle'] = chunk_shuffle @@ -296,6 +313,8 @@ def _plan(self, all_files=False) -> Optional[Plan]: row_ranges = list(request.row_ranges) if request.row_ranges is not None else None if request.global_index_result is not None: extra_options['global_index_result'] = request.global_index_result + if request.file_filters is not None: + extra_options['file_filters'] = request.file_filters if request.partition_filter is not None: # Explicit partition selectors are scan-only. Keep the ordinary # row predicate separate so core can apply Java's override and diff --git a/paimon-python/pypaimon/execution/native/read/streaming_table_scan.py b/paimon-python/pypaimon/execution/native/read/streaming_table_scan.py index d3c4f342c109..4107acb2765a 100644 --- a/paimon-python/pypaimon/execution/native/read/streaming_table_scan.py +++ b/paimon-python/pypaimon/execution/native/read/streaming_table_scan.py @@ -141,6 +141,9 @@ def _check_generation(self, generation): if generation != self._generation: raise RuntimeError("stream scan was restored during planning") + def cancel_prefetch(self): + """Native plans frames on demand, with no speculative frame to cancel.""" + def close(self): """Stop polling without acknowledging a plan still owned by the caller.""" self._closed = True diff --git a/paimon-python/pypaimon/execution/native/write/bucket_metadata.py b/paimon-python/pypaimon/execution/native/write/bucket_metadata.py index 1dec9016e519..152b3f3bb83f 100644 --- a/paimon-python/pypaimon/execution/native/write/bucket_metadata.py +++ b/paimon-python/pypaimon/execution/native/write/bucket_metadata.py @@ -24,7 +24,7 @@ def dynamic_bucket_data_buckets(table, partition, snapshot): from pypaimon.index.dynamic_bucket import bucket_read_builder read_builder = bucket_read_builder(table, partition, snapshot) - # plan_all_files includes level zero and requires unrestricted query + # plan_all includes level zero and requires unrestricted query # authorization. A policy which cannot supply complete metadata must fail # preparation before input assignment, never omit buckets from validation. plan = native_plan(read_builder.table, predicate=read_builder._predicate, all_files=True) diff --git a/paimon-python/pypaimon/execution/native/write/native_merge_into.py b/paimon-python/pypaimon/execution/native/write/native_merge_into.py index 7dbbcb1b56fb..41eca96baf11 100644 --- a/paimon-python/pypaimon/execution/native/write/native_merge_into.py +++ b/paimon-python/pypaimon/execution/native/write/native_merge_into.py @@ -81,7 +81,7 @@ def create_native_merge_into(table, source, on, matched, not_matched, commit_use if not _rest_catalog_supported(source) or not _native_query_auth_supported(source): return None # A source is a read, not a write. Core obtains a fresh REST grant - # and requires it to be unrestricted, as Python plan_all_files does. + # and requires it to be unrestricted, as Python plan_all does. native_source = _native_table(source) else: native_source = create_native_write_table(source) diff --git a/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py b/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py index 04aa7dc2c939..40dacc4f6861 100644 --- a/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py +++ b/paimon-python/pypaimon/execution/python/globalindex/create_global_index.py @@ -78,7 +78,7 @@ def build(self, partition_filter, *, execution="local", concurrency=None, ray_re read_builder = read_builder.with_partition_filter(partition_filter) scan = read_builder.new_scan() - plan = scan.plan_all_files() + plan = scan.plan_all() splits = plan.splits() if not splits: return [] diff --git a/paimon-python/pypaimon/execution/python/read/auth_support.py b/paimon-python/pypaimon/execution/python/read/auth_support.py index f414da2cf689..378cbfbdc08f 100644 --- a/paimon-python/pypaimon/execution/python/read/auth_support.py +++ b/paimon-python/pypaimon/execution/python/read/auth_support.py @@ -29,7 +29,10 @@ def prune_scanner_by_auth(table, scanner, auth_result): predicate = scanner._user_predicate effective = exclude_predicate_with_fields(predicate, masked) if effective != scanner.predicate: + partition_key_predicate = scanner.partition_key_predicate scanner._set_predicate(effective) + if scanner._explicit_partition_filter: + scanner.partition_key_predicate = partition_key_predicate scanner.auth_partition_predicate = None scanner.auth_has_non_partition_filter = bool( predicate is not None and masked & predicate_field_names(predicate)) diff --git a/paimon-python/pypaimon/execution/python/read/merge_engine_support.py b/paimon-python/pypaimon/execution/python/read/merge_engine_support.py index a47dcbda3129..39c6a412456a 100644 --- a/paimon-python/pypaimon/execution/python/read/merge_engine_support.py +++ b/paimon-python/pypaimon/execution/python/read/merge_engine_support.py @@ -81,6 +81,14 @@ _DEFAULT_AGGREGATE_FUNCTION_KEY = "fields.default-aggregate-function" +def validate_read(read) -> None: + """Check Python merge capabilities once, after this backend is selected.""" + if getattr(read, '_python_merge_validated', False): + return + check_supported(read.table) + read._python_merge_validated = True + + def check_supported(table) -> None: """Raise ``NotImplementedError`` if the table's merge-engine configuration is outside what pypaimon's read path implements, or @@ -206,6 +214,6 @@ def _option_is_truthy(raw) -> bool: return bool(raw) __all__ = [ - "check_supported", "check_sequence_field_valid", "check_sequence_field_supported", + "validate_read", "check_supported", "check_sequence_field_valid", "check_sequence_field_supported", "partial_update_unsupported_options", "aggregation_unsupported_options", ] diff --git a/paimon-python/pypaimon/execution/python/read/streaming_table_scan.py b/paimon-python/pypaimon/execution/python/read/streaming_table_scan.py index 09e5877a700d..f462dc0c889a 100644 --- a/paimon-python/pypaimon/execution/python/read/streaming_table_scan.py +++ b/paimon-python/pypaimon/execution/python/read/streaming_table_scan.py @@ -294,6 +294,10 @@ def _discard_prefetch(self): if future is not None: future.cancel() + def cancel_prefetch(self): + """Discard speculative work when a public polling iterator stops.""" + self._discard_prefetch() + def close(self): """Discard speculative work without acknowledging pending rows. diff --git a/paimon-python/pypaimon/execution/python/read/table_read.py b/paimon-python/pypaimon/execution/python/read/table_read.py index 4c5a56b8bd4a..7d120a85c287 100644 --- a/paimon-python/pypaimon/execution/python/read/table_read.py +++ b/paimon-python/pypaimon/execution/python/read/table_read.py @@ -27,6 +27,7 @@ from pypaimon.common.predicate_json_parser import ( _collect_all_field_refs_from_transform, extract_referenced_fields) from pypaimon.data.map_shared_shredding import map_selected_keys_field +from pypaimon.execution.python.read.merge_engine_support import validate_read from pypaimon.read.push_down_utils import ( exclude_predicate_with_fields, predicate_field_names, rewrite_predicate_indices) from pypaimon.read.query_auth_split import QueryAuthSplit @@ -134,14 +135,14 @@ def _emit_overflow_safe_batches( def read_rows(self, splits): read = self._read - read._check_python_merge_supported() + validate_read(read) limit = read.limit count = 0 for split in splits: if limit is not None and count >= limit: return remaining = None if limit is None else limit - count - reader = read._TableRead__create_reader_for_split(split, limit=remaining) + reader = self._create_reader_for_split(split, limit=remaining) failure = None try: if read.output_projection is not None or not read._adapter_read_type: @@ -179,7 +180,7 @@ def _arrow_batch_generator(self, splits: List[Split], schema: pyarrow.Schema, for split in splits: if remaining is not None and remaining <= 0: break - reader = read._TableRead__create_reader_for_split( + reader = self._create_reader_for_split( split, blob_parallelism, limit=remaining) failure = None try: @@ -189,7 +190,7 @@ def _arrow_batch_generator(self, splits: List[Split], schema: pyarrow.Schema, continue if remaining is not None and batch.num_rows > remaining: batch = batch.slice(0, remaining) - batch = read._serialize_blob_views_as_descriptors( + batch = self._serialize_blob_views_as_descriptors( batch, reader) batch = read._project_batch_to_output(batch) if read.include_row_kind: @@ -212,7 +213,7 @@ def _arrow_batch_generator(self, splits: List[Split], schema: pyarrow.Schema, if not isinstance(row, OffsetRow): raise TypeError(f"Expected OffsetRow, but got {type(row).__name__}") row_tuple_chunk.append( - read._serialize_blob_view_row_tuple(row, reader)) + self._serialize_blob_view_row_tuple(row, reader)) if read.include_row_kind: row_kind_chunk.append(row.get_row_kind().to_string()) @@ -223,7 +224,7 @@ def _arrow_batch_generator(self, splits: List[Split], schema: pyarrow.Schema, break if len(row_tuple_chunk) >= chunk_size: - yield from read._convert_rows_to_output_batches( + yield from self._convert_rows_to_output_batches( row_tuple_chunk, row_kind_chunk, schema ) row_tuple_chunk = [] @@ -232,7 +233,7 @@ def _arrow_batch_generator(self, splits: List[Split], schema: pyarrow.Schema, break if row_tuple_chunk: - yield from read._convert_rows_to_output_batches( + yield from self._convert_rows_to_output_batches( row_tuple_chunk, row_kind_chunk, schema ) except BaseException as error: @@ -268,7 +269,7 @@ def _to_arrow_parallel( ) as executor: futures = { executor.submit( - read._read_one_split_to_batches, + self._read_one_split_to_batches, split, schema, remaining_state, @@ -312,7 +313,7 @@ def _read_one_split_to_batches( read = self._read chunk_size = 65536 out: List[pyarrow.RecordBatch] = [] - reader = read._TableRead__create_reader_for_split(split, blob_parallelism) + reader = self._create_reader_for_split(split, blob_parallelism) failure = None try: if isinstance(reader, RecordBatchReader): @@ -324,7 +325,7 @@ def _read_one_split_to_batches( break if allowed < batch.num_rows: batch = batch.slice(0, allowed) - batch = read._serialize_blob_views_as_descriptors( + batch = self._serialize_blob_views_as_descriptors( batch, reader) batch = read._project_batch_to_output(batch) if read.include_row_kind: @@ -349,17 +350,17 @@ def _read_one_split_to_batches( stop = True break row_tuple_chunk.append( - read._serialize_blob_view_row_tuple(row, reader)) + self._serialize_blob_view_row_tuple(row, reader)) if read.include_row_kind: row_kind_chunk.append(row.get_row_kind().to_string()) if len(row_tuple_chunk) >= chunk_size: - out.extend(read._convert_rows_to_output_batches( + out.extend(self._convert_rows_to_output_batches( row_tuple_chunk, row_kind_chunk, schema)) row_tuple_chunk = [] row_kind_chunk = [] if row_tuple_chunk: - out.extend(read._convert_rows_to_output_batches( + out.extend(self._convert_rows_to_output_batches( row_tuple_chunk, row_kind_chunk, schema)) except BaseException as error: failure = error @@ -376,13 +377,13 @@ def _convert_rows_to_output_batches( ) -> Iterator[pyarrow.RecordBatch]: read = self._read if read.output_projection is None: - yield from read._convert_rows_to_arrow_batches_with_row_kind( + yield from self._convert_rows_to_arrow_batches_with_row_kind( row_tuples, row_kinds, schema) return physical_schema = read_arrow_schema(read._adapter_read_type) if read.include_row_kind: physical_schema = read._add_row_kind_to_schema(physical_schema) - for batch in read._convert_rows_to_arrow_batches_with_row_kind( + for batch in self._convert_rows_to_arrow_batches_with_row_kind( row_tuples, row_kinds, physical_schema): yield read._try_to_pad_batch_by_schema( read._project_batch_to_output(batch), schema, allow_type_cast=True) @@ -424,7 +425,7 @@ def _create_split_read(self, split: Split, blob_parallelism: int = 1, predicate=_DEFAULT_PREDICATE, defer_blob_resolution: bool = False) -> SplitRead: read = self._read - sr = read._build_split_read( + sr = self._build_split_read( split, read_type, limit, @@ -469,7 +470,7 @@ def _build_split_read(self, split: Split, read_type=None, # Inner: full ROW for the merge function. Outer: extract # the requested sub-paths back to the user's flat schema. inner_read_type = read._with_predicate_extra_fields( - read._widen_to_top_level_for_merge(nested_name_paths)) + self._widen_to_top_level_for_merge(nested_name_paths)) outer_extract_name_paths = nested_name_paths # When the user's projection drops a ``sequence.field``, the merge @@ -513,12 +514,12 @@ def _build_split_read(self, split: Split, read_type=None, elif read.table.options.data_evolution_enabled(): if nested_name_paths and any( len(p) > 1 for p in nested_name_paths): - if not read._only_map_key_nested_paths(): + if not self._only_map_key_nested_paths(): raise NotImplementedError( "ROW nested-field projection on data-evolution tables " "is not yet supported") scan_read_type = read._with_predicate_extra_fields( - read._widen_to_top_level_for_merge(nested_name_paths)) + self._widen_to_top_level_for_merge(nested_name_paths)) outer_extract_name_paths = nested_name_paths else: outer_extract_name_paths = None @@ -555,7 +556,7 @@ def _build_split_read(self, split: Split, read_type=None, # names/types), then extract the requested sub-paths back to # the user's flat schema. inner_read_type = read._with_predicate_extra_fields( - read._widen_to_top_level_for_merge(nested_name_paths)) + self._widen_to_top_level_for_merge(nested_name_paths)) outer_extract_name_paths = nested_name_paths if read_type is None and outer_extract_name_paths is None and read._needs_output_projection(): outer_extract_name_paths = read._output_extract_name_paths() @@ -631,7 +632,7 @@ def _widen_to_top_level_for_merge(self, name_paths=None) -> List[DataField]: if (isinstance(field.type, MapType) and all(len(path) > 1 for path in paths) and not is_map_blob_type(field.type) - and not read._map_has_aggregator(top_name)): + and not self._map_has_aggregator(top_name)): keys = [] for path in paths: if path[1] not in keys: @@ -675,33 +676,33 @@ def _map_has_aggregator(self, field_name: str) -> bool: or "fields.default-aggregate-function" in options ) - def _TableRead__create_reader_for_split( + def _create_reader_for_split( self, split, blob_parallelism=1, limit: Optional[int] = None): # Raw-convertible splits skip MergeFileSplitRead, so validate before # dispatch, including when native setup failed or bindings are absent. read = self._read - read._check_python_merge_supported() + validate_read(read) auth_result = None if isinstance(split, QueryAuthSplit): auth_result = split.auth_result split = split.split if auth_result is not None: - return read._TableRead__authed_reader( + return self._authed_reader( split, auth_result, blob_parallelism, limit) if limit is None: - return read._create_split_read( + return self._create_split_read( split, blob_parallelism=blob_parallelism).create_reader() - return read._create_split_read( + return self._create_split_read( split, blob_parallelism=blob_parallelism, limit=limit, ).create_reader() - def _TableRead__authed_reader( + def _authed_reader( self, split, auth_result, blob_parallelism=1, limit: Optional[int] = None): read = self._read - if read._has_map_key_projection(): + if self._has_map_key_projection(): raise NotImplementedError( "MAP-key projection with query authorization is not supported") table_fields = read.table.fields @@ -727,7 +728,7 @@ def _TableRead__authed_reader( filter_fn = auth_result.extract_row_filter() effective_limit = read.limit if limit is None else limit auth_fields = ( - read._auth_filter_field_names(auth_result, effective_read_type) + self._auth_filter_field_names(auth_result, effective_read_type) if filter_fn is not None else set() ) eager_blob_fields = set(auth_fields) @@ -751,7 +752,7 @@ def _TableRead__authed_reader( # If the auth filter references an inline BLOB, run it after inline resolution (in # the split read) so it sees resolved payloads while scalar BLOBs still defer. post_filter_after_inline = embed_filter and bool(auth_fields & inline_blob_fields) - split_read = read._create_split_read( + split_read = self._create_split_read( split, blob_parallelism=blob_parallelism, read_type=effective_read_type, @@ -826,12 +827,12 @@ def _auth_filter_field_names(auth_result, read_fields) -> set: def _materialize(self, splits, schema, parallelism=None, blob_parallelism=1): read = self._read - read._check_python_merge_supported() + validate_read(read) effective = read._effective_parallelism(parallelism, len(splits)) if read._should_run_parallel(splits, effective): - return read._to_arrow_parallel(splits, schema, effective, blob_parallelism) + return self._to_arrow_parallel(splits, schema, effective, blob_parallelism) - batch_iterator = read._arrow_batch_generator( + batch_iterator = self._arrow_batch_generator( splits, schema, blob_parallelism) batch_reader = pyarrow.ipc.RecordBatchReader.from_batches( schema, batch_iterator) @@ -855,10 +856,10 @@ def read_batches( ordered: bool = False): """Read lazily in split order; Python fallback readers remain serial.""" read = self._read - read._check_python_merge_supported() + validate_read(read) effective = read._effective_parallelism(parallelism, len(splits)) if read._should_run_parallel(splits, effective): blob_parallelism = read._cap_blob_parallelism( min(effective, len(splits)), blob_parallelism) - return read._arrow_batch_generator( + return self._arrow_batch_generator( splits, schema, blob_parallelism) diff --git a/paimon-python/pypaimon/execution/python/read/table_scan.py b/paimon-python/pypaimon/execution/python/read/table_scan.py index dbbb74f90a0d..8ca93d8c013a 100644 --- a/paimon-python/pypaimon/execution/python/read/table_scan.py +++ b/paimon-python/pypaimon/execution/python/read/table_scan.py @@ -21,7 +21,40 @@ from pypaimon.manifest.manifest_list_manager import ManifestListManager from pypaimon.execution.python.read.auth_support import prune_scanner_by_auth from pypaimon.read.query_auth_split import wrap_plan_with_auth +from pypaimon.read.scan_config import validate_scan_options from pypaimon.execution.python.read.scanner.file_scanner import FileScanner +from pypaimon.utils.range import Range + + +def _resolve_incremental_snapshot_range(table): + """Resolve Python scan boundaries only after selecting this backend.""" + window = validate_scan_options(table) + if window is None: + return None + start_timestamp, end_timestamp = window + snapshot_manager = table.snapshot_manager() + latest_snapshot = snapshot_manager.get_latest_snapshot() + earliest_snapshot = snapshot_manager.try_get_earliest_snapshot() if latest_snapshot is not None else None + if earliest_snapshot is None or latest_snapshot is None: + return None + if start_timestamp > end_timestamp: + raise ValueError( + "Ending timestamp %s should be >= starting timestamp %s." + % (end_timestamp, start_timestamp)) + if (start_timestamp == end_timestamp or start_timestamp > latest_snapshot.time_millis + or end_timestamp < earliest_snapshot.time_millis): + return None + + starting_snapshot = snapshot_manager.earlier_or_equal_time_mills(start_timestamp) + earliest_snapshot = snapshot_manager.try_get_earliest_snapshot() + if starting_snapshot is None or (earliest_snapshot and earliest_snapshot.time_millis > start_timestamp): + start_id = earliest_snapshot.id - 1 if earliest_snapshot else -1 + else: + start_id = starting_snapshot.id + end_snapshot = snapshot_manager.earlier_or_equal_time_mills(end_timestamp) + latest_snapshot = snapshot_manager.get_latest_snapshot() + end_id = end_snapshot.id if end_snapshot else (latest_snapshot.id if latest_snapshot else -1) + return start_id, end_id class PythonTableScan: @@ -41,13 +74,14 @@ def _create_file_scanner(self) -> FileScanner: has_incremental = options.contains(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP) if has_incremental: - if scan._incremental_snapshot_range is None: + snapshot_range = _resolve_incremental_snapshot_range(scan.table) + if snapshot_range is None: return FileScanner( scan.table, lambda: ([], None), partition_predicate=scan.partition_predicate, ) - start_id, end_id = scan._incremental_snapshot_range + start_id, end_id = snapshot_range # Java's AUTO mode uses physical changelogs when a producer is # configured, and APPEND deltas otherwise. changelog = scan.table.options.changelog_producer() != ChangelogProducer.NONE @@ -111,9 +145,32 @@ def all_manifests(): skip_level0=scan.table.options.batch_scan_skip_level0(), ) + def _file_scanner(self): + """Return the private Python scanner without entering public dispatch.""" + scan = self._scan + scanner = getattr(scan, '_python_file_scanner', None) + if scanner is None: + scanner = self._create_file_scanner() + request = scan._scan_request() + if request.idx_of_this_subtask is not None: + scanner.with_shard(request.idx_of_this_subtask, request.number_of_para_subtasks) + if request.start_pos_of_this_subtask is not None: + scanner.with_slice(request.start_pos_of_this_subtask, request.end_pos_of_this_subtask) + if request.global_index_result is not None: + scanner.with_global_index_result(request.global_index_result) + if request.row_ranges is not None: + scanner.with_row_ranges([Range(start, end) for start, end in request.row_ranges]) + if request.chunk_shuffle is not None: + scanner.with_chunk_shuffle(*request.chunk_shuffle) + public_scanner = getattr(scan, '_file_scanner', None) + if getattr(type(public_scanner), '_is_dispatch_file_scanner', False) is True: + public_scanner._configure_python_scanner(scanner) + scan._python_file_scanner = scanner + return scanner + def _scanner(self, auth_result): scan = self._scan - scanner = scan.file_scanner + scanner = self._file_scanner() if scan._query_auth_fn is not None: prune_scanner_by_auth(scan.table, scanner, auth_result) return scanner @@ -121,7 +178,7 @@ def _scanner(self, auth_result): def plan(self, auth_result=None): return wrap_plan_with_auth(auth_result, self._scanner(auth_result).scan()) - def plan_all_files(self): + def plan_all(self): scanner = self._scanner(None) # Writer restore/overwrite must see L0 even when batch reads hide it. skip_level0 = scanner.skip_level0 diff --git a/paimon-python/pypaimon/execution/python/write/bucket_metadata.py b/paimon-python/pypaimon/execution/python/write/bucket_metadata.py index 10fd80393044..8794bf5f605c 100644 --- a/paimon-python/pypaimon/execution/python/write/bucket_metadata.py +++ b/paimon-python/pypaimon/execution/python/write/bucket_metadata.py @@ -25,6 +25,7 @@ def dynamic_bucket_data_buckets(table, partition, snapshot): read_builder = bucket_read_builder(table, partition, snapshot) # This internal manifest lookup returns no rows and must include every # live bucket, independently of a SELECT policy or level-zero read option. - scanner = read_builder.new_scan().file_scanner + from pypaimon.execution.python.read.table_scan import PythonTableScan + scanner = PythonTableScan(read_builder.new_scan())._file_scanner() scanner.skip_level0 = False return {entry.bucket for entry in scanner.plan_files()} diff --git a/paimon-python/pypaimon/execution/python/write/file_store_write.py b/paimon-python/pypaimon/execution/python/write/file_store_write.py index 21758d212c6b..af5bb5a127d4 100644 --- a/paimon-python/pypaimon/execution/python/write/file_store_write.py +++ b/paimon-python/pypaimon/execution/python/write/file_store_write.py @@ -525,10 +525,11 @@ def _load_seq_number_stats(self, partition: Tuple) -> dict: else predicate_builder.equal(key, value)) partition_filter = predicate_builder.and_predicates(sub_predicates) - scanner = read_builder.with_filter(partition_filter).new_scan().file_scanner + from pypaimon.execution.python.read.table_scan import PythonTableScan + scanner = PythonTableScan(read_builder.with_filter(partition_filter).new_scan())._file_scanner() # Sequence restoration reads file metadata, not query results. As in # Java's writer scan, include L0 and do not require SELECT or apply row - # filters/masks to internal state. Public plan_all_files stays guarded. + # filters/masks to internal state. Public plan_all stays guarded. scanner.skip_level0 = False max_seq_numbers = {} diff --git a/paimon-python/pypaimon/execution/python/write/merge_into.py b/paimon-python/pypaimon/execution/python/write/merge_into.py index 3f80f0ff6d7e..e60defadc98c 100644 --- a/paimon-python/pypaimon/execution/python/write/merge_into.py +++ b/paimon-python/pypaimon/execution/python/write/merge_into.py @@ -536,7 +536,7 @@ def _read_table(table, projection: Optional[List[str]] = None, if projection is not None: read_builder.with_projection(projection) scan = read_builder.new_scan() - return read_builder.new_read().to_arrow(scan.plan_all_files().splits()) + return read_builder.new_read().to_arrow(scan.plan_all().splits()) def _copy_at_snapshot(table, snapshot_id: Optional[int]): diff --git a/paimon-python/pypaimon/execution/python/write/postpone_bucket.py b/paimon-python/pypaimon/execution/python/write/postpone_bucket.py index 1dad6cdc7209..f7e6a95ff513 100644 --- a/paimon-python/pypaimon/execution/python/write/postpone_bucket.py +++ b/paimon-python/pypaimon/execution/python/write/postpone_bucket.py @@ -23,7 +23,8 @@ class PythonPostponeBucketPlanner(PostponeBucketPlanning): @staticmethod def _load_bucket_metadata(table): - scan = table.new_read_builder().new_scan().file_scanner + from pypaimon.execution.python.read.table_scan import PythonTableScan + scan = PythonTableScan(table.new_read_builder().new_scan())._file_scanner() manifest_files, _ = scan.manifest_scanner() entries = scan.manifest_file_manager.read_entries_parallel( manifest_files, diff --git a/paimon-python/pypaimon/execution/python/write/row_id_file_index.py b/paimon-python/pypaimon/execution/python/write/row_id_file_index.py index 4ae14c7f93cd..8eb81b32592e 100644 --- a/paimon-python/pypaimon/execution/python/write/row_id_file_index.py +++ b/paimon-python/pypaimon/execution/python/write/row_id_file_index.py @@ -42,7 +42,7 @@ class RowIdFileIndex: def from_table(cls, table) -> "RowIdFileIndex": """Plan the table's write snapshot and index its splits.""" scan = table.new_read_builder().new_scan() - plan = scan.plan_all_files() + plan = scan.plan_all() snapshot_id = plan.snapshot_id if plan.snapshot_id is not None else -1 return cls.from_splits(snapshot_id, plan.splits()) diff --git a/paimon-python/pypaimon/execution/python/write/table_delete.py b/paimon-python/pypaimon/execution/python/write/table_delete.py index 2a672880adc6..8cfd12595189 100644 --- a/paimon-python/pypaimon/execution/python/write/table_delete.py +++ b/paimon-python/pypaimon/execution/python/write/table_delete.py @@ -143,7 +143,7 @@ def _load_anchor_ranges(self) -> Tuple[int, List[_AnchorRange]]: def _scan_anchor_ranges(self) -> Tuple[int, List[_AnchorRange]]: context = self._context scan = context.table.new_read_builder().new_scan() - plan = scan.plan_all_files() + plan = scan.plan_all() snapshot_id = plan.snapshot_id if plan.snapshot_id is not None else -1 anchors = [] diff --git a/paimon-python/pypaimon/execution/python/write/table_update.py b/paimon-python/pypaimon/execution/python/write/table_update.py index 069d74d99296..21be8bc245df 100644 --- a/paimon-python/pypaimon/execution/python/write/table_update.py +++ b/paimon-python/pypaimon/execution/python/write/table_update.py @@ -157,7 +157,7 @@ def _matched_delete_row_ids( read_builder.with_projection([SpecialFields.ROW_ID.name]) scan = read_builder.new_scan() - splits = scan.plan_all_files().splits() + splits = scan.plan_all().splits() matched = read_builder.new_read().to_arrow(splits) if matched.num_rows == 0: return [] @@ -434,7 +434,7 @@ def _execute_predicate_update( else: read_builder.with_projection([SpecialFields.ROW_ID.name]) - plan = read_builder.new_scan().plan_all_files() + plan = read_builder.new_scan().plan_all() splits = plan.splits() snapshot_id = plan.snapshot_id if plan.snapshot_id is not None else -1 files_info = RowIdFileIndex.from_splits( @@ -546,7 +546,7 @@ def initialize( context.total_shard_count = total_shard_count context.shard_num = shard_num scanner = context.table.new_read_builder().new_scan() - plan = scanner.plan_all_files() + plan = scanner.plan_all() context.snapshot_id = plan.snapshot_id if plan.snapshot_id is not None else -1 splits = plan.splits() splits = _filter_by_whole_file_shard(splits, shard_num, total_shard_count) diff --git a/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py b/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py index 95a769cd2f4d..22e7803ccf88 100644 --- a/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py +++ b/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py @@ -324,7 +324,7 @@ def _build_key_to_row_ids_map( read_builder = read_builder.with_filter(partition_predicate) scan = read_builder.new_scan() - splits = scan.plan_all_files().splits() + splits = scan.plan_all().splits() if not splits: return {} diff --git a/paimon-python/pypaimon/multimodal/blob_store.py b/paimon-python/pypaimon/multimodal/blob_store.py index e2843812c5c7..149397a32e06 100644 --- a/paimon-python/pypaimon/multimodal/blob_store.py +++ b/paimon-python/pypaimon/multimodal/blob_store.py @@ -359,7 +359,7 @@ def _read_update_targets( read_builder = read_builder.with_projection(projection) read_builder = read_builder.with_filter(self._keys_predicate(keys)) scan = read_builder.new_scan() - plan = scan.plan_all_files() + plan = scan.plan_all() table = read_builder.new_read().to_arrow(plan.splits()) targets = {} diff --git a/paimon-python/pypaimon/ray/bucket_join.py b/paimon-python/pypaimon/ray/bucket_join.py index 2b377126dfd4..c3932aae6c26 100644 --- a/paimon-python/pypaimon/ray/bucket_join.py +++ b/paimon-python/pypaimon/ray/bucket_join.py @@ -73,8 +73,6 @@ def _plan_splits_by_bucket(table_id, catalog_options, projection, expected_total raise ValueError( f"bucket_join needs {table_id} fully in bucket count {expected_total_buckets}, " f"but files exist under {sorted(stale)} (rescale in progress); rewrite first.") - # Reuse those entries: scan.plan() re-reads plan_files() (append/pk) otherwise. - fs.plan_files = lambda: entries by_bucket = {} for s in scan.plan().splits(): by_bucket.setdefault(s.bucket, []).append(s) diff --git a/paimon-python/pypaimon/ray/data_evolution_merge_join.py b/paimon-python/pypaimon/ray/data_evolution_merge_join.py index e70da5517d00..db10eb2e92af 100644 --- a/paimon-python/pypaimon/ray/data_evolution_merge_join.py +++ b/paimon-python/pypaimon/ray/data_evolution_merge_join.py @@ -200,7 +200,7 @@ def build_self_merge_update_plan( read_builder = scan_table.new_read_builder().with_projection(projection) if scan_predicate is not None: read_builder.with_filter(scan_predicate) - scan_plan = read_builder.new_scan().plan_all_files() + scan_plan = read_builder.new_scan().plan_all() planned_snapshot_id = ( scan_plan.snapshot_id if scan_plan.snapshot_id is not None else -1 diff --git a/paimon-python/pypaimon/ray/process_row_id_ranges.py b/paimon-python/pypaimon/ray/process_row_id_ranges.py index eb881dc855a7..e2ffa6e2bd5e 100644 --- a/paimon-python/pypaimon/ray/process_row_id_ranges.py +++ b/paimon-python/pypaimon/ray/process_row_id_ranges.py @@ -81,7 +81,7 @@ def _validate_arguments(rows_per_commit, processor) -> None: def _file_group_ranges(table) -> List[Range]: - plan = table.new_read_builder().new_scan().plan_all_files() + plan = table.new_read_builder().new_scan().plan_all() files = [data_file for split in plan.splits() for data_file in split.files] file_groups = RangeHelper( lambda data_file: data_file.non_null_row_id_range() diff --git a/paimon-python/pypaimon/ray/row_id_conflict_rewriter.py b/paimon-python/pypaimon/ray/row_id_conflict_rewriter.py index 0ea623d0863f..d289ad439ea4 100644 --- a/paimon-python/pypaimon/ray/row_id_conflict_rewriter.py +++ b/paimon-python/pypaimon/ray/row_id_conflict_rewriter.py @@ -205,7 +205,7 @@ def _rewrite_updates( scan_table = table.copy_without_time_travel({ CoreOptions.SCAN_SNAPSHOT_ID.key(): str(latest_snapshot.id), }) - scan_plan = scan_table.new_read_builder().new_scan().plan_all_files() + scan_plan = scan_table.new_read_builder().new_scan().plan_all() if scan_plan.snapshot_id != latest_snapshot.id: return None current_splits = list(scan_plan.splits()) diff --git a/paimon-python/pypaimon/read/file_scanner.py b/paimon-python/pypaimon/read/file_scanner.py new file mode 100644 index 000000000000..b8a56cb20fe8 --- /dev/null +++ b/paimon-python/pypaimon/read/file_scanner.py @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""File-level configuration and dispatch for the public table scanner.""" + +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.read.scan_request import scan_partition_predicate +from pypaimon.table.bucket_mode import BucketMode +from pypaimon.utils.range import Range + + +class FileScanner: + """Preserve the file scanner API without selecting an executor on access. + + Planning uses the table's Native options. Manifest sources and follow-up + strategies are internal to the selected executor. + """ + + _is_dispatch_file_scanner = True + _CONFIGURATION_FIELDS = frozenset(( + 'predicate', '_user_predicate', '_partition_predicate', + 'partition_key_predicate', '_explicit_partition_filter', 'limit', + 'skip_level0', 'only_read_real_buckets', 'is_streaming', + 'idx_of_this_subtask', 'number_of_para_subtasks', + 'start_pos_of_this_subtask', 'end_pos_of_this_subtask', + 'chunk_shuffle', '_global_index_result', '_row_ranges', + )) + + def __init__(self, scan): + self._scan = scan + self.table = scan.table + request = scan._scan_request() + self.predicate = request.predicate + self._user_predicate = request.user_predicate + self._partition_predicate = scan.partition_predicate + self.partition_key_predicate = request.partition_key_predicate + self._explicit_partition_filter = request.explicit_partition_filter + self.limit = request.limit + self.data_evolution = request.data_evolution + self.deletion_vectors_enabled = request.deletion_vectors_enabled + self.idx_of_this_subtask = request.idx_of_this_subtask + self.number_of_para_subtasks = request.number_of_para_subtasks + self.start_pos_of_this_subtask = request.start_pos_of_this_subtask + self.end_pos_of_this_subtask = request.end_pos_of_this_subtask + self.chunk_shuffle = request.chunk_shuffle + self._global_index_result = request.global_index_result + self._row_ranges = (None if request.row_ranges is None else + [Range(start, end) for start, end in request.row_ranges]) + self.is_streaming = self.table.options.options.contains(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP) + self.skip_level0 = False if self.is_streaming else self.table.options.batch_scan_skip_level0() + self.only_read_real_buckets = self.table.options.bucket() == BucketMode.POSTPONE_BUCKET.value + self.auth_partition_predicate = None + self.auth_has_non_partition_filter = False + self.scan_stats = None + self._scanned_snapshot = None + self._scanned_snapshot_id = None + self._policy_overrides = set() + self._initialized = True + + def __setattr__(self, name, value): + initialized = self.__dict__.get('_initialized', False) + if initialized and name == 'partition_key_predicate': + object.__setattr__(self, '_explicit_partition_filter', value is not None) + if value is None: + # Like ReadBuilder.withPartitionFilter(null), remove the + # explicit selection and derive pruning from the row filter. + object.__setattr__(self, '_partition_predicate', None) + value = scan_partition_predicate(self.table, self.predicate, None) + object.__setattr__(self, name, value) + if initialized and name in self._CONFIGURATION_FIELDS: + if name == 'predicate': + object.__setattr__(self, '_user_predicate', value) + if not self._explicit_partition_filter: + object.__setattr__(self, 'partition_key_predicate', scan_partition_predicate( + self.table, value, self._partition_predicate)) + if name in ('skip_level0', 'only_read_real_buckets'): + self._policy_overrides.add(name) + self._invalidate_python_scanner() + + def _invalidate_python_scanner(self): + if getattr(self._scan, '_file_scanner', None) is self: + self._scan._python_file_scanner = None + + def _configure_python_scanner(self, scanner): + scanner._user_predicate = self._user_predicate + scanner._partition_predicate = self._partition_predicate + scanner._explicit_partition_filter = self._explicit_partition_filter + scanner._set_predicate(self.predicate) + scanner.partition_key_predicate = self.partition_key_predicate + for name in ('limit', 'skip_level0', 'only_read_real_buckets', 'is_streaming'): + setattr(scanner, name, getattr(self, name)) + # Preserve explicit caller overrides without routing Python execution + # through the public facade again. + for name in ('scan', 'scan_with_stats', 'plan_files'): + if name in self.__dict__: + setattr(scanner, name, self.__dict__[name]) + + def _request(self): + return self._scan._scan_request() + + def scan(self): + from pypaimon.read.read_dispatch import plan_scan + return plan_scan(self._scan) + + def scan_with_stats(self): + from pypaimon.read.read_dispatch import scan_with_stats + plan, stats = scan_with_stats(self._scan) + self.scan_stats = stats + return plan, stats + + def plan_files(self): + from pypaimon.read.read_dispatch import plan_files + return plan_files(self) + + def with_all_buckets(self): + self.only_read_real_buckets = False + return self + + def _apply_request(self, request): + self.idx_of_this_subtask = request.idx_of_this_subtask + self.number_of_para_subtasks = request.number_of_para_subtasks + self.start_pos_of_this_subtask = request.start_pos_of_this_subtask + self.end_pos_of_this_subtask = request.end_pos_of_this_subtask + self.chunk_shuffle = request.chunk_shuffle + self._global_index_result = request.global_index_result + self._row_ranges = (None if request.row_ranges is None else + [Range(start, end) for start, end in request.row_ranges]) + + def with_shard(self, idx_of_this_subtask, number_of_para_subtasks): + self._apply_request(self._request().with_shard(idx_of_this_subtask, number_of_para_subtasks)) + return self + + def with_slice(self, start_pos, end_pos): + self._apply_request(self._request().with_slice(start_pos, end_pos)) + return self + + def with_global_index_result(self, result): + self._apply_request(self._request().with_global_index_result(result)) + return self + + def with_row_ranges(self, row_ranges): + self._apply_request(self._request().with_row_ranges(row_ranges)) + return self + + def with_chunk_shuffle(self, seed, chunk_size): + self._apply_request(self._request().with_chunk_shuffle(seed, chunk_size)) + return self + + def _set_predicate(self, predicate): + self.predicate = predicate diff --git a/paimon-python/pypaimon/read/read_dispatch.py b/paimon-python/pypaimon/read/read_dispatch.py index 195c0826ab50..31fd58ff9a84 100644 --- a/paimon-python/pypaimon/read/read_dispatch.py +++ b/paimon-python/pypaimon/read/read_dispatch.py @@ -28,6 +28,13 @@ from pypaimon.read.scan_stats import ScanStats +def validate_read_backend(read): + """Validate the explicitly selected Python backend without loading Native.""" + if not read.table.options.native_read_enabled(): + from pypaimon.execution.python.read.merge_engine_support import validate_read + validate_read(read) + + def to_iterator(self, splits: List[Split]) -> Iterator: """Lazily read InternalRows, using native reads when available. @@ -149,9 +156,6 @@ def plan_scan(self) -> Plan: if native is not None: return native auth_result = self._TableScan__auth_query() - if self._query_auth_fn is not None and self._file_scanner is not None: - from pypaimon.execution.python.read.auth_support import prune_scanner_by_auth - prune_scanner_by_auth(self.table, self._file_scanner, auth_result) # The native planner bypasses the auth-aware file scanner. Resolve auth # before selecting a planning backend. if (not native_auth and auth_result is None and self._native_requested() @@ -163,8 +167,56 @@ def plan_scan(self) -> Plan: return PythonTableScan(self).plan(auth_result) -def plan_all_files(self) -> Plan: - """Plan matching files including L0 and all data-evolution column files. +def new_file_scanner(scan): + """Create public file configuration without constructing either executor.""" + from pypaimon.read.file_scanner import FileScanner + return FileScanner(scan) + + +def python_file_scanner(scan): + """Enter the selected Python executor's internal file scanner.""" + from pypaimon.execution.python.read.table_scan import PythonTableScan + return PythonTableScan(scan)._file_scanner() + + +def file_scan_has_bucket_pruning(scan): + """Inspect Python statistics only after that backend planned the scan.""" + scanner = python_file_scanner(scan) + return scanner._bucket_selector is not None or scanner.only_read_real_buckets + + +def plan_files(scanner): + """Plan raw file metadata only with unrestricted whole-table permission.""" + scan = scanner._scan + if scan._native_requested(): + from pypaimon.execution.native.table import _native_query_auth_supported + # Raw file metadata precedes query result transformations. Do not let + # their capability checks (shuffle, global indexes or slices) select + # Python for an operation which does not consume those settings. + native_supported = _native_query_auth_supported(scan.table) + else: + native_supported = False + if native_supported: + from pypaimon.execution.native.read.plan import NativeTableScan + result = NativeTableScan(scan).plan_files(scanner.skip_level0, scanner.only_read_real_buckets) + if result is not None: + entries, snapshot_id = result + scanner._scanned_snapshot = None + scanner._scanned_snapshot_id = snapshot_id + return entries + if scan._TableScan__auth_query(all_columns=True) is not None: + raise TableNoPermissionException(scan.table.identifier) + from pypaimon.execution.python.read.table_scan import PythonTableScan + # Let the selected executor refresh cached rules for this unrestricted scope. + source = PythonTableScan(scan)._scanner(None) + entries = source.plan_files() + scanner._scanned_snapshot = source._scanned_snapshot + scanner._scanned_snapshot_id = source._scanned_snapshot_id + return entries + + +def plan_all(self) -> Plan: + """Plan splits including L0 and all matching data-evolution column files. Existing filters and limits still apply. Query authorization must be unrestricted, as required by writer restore, overwrite and indexing. @@ -181,7 +233,7 @@ def plan_all_files(self) -> Plan: if self._TableScan__auth_query(all_columns=True) is not None: raise TableNoPermissionException(self.table.identifier) from pypaimon.execution.python.read.table_scan import PythonTableScan - return PythonTableScan(self).plan_all_files() + return PythonTableScan(self).plan_all() def scan_with_stats(self) -> Tuple[Plan, Optional[ScanStats]]: @@ -199,9 +251,6 @@ def scan_with_stats(self) -> Tuple[Plan, Optional[ScanStats]]: if native is not None: return native auth_result = self._TableScan__auth_query() - if self._query_auth_fn is not None and self._file_scanner is not None: - from pypaimon.execution.python.read.auth_support import prune_scanner_by_auth - prune_scanner_by_auth(self.table, self._file_scanner, auth_result) if (not native_auth and auth_result is None and self._native_requested() and self._native_plan_supported()): from pypaimon.execution.native.read.plan import NativeTableScan @@ -213,36 +262,48 @@ def scan_with_stats(self) -> Tuple[Plan, Optional[ScanStats]]: def new_streaming_scan(self): - """Create a stateful streaming scan using the configured backend.""" - from pypaimon.write.backend_dispatch import propagate_fork_safety_error - + """Create the public streaming facade with this builder's read request.""" projection = self._projection_builder() projection._validate_map_key_filter() - if self.table.options.native_plan_enabled(): + from pypaimon.read.streaming_table_scan import StreamTableScan + return StreamTableScan( + self.table, predicate=self._predicate, read_type=projection.read_type(), + poll_interval_ms=self._poll_interval_ms, bucket_filter=self._bucket_filter, + consumer_id=self._consumer_id) + + +def create_streaming_scan(scan, poll_interval_ms=1000, prefetch_enabled=True, + diff_threshold=10): + """Prepare one executor before any frame or checkpoint is consumed.""" + from pypaimon.write.backend_dispatch import propagate_fork_safety_error + + if native_plan_requested(scan): from pypaimon.execution.native.table import _native_query_auth_supported - native_supported = _native_query_auth_supported(self.table) + native_supported = _native_query_auth_supported(scan.table) else: native_supported = False if native_supported: try: from pypaimon.execution.native.read.streaming_table_scan import StreamTableScan return StreamTableScan( - self.table, predicate=self._predicate, read_type=projection.read_type(), - poll_interval_ms=self._poll_interval_ms, bucket_filter=self._bucket_filter, - consumer_id=self._consumer_id) + scan.table, predicate=scan.predicate, read_type=scan._read_type, + poll_interval_ms=poll_interval_ms, bucket_filter=scan._bucket_filter, + consumer_id=scan._consumer_id) except Exception as error: propagate_fork_safety_error(error) logging.warning("Cannot create Native stream scan, using Python: %s", error) from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan - scan = AsyncStreamingTableScan( - table=self.table, - predicate=self._predicate, - poll_interval_ms=self._poll_interval_ms, - bucket_filter=self._bucket_filter, - consumer_id=self._consumer_id + execution = AsyncStreamingTableScan( + table=scan.table, + predicate=scan.predicate, + poll_interval_ms=poll_interval_ms, + bucket_filter=scan._bucket_filter, + prefetch_enabled=prefetch_enabled, + diff_threshold=diff_threshold, + consumer_id=scan._consumer_id ) - scan._read_type = projection.read_type() - return scan + execution._read_type = scan._read_type + return execution def native_plan_requested(scan) -> bool: diff --git a/paimon-python/pypaimon/read/read_explain.py b/paimon-python/pypaimon/read/read_explain.py index 1f34f979f36a..9197d4631fe3 100644 --- a/paimon-python/pypaimon/read/read_explain.py +++ b/paimon-python/pypaimon/read/read_explain.py @@ -198,8 +198,8 @@ def _bucket_pruning(stats: ScanStats, scan: TableScan) -> Optional[PruningStat]: # HASH_FIXED predicate-driven selector OR the POSTPONE_BUCKET # synthetic-bucket skip. Tables with neither (e.g. BUCKET_UNAWARE # append) leave this counter as ``None``. - fs = scan.file_scanner - if fs._bucket_selector is None and not fs.only_read_real_buckets: + from pypaimon.read.read_dispatch import file_scan_has_bucket_pruning + if not file_scan_has_bucket_pruning(scan): return None return PruningStat(before=stats.entries_after_partition, after=stats.entries_after_bucket) diff --git a/paimon-python/pypaimon/read/scan_config.py b/paimon-python/pypaimon/read/scan_config.py index a9e890e15ceb..e31106625c6f 100644 --- a/paimon-python/pypaimon/read/scan_config.py +++ b/paimon-python/pypaimon/read/scan_config.py @@ -140,12 +140,8 @@ def _parse_incremental_timestamp_window(value): return start, end -def resolve_scan_snapshot_range(table): - """Validate public scan inputs and resolve incremental snapshot boundaries. - - This is metadata selection shared by both planners. Manifest readers and - execution resources are created only by the selected backend. - """ +def validate_scan_options(table): + """Validate public scan inputs without opening snapshot metadata.""" from pypaimon.snapshot.time_travel_util import SCAN_KEYS _validate_scan_mode(table) @@ -157,28 +153,5 @@ def resolve_scan_snapshot_range(table): raise ValueError( "incremental-between-timestamp cannot be used together with " "point-in-time scan options: %s" % SCAN_KEYS) - start_timestamp, end_timestamp = _parse_incremental_timestamp_window( + return _parse_incremental_timestamp_window( options.get(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP)) - snapshot_manager = table.snapshot_manager() - latest_snapshot = snapshot_manager.get_latest_snapshot() - earliest_snapshot = snapshot_manager.try_get_earliest_snapshot() if latest_snapshot is not None else None - if earliest_snapshot is None or latest_snapshot is None: - return None - if start_timestamp > end_timestamp: - raise ValueError( - "Ending timestamp %s should be >= starting timestamp %s." - % (end_timestamp, start_timestamp)) - if (start_timestamp == end_timestamp or start_timestamp > latest_snapshot.time_millis - or end_timestamp < earliest_snapshot.time_millis): - return None - - starting_snapshot = snapshot_manager.earlier_or_equal_time_mills(start_timestamp) - earliest_snapshot = snapshot_manager.try_get_earliest_snapshot() - if starting_snapshot is None or (earliest_snapshot and earliest_snapshot.time_millis > start_timestamp): - start_id = earliest_snapshot.id - 1 if earliest_snapshot else -1 - else: - start_id = starting_snapshot.id - end_snapshot = snapshot_manager.earlier_or_equal_time_mills(end_timestamp) - latest_snapshot = snapshot_manager.get_latest_snapshot() - end_id = end_snapshot.id if end_snapshot else (latest_snapshot.id if latest_snapshot else -1) - return start_id, end_id diff --git a/paimon-python/pypaimon/read/scan_request.py b/paimon-python/pypaimon/read/scan_request.py index a14a134d5dc0..b17eb962758b 100644 --- a/paimon-python/pypaimon/read/scan_request.py +++ b/paimon-python/pypaimon/read/scan_request.py @@ -22,7 +22,8 @@ from pypaimon.common.predicate import Predicate from pypaimon.read.push_down_utils import ( - _get_all_fields, predicate_for_physical_fields, predicate_matches_physical_names, + _get_all_fields, + predicate_for_physical_fields, predicate_matches_physical_names, rewrite_predicate_indices, trim_and_transform_predicate, trim_predicate_by_fields) from pypaimon.read.scan_distribution import validate_shard, validate_slice from pypaimon.read.split import DataSplit @@ -62,7 +63,7 @@ class ScanRequest: read_limit: Optional[int] = None partition_filter: Optional[Predicate] = None read_type: Optional[Tuple[DataField, ...]] = None - incremental_snapshot_range: Optional[Tuple[int, int]] = None + file_filters: Optional[Tuple[bool, bool]] = None @classmethod def for_table(cls, table, predicate=None, limit=None, partition_predicate=None): diff --git a/paimon-python/pypaimon/read/streaming_table_scan.py b/paimon-python/pypaimon/read/streaming_table_scan.py index bf64f9a897b3..382362217a55 100644 --- a/paimon-python/pypaimon/read/streaming_table_scan.py +++ b/paimon-python/pypaimon/read/streaming_table_scan.py @@ -15,33 +15,165 @@ # specific language governing permissions and limitations # under the License. -"""Public streaming scan classes backed by Python or Native execution.""" +"""Public streaming scans selecting Python or Native execution.""" -import importlib -import sys +import asyncio +from typing import AsyncGenerator, Callable, Iterator, Optional -if sys.version_info[:2] < (3, 7): - # Python 3.6 does not resolve module attributes through __getattr__. - from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan - from pypaimon.execution.native.read.streaming_table_scan import StreamTableScan +from pypaimon.common.predicate import Predicate +from pypaimon.read.plan import Plan -__all__ = ['AsyncStreamingTableScan', 'StreamTableScan'] -_MODULE_BY_EXPORT = { - 'AsyncStreamingTableScan': 'pypaimon.execution.python.read.streaming_table_scan', - 'StreamTableScan': 'pypaimon.execution.native.read.streaming_table_scan', -} +class _StreamingScan: + """Public lifecycle and polling over one prepared streaming executor.""" + + def _initialize(self, table, predicate, read_type, poll_interval_ms, + bucket_filter, consumer_id, prefetch_enabled=True, + diff_threshold=10): + self.table = table + self.predicate = predicate + self._read_type = read_type + self.poll_interval = poll_interval_ms / 1000.0 + self._bucket_filter = bucket_filter + self._consumer_id = consumer_id + self._pending_consumer_snapshot = None + self._pending_consumer_owner = None + self._closed = False + self._generation = 0 + from pypaimon.read.read_dispatch import create_streaming_scan + self._execution = create_streaming_scan( + self, poll_interval_ms, prefetch_enabled, diff_threshold) + + @property + def next_snapshot_id(self): + return self.checkpoint() + + @next_snapshot_id.setter + def next_snapshot_id(self, next_snapshot_id): + self.restore(next_snapshot_id) + + def checkpoint(self): + """Return the next snapshot to consume, without acknowledging it.""" + return self._execution.checkpoint() + + def watermark(self): + """Return the watermark of the last successfully planned frame.""" + return self._execution.watermark() + + def restore(self, next_snapshot_id=None): + """Restore a checkpoint, or reset to the configured starting position.""" + self._require_open() + self._execution.restore(next_snapshot_id) + self._generation += 1 + self._pending_consumer_snapshot = None + self._pending_consumer_owner = None + + def notify_checkpoint_complete(self, next_snapshot): + """Persist progress only after the caller has processed its plan.""" + self._require_open() + self._execution.notify_checkpoint_complete(next_snapshot) + + def plan(self): + """Plan one available frame without changing execution backends.""" + self._require_open() + generation = self._generation + plan = self._execution.plan() + self._require_open() + if generation != self._generation: + raise RuntimeError("stream scan was restored during planning") + return plan + + def _flush_pending_consumer(self): + if self._pending_consumer_snapshot is not None: + self.notify_checkpoint_complete(self._pending_consumer_snapshot) + self._pending_consumer_snapshot = None + self._pending_consumer_owner = None + async def stream(self) -> AsyncGenerator[Plan, None]: + """Poll public controls and acknowledge a yielded frame on resume.""" + owner = object() + try: + while True: + self._flush_pending_consumer() + checkpoint = self.checkpoint() + plan = self.plan() + if plan is None: + next_snapshot = self.checkpoint() + if next_snapshot is not None and next_snapshot != checkpoint: + self._pending_consumer_snapshot = next_snapshot + self._pending_consumer_owner = owner + self._flush_pending_consumer() + await asyncio.sleep(self.poll_interval) + continue + self._pending_consumer_snapshot = self.checkpoint() + self._pending_consumer_owner = owner + yield plan + except (GeneratorExit, asyncio.CancelledError): + if self._pending_consumer_owner is owner: + self._pending_consumer_snapshot = None + self._pending_consumer_owner = None + self._execution.cancel_prefetch() + raise -def __getattr__(name): - module_name = _MODULE_BY_EXPORT.get(name) - if module_name is None: - raise AttributeError( - "module 'pypaimon.read.streaming_table_scan' has no attribute {!r}".format(name)) - value = getattr(importlib.import_module(module_name), name) - globals()[name] = value - return value + def stream_sync(self) -> Iterator[Plan]: + """Synchronously poll this scan and close its asynchronous iterator.""" + loop = asyncio.new_event_loop() + iterator = self.stream() + try: + while True: + yield loop.run_until_complete(iterator.__anext__()) + finally: + try: + loop.run_until_complete(iterator.aclose()) + finally: + loop.close() + def _require_open(self): + if self._closed: + raise RuntimeError("stream scan is closed") -def __dir__(): - return sorted(set(globals()) | set(__all__)) + def close(self): + """Stop polling without acknowledging rows still owned by the caller.""" + if self._closed: + return + self._closed = True + self._generation += 1 + self._pending_consumer_snapshot = None + self._pending_consumer_owner = None + self._execution.close() + + +class AsyncStreamingTableScan(_StreamingScan): + """Continuous async or synchronous polling with the configured backend. + + ``prefetch_enabled`` controls Python's speculative planning; + Native plans frames on demand. + ``diff_threshold`` is retained without replacing physical change events + with a net snapshot diff. + """ + + def __init__( + self, + table, + predicate: Optional[Predicate] = None, + poll_interval_ms: int = 1000, + bucket_filter: Optional[Callable[[int], bool]] = None, + prefetch_enabled: bool = True, + diff_threshold: int = 10, + consumer_id: Optional[str] = None + ): + self._initialize( + table, predicate, None, poll_interval_ms, bucket_filter, consumer_id, + prefetch_enabled, diff_threshold) + + +class StreamTableScan(_StreamingScan): + """Continuous scan with a canonical reader type and configured backend.""" + + def __init__(self, table, predicate=None, read_type=None, poll_interval_ms=1000, + bucket_filter=None, consumer_id=None): + self._initialize(table, predicate, read_type, poll_interval_ms, + bucket_filter, consumer_id) + + +__all__ = ['AsyncStreamingTableScan', 'StreamTableScan'] diff --git a/paimon-python/pypaimon/read/table_read.py b/paimon-python/pypaimon/read/table_read.py index 0108f7204621..ccac665d3437 100644 --- a/paimon-python/pypaimon/read/table_read.py +++ b/paimon-python/pypaimon/read/table_read.py @@ -30,7 +30,7 @@ from pypaimon.read.blob_read_support import deferred_blob_field_names from pypaimon.schema.data_types import DataField -from pypaimon.read.constants import ROW_KIND_COLUMN, _DEFAULT_PREDICATE +from pypaimon.read.constants import ROW_KIND_COLUMN if TYPE_CHECKING: @@ -57,10 +57,9 @@ def __init__( from pypaimon.read.merge_engine_support import check_sequence_field_valid if table.is_primary_key_table: check_sequence_field_valid(table) - self._python_merge_validation_pending = True self.table: "FileStoreTable" = table - if not table.options.native_read_enabled(): - self._check_python_merge_supported() + from pypaimon.read.read_dispatch import validate_read_backend + validate_read_backend(self) self.predicate = predicate self.read_type = read_type self.output_projection = output_projection @@ -645,164 +644,6 @@ def convert_rows_to_arrow_batch(row_tuples: List[tuple], schema: pyarrow.Schema) from pypaimon.read.read_support import convert_rows_to_arrow_batch return convert_rows_to_arrow_batch(row_tuples, schema) - def _arrow_batch_generator(self, splits: List[Split], schema: pyarrow.Schema, - blob_parallelism: int = 1) -> Iterator[pyarrow.RecordBatch]: - from pypaimon.execution.python.read.table_read import PythonTableRead - yield from PythonTableRead(self)._arrow_batch_generator(splits, schema, blob_parallelism) - - def _to_arrow_parallel( - self, - splits: List[Split], - schema: pyarrow.Schema, - effective: int, - blob_parallelism: int = 1, - ) -> pyarrow.Table: - """Read ``splits`` concurrently and assemble the result in input order. - - Each split is read in its own worker thread; row quota for ``limit`` - is shared through :class:`_RemainingRows` so the combined output - never exceeds ``self.limit`` rows. Per-split batches are collected - by submission index, so the merged table preserves the order of the - input ``splits`` list. - """ - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._to_arrow_parallel(splits, schema, effective, blob_parallelism) - - def _read_one_split_to_batches( - self, - split: Split, - schema: pyarrow.Schema, - remaining_state: "_RemainingRows", - blob_parallelism: int = 1, - ) -> List[pyarrow.RecordBatch]: - """Read a single split into arrow batches under soft-stop control. - - Row quota is debited against the shared ``remaining_state``; once a - request returns 0, the worker stops emitting further batches. The - reader is always closed via ``finally``. - """ - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._read_one_split_to_batches(split, schema, remaining_state, blob_parallelism) - - def _convert_rows_to_output_batches( - self, - row_tuples: List[tuple], - row_kinds: List[str], - schema: pyarrow.Schema, - ) -> Iterator[pyarrow.RecordBatch]: - from pypaimon.execution.python.read.table_read import PythonTableRead - yield from PythonTableRead(self)._convert_rows_to_output_batches(row_tuples, row_kinds, schema) - - def _convert_rows_to_arrow_batches_with_row_kind( - self, - row_tuples: List[tuple], - row_kinds: List[str], - schema: pyarrow.Schema - ) -> Iterator[pyarrow.RecordBatch]: - """Convert rows to one or more Arrow batches, optionally including row kind column. - - Yields more than one batch only when a column overflows pyarrow's 2GB - per-column limit (see ``_emit_overflow_safe_batches``); otherwise a single - batch is produced as before. - """ - from pypaimon.execution.python.read.table_read import PythonTableRead - yield from PythonTableRead(self)._convert_rows_to_arrow_batches_with_row_kind( - row_tuples, - row_kinds, - schema, - ) - - def _create_split_read(self, split: Split, blob_parallelism: int = 1, - read_type=None, limit: Optional[int] = None, - push_down_limit: bool = True, - post_merge_filter=None, - eager_blob_fields=None, - post_filter_after_inline: bool = False, - predicate=_DEFAULT_PREDICATE, - defer_blob_resolution: bool = False) -> "SplitRead": - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._create_split_read( - split, - blob_parallelism, - read_type, - limit, - push_down_limit, - post_merge_filter, - eager_blob_fields, - post_filter_after_inline, - predicate, - defer_blob_resolution, - ) - - def _build_split_read(self, split: Split, read_type=None, - limit: Optional[int] = None, - push_down_limit: bool = True, - post_merge_filter=None, - eager_blob_fields=None, - post_filter_after_inline: bool = False, - predicate=_DEFAULT_PREDICATE, - defer_blob_resolution: bool = False) -> "SplitRead": - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._build_split_read( - split, - read_type, - limit, - push_down_limit, - post_merge_filter, - eager_blob_fields, - post_filter_after_inline, - predicate, - defer_blob_resolution, - ) - - def _serialize_blob_views_as_descriptors(self, batch: pyarrow.RecordBatch, reader) -> pyarrow.RecordBatch: - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._serialize_blob_views_as_descriptors(batch, reader) - - def _serialize_blob_view_row_tuple(self, row: "OffsetRow", reader) -> tuple: - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._serialize_blob_view_row_tuple(row, reader) - - def _widen_to_top_level_for_merge(self, name_paths=None) -> List[DataField]: - """Unique top-level fields from ``name_paths``, in path order.""" - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._widen_to_top_level_for_merge(name_paths) - - def _only_map_key_nested_paths(self) -> bool: - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._only_map_key_nested_paths() - - def _has_map_key_projection(self) -> bool: - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._has_map_key_projection() - - def _map_has_aggregator(self, field_name: str) -> bool: - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._map_has_aggregator(field_name) - - def _check_python_merge_supported(self): - if getattr(self, '_python_merge_validation_pending', False): - from pypaimon.execution.python.read.merge_engine_support import check_supported - check_supported(self.table) - self._python_merge_validation_pending = False - - def _TableRead__create_reader_for_split( - self, split, blob_parallelism=1, limit: Optional[int] = None): - # Raw-convertible splits skip MergeFileSplitRead, so validate before - # dispatch, including when native setup failed or bindings are absent. - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._TableRead__create_reader_for_split(split, blob_parallelism, limit) - - def _TableRead__authed_reader( - self, split, auth_result, blob_parallelism=1, limit: Optional[int] = None): - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead(self)._TableRead__authed_reader(split, auth_result, blob_parallelism, limit) - - @staticmethod - def _auth_filter_field_names(auth_result, read_fields) -> set: - from pypaimon.execution.python.read.table_read import PythonTableRead - return PythonTableRead._auth_filter_field_names(auth_result, read_fields) - def _native_fallback( self, reason: str, diff --git a/paimon-python/pypaimon/read/table_scan.py b/paimon-python/pypaimon/read/table_scan.py index 228aaaaba3d8..6743269401e5 100755 --- a/paimon-python/pypaimon/read/table_scan.py +++ b/paimon-python/pypaimon/read/table_scan.py @@ -19,15 +19,13 @@ from typing import List, Optional, Tuple, TYPE_CHECKING from pypaimon.common.predicate import Predicate -from pypaimon.common.options.core_options import CoreOptions from pypaimon.read.auth_support import ( authorize, latest_auth_fields, validate_auth_rules) from pypaimon.read.plan import Plan from pypaimon.read.scan_config import ( - _validate_scan_mode, _parse_incremental_timestamp_window, resolve_scan_snapshot_range) + _validate_scan_mode, _parse_incremental_timestamp_window, validate_scan_options) from pypaimon.read.scan_request import ScanRequest from pypaimon.read.scan_stats import ScanStats -from pypaimon.utils.range import Range if TYPE_CHECKING: @@ -52,37 +50,23 @@ def __init__( self._query_auth_fn = self.table.catalog_environment.table_query_auth( self.table.options, self.table.identifier) self._file_scanner = None - self._incremental_snapshot_range = resolve_scan_snapshot_range(self.table) - empty_incremental = ( - self.table.options.options.contains(CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP) - and self._incremental_snapshot_range is None) - # Empty incremental scanners historically carry no row predicate or LIMIT. + self._python_file_scanner = None + validate_scan_options(self.table) self._scan_configuration = ScanRequest.for_table( - table, None if empty_incremental else predicate, - None if empty_incremental else limit, partition_predicate) + table, predicate, limit, partition_predicate) @property def file_scanner(self): - """Create the Python scanner when callers explicitly request its API.""" + """Return file-level configuration using the selected execution backend.""" if self._file_scanner is None: - scanner = self._create_file_scanner() - request = self._scan_configuration - if request.idx_of_this_subtask is not None: - scanner.with_shard(request.idx_of_this_subtask, request.number_of_para_subtasks) - if request.start_pos_of_this_subtask is not None: - scanner.with_slice(request.start_pos_of_this_subtask, request.end_pos_of_this_subtask) - if request.global_index_result is not None: - scanner.with_global_index_result(request.global_index_result) - if request.row_ranges is not None: - scanner.with_row_ranges([Range(start, end) for start, end in request.row_ranges]) - if request.chunk_shuffle is not None: - scanner.with_chunk_shuffle(*request.chunk_shuffle) - self._file_scanner = scanner + self._file_scanner = self._create_file_scanner() return self._file_scanner @file_scanner.setter def file_scanner(self, scanner): self._file_scanner = scanner + self._python_file_scanner = ( + None if getattr(type(scanner), '_is_dispatch_file_scanner', False) is True else scanner) def _scan_request(self) -> ScanRequest: """Capture current public scanner configuration for one preparation.""" @@ -94,8 +78,8 @@ def _scan_request(self) -> ScanRequest: read_limit=self.limit, partition_filter=self.partition_predicate, read_type=tuple(self._read_type) if self._read_type is not None else None, - incremental_snapshot_range=self._incremental_snapshot_range, ) + neutral = getattr(type(scanner), '_is_dispatch_file_scanner', False) is True row_ranges = getattr(scanner, '_row_ranges', None) chunk_shuffle = getattr(scanner, 'chunk_shuffle', None) return ScanRequest( @@ -117,29 +101,32 @@ def _scan_request(self) -> ScanRequest: global_index_result=scanner._global_index_result, row_ranges=(tuple((range_.from_, range_.to) for range_ in row_ranges) if row_ranges is not None else None), - read_predicate=self.predicate, - read_limit=self.limit, - partition_filter=self.partition_predicate, + read_predicate=scanner._user_predicate if neutral else self.predicate, + read_limit=scanner.limit if neutral else self.limit, + partition_filter=((scanner.partition_key_predicate if scanner._explicit_partition_filter else None) + if neutral else self.partition_predicate), read_type=tuple(self._read_type) if self._read_type is not None else None, - incremental_snapshot_range=getattr(self, '_incremental_snapshot_range', None), + file_filters=((scanner.skip_level0, scanner.only_read_real_buckets) + if neutral and scanner._policy_overrides else None), ) def plan(self) -> Plan: from pypaimon.read.read_dispatch import plan_scan return plan_scan(self) - def plan_all_files(self) -> Plan: - """Plan matching files including L0 and all data-evolution column files. + def plan_all(self) -> Plan: + """Plan splits including L0 and all matching data-evolution column files. Existing filters and limits still apply. Query authorization must be unrestricted, as required by writer restore, overwrite and indexing. """ - from pypaimon.read.read_dispatch import plan_all_files - return plan_all_files(self) + from pypaimon.read.read_dispatch import plan_all + return plan_all(self) def __auth_query(self, all_columns=False): + request = self._scan_request() return authorize(self.table, self._query_auth_fn, None if all_columns else self._read_type, - self.predicate, self.partition_predicate) + request.read_predicate, request.partition_filter) def scan_with_stats(self) -> Tuple[Plan, Optional[ScanStats]]: """Run :meth:`plan` while recording manifest / pruning counters. @@ -153,6 +140,7 @@ def scan_with_stats(self) -> Tuple[Plan, Optional[ScanStats]]: def with_shard(self, idx_of_this_subtask, number_of_para_subtasks) -> 'TableScan': if self._file_scanner is None: + self._python_file_scanner = None self._scan_configuration = self._scan_configuration.with_shard(idx_of_this_subtask, number_of_para_subtasks) else: self._file_scanner.with_shard(idx_of_this_subtask, number_of_para_subtasks) @@ -160,6 +148,7 @@ def with_shard(self, idx_of_this_subtask, number_of_para_subtasks) -> 'TableScan def with_slice(self, start_pos, end_pos) -> 'TableScan': if self._file_scanner is None: + self._python_file_scanner = None self._scan_configuration = self._scan_configuration.with_slice(start_pos, end_pos) else: self._file_scanner.with_slice(start_pos, end_pos) @@ -167,6 +156,7 @@ def with_slice(self, start_pos, end_pos) -> 'TableScan': def with_global_index_result(self, result) -> 'TableScan': if self._file_scanner is None: + self._python_file_scanner = None self._scan_configuration = self._scan_configuration.with_global_index_result(result) else: self._file_scanner.with_global_index_result(result) @@ -174,6 +164,7 @@ def with_global_index_result(self, result) -> 'TableScan': def with_row_ranges(self, row_ranges) -> 'TableScan': if self._file_scanner is None: + self._python_file_scanner = None self._scan_configuration = self._scan_configuration.with_row_ranges(row_ranges) else: self._file_scanner.with_row_ranges(row_ranges) @@ -181,6 +172,7 @@ def with_row_ranges(self, row_ranges) -> 'TableScan': def with_chunk_shuffle(self, seed: int, chunk_size: int) -> 'TableScan': if self._file_scanner is None: + self._python_file_scanner = None self._scan_configuration = self._scan_configuration.with_chunk_shuffle(seed, chunk_size) else: self._file_scanner.with_chunk_shuffle(seed, chunk_size) @@ -190,8 +182,8 @@ def _validate_scan_mode(self): _validate_scan_mode(self.table) def _create_file_scanner(self) -> "FileScanner": - from pypaimon.execution.python.read.table_scan import PythonTableScan - return PythonTableScan(self)._create_file_scanner() + from pypaimon.read.read_dispatch import new_file_scanner + return new_file_scanner(self) def _native_requested(self) -> bool: from pypaimon.read.read_dispatch import native_plan_requested @@ -229,7 +221,7 @@ def _try_native_plan(self, all_files=False) -> Optional[Plan]: """ from pypaimon.execution.native.read.plan import NativeTableScan execution = NativeTableScan(self) - return execution.plan_all_files() if all_files else execution.plan() + return execution.plan_all() if all_files else execution.plan() __all__ = [ diff --git a/paimon-python/pypaimon/tests/batch_scan_level_filter_test.py b/paimon-python/pypaimon/tests/batch_scan_level_filter_test.py index 15ef6e93b3b5..d2e5563ac47e 100644 --- a/paimon-python/pypaimon/tests/batch_scan_level_filter_test.py +++ b/paimon-python/pypaimon/tests/batch_scan_level_filter_test.py @@ -119,7 +119,7 @@ def test_write_and_incremental_scans_keep_l0(tmp_path, engine, dv): _write(table, [{'id': 1, 'v': 'updated'}, {'id': 2, 'v': 'new'}], 200) scan = table.new_read_builder().new_scan() assert sum(len(split.files) for split in scan.plan().splits()) == 1 - assert sum(len(split.files) for split in scan.plan_all_files().splits()) == 2 + assert sum(len(split.files) for split in scan.plan_all().splits()) == 2 assert sum(len(split.files) for split in scan.plan().splits()) == 1 incremental = table.copy({'incremental-between-timestamp': '100,200'}) assert _read(incremental, False)[1] == [{'id': 1, 'v': 'updated'}, {'id': 2, 'v': 'new'}] @@ -136,7 +136,7 @@ def test_copy_can_disable_persisted_merge_on_read(tmp_path, native, override): def test_first_row_streaming_and_overwrite_see_uncompacted_files(tmp_path): - from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan + from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan table = _table(tmp_path, 'first-row', False, False) _write(table, [{'id': 1, 'v': 'before'}], 100) @@ -149,6 +149,6 @@ def test_first_row_streaming_and_overwrite_see_uncompacted_files(tmp_path): _write(table, [{'id': 2, 'v': 'after'}], 200, overwrite=True) builder = table.new_read_builder() - plan = builder.new_scan().plan_all_files() + plan = builder.new_scan().plan_all() assert sum(len(split.files) for split in plan.splits()) == 1 assert builder.new_read().to_arrow(plan.splits()).to_pylist() == [{'id': 2, 'v': 'after'}] diff --git a/paimon-python/pypaimon/tests/batch_vector_raw_scan_test.py b/paimon-python/pypaimon/tests/batch_vector_raw_scan_test.py index b998cc65cfa1..58f26a51c039 100644 --- a/paimon-python/pypaimon/tests/batch_vector_raw_scan_test.py +++ b/paimon-python/pypaimon/tests/batch_vector_raw_scan_test.py @@ -22,6 +22,7 @@ import pyarrow as pa import pytest +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.read.table_read import TableRead from pypaimon.execution.python.read.search.vector_search_read import BatchVectorSearchReadImpl from pypaimon.tests.data_evolution_test_helpers import BatchModeMixin, DataEvolutionTestBase @@ -124,7 +125,7 @@ def test_scoring_finishes_each_batch_before_reading_the_next(self): self._write_arrow(table, self._data([[1, 0], [0, 1], [2, 0], [0, 2]])) reader = self._reader(table, [[1, 0], [0, 1]]) original = TableRead._new_arrow_batch_reader - original_generator = TableRead._arrow_batch_generator + original_generator = PythonTableRead._arrow_batch_generator generators = [] batch_sizes = [] module = 'pypaimon.execution.python.read.search.vector_search_read' @@ -148,7 +149,7 @@ def tracked_generator(table_read, *args): source.close() with patch.object(TableRead, '_new_arrow_batch_reader', batches), \ - patch.object(TableRead, '_arrow_batch_generator', tracked_generator), \ + patch.object(PythonTableRead, '_arrow_batch_generator', tracked_generator), \ patch(module + '._compute_scores', wraps=_compute_scores) as score: reader._read_raw_batch_search([Range(0, 3)], None, 'ivf-flat') self.assertEqual(4, score.call_count) @@ -206,7 +207,7 @@ def test_public_batch_search_preserves_split_parallelism(self): for partition in range(4): self._write_arrow(table, self._data( [[1, 0], [0, 1], None, [0, 0], [partition, 1]], partition)) - original = TableRead._arrow_batch_generator + original = PythonTableRead._arrow_batch_generator for parallelism in (1, 2, 4, None): options = {} if parallelism is None else {'read.parallelism': str(parallelism)} read_table = table.copy(options) @@ -238,7 +239,7 @@ def tracked(table_read, splits, *args): state['active'] -= 1 state['closed'] += 1 - with patch.object(TableRead, '_arrow_batch_generator', tracked), \ + with patch.object(PythonTableRead, '_arrow_batch_generator', tracked), \ patch('pypaimon.read.read_support.os.cpu_count', return_value=4): actual = (read_table.new_batch_vector_search_builder() .with_vector_column('embedding').with_query_vectors(queries) @@ -255,7 +256,7 @@ def test_parallel_failure_closes_all_started_readers(self): partition_keys=['pt'], options=dict(self.table_options, **{'read.parallelism': '2'})) for partition in range(4): self._write_arrow(table, self._data([[1, 0], [0, 1]], partition)) - original = TableRead._arrow_batch_generator + original = PythonTableRead._arrow_batch_generator for failure in ('dimension', 'read'): with self.subTest(failure=failure): barrier = threading.Barrier(2) @@ -282,7 +283,7 @@ def tracked(table_read, *args): query = [1] if failure == 'dimension' else [1, 0] exception = ValueError if failure == 'dimension' else RuntimeError message = 'dimension mismatch' if failure == 'dimension' else 'injected parallel read failure' - with patch.object(TableRead, '_arrow_batch_generator', tracked): + with patch.object(PythonTableRead, '_arrow_batch_generator', tracked): with self.assertRaisesRegex(exception, message): (table.new_batch_vector_search_builder().with_vector_column('embedding') .with_query_vectors([query]).with_limit(2).execute_batch_local()) diff --git a/paimon-python/pypaimon/tests/blob_table_test.py b/paimon-python/pypaimon/tests/blob_table_test.py index a972c040de45..c65686f1ea24 100755 --- a/paimon-python/pypaimon/tests/blob_table_test.py +++ b/paimon-python/pypaimon/tests/blob_table_test.py @@ -26,6 +26,7 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.schema.schema_change import SchemaChange from pypaimon.table.file_store_table import FileStoreTable from pypaimon.table.row.blob import Blob @@ -5474,7 +5475,6 @@ def test_blob_view_raw_split_predicate_and_limit_resolves_filtered_row(self): from pypaimon import Schema from pypaimon.globalindex.indexed_split import IndexedSplit from pypaimon.execution.python.read.split_read import (RawFileSplitRead) - from pypaimon.read.table_read import TableRead from pypaimon.table.row.blob import BlobViewStruct from pypaimon.utils.range import Range @@ -5550,14 +5550,14 @@ def test_blob_view_raw_split_predicate_and_limit_resolves_filtered_row(self): predicate = read_builder.new_predicate_builder().equal("id", 9) read_builder = read_builder.with_filter(predicate).with_limit(1) split_types = [] - orig_build = TableRead._build_split_read + orig_build = PythonTableRead._build_split_read def capturing_build(self, *args, **kwargs): split_read = orig_build(self, *args, **kwargs) split_types.append(type(split_read)) return split_read - with mock.patch.object(TableRead, '_build_split_read', capturing_build): + with mock.patch.object(PythonTableRead, '_build_split_read', capturing_build): result = read_builder.new_read().to_arrow( read_builder.new_scan().plan().splits() ) diff --git a/paimon-python/pypaimon/tests/blob_test.py b/paimon-python/pypaimon/tests/blob_test.py index c211ec73d655..ed8299380046 100644 --- a/paimon-python/pypaimon/tests/blob_test.py +++ b/paimon-python/pypaimon/tests/blob_test.py @@ -34,6 +34,7 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.common.delta_varint_compressor import DeltaVarintCompressor from pypaimon.common.file_io import FileIO from pypaimon.common.options import Options @@ -2998,19 +2999,19 @@ def test_table_read_serializes_only_configured_blob_view_fields(self): reader = SimpleNamespace(blob_view_lookup=lookup) row = OffsetRow((view_bytes, view_bytes), 0, 2) - converted = table_read._serialize_blob_view_row_tuple(row, reader) + converted = PythonTableRead(table_read)._serialize_blob_view_row_tuple(row, reader) self.assertEqual(converted[0], view_bytes) self.assertEqual(converted[1], descriptor.serialize()) table_read._blob_view_output_indices = () - skipped = table_read._serialize_blob_view_row_tuple( + skipped = PythonTableRead(table_read)._serialize_blob_view_row_tuple( row, SimpleNamespace()) self.assertEqual(skipped, (view_bytes, view_bytes)) table_read._blob_view_output_indices = (1,) table_read._blob_as_descriptor = False - not_descriptor = table_read._serialize_blob_view_row_tuple(row, reader) + not_descriptor = PythonTableRead(table_read)._serialize_blob_view_row_tuple(row, reader) self.assertEqual(not_descriptor, (view_bytes, view_bytes)) def test_offset_row_get_blob_v1_resolved_blob_view_field(self): diff --git a/paimon-python/pypaimon/tests/daft/daft_pk_distributed_write_test.py b/paimon-python/pypaimon/tests/daft/daft_pk_distributed_write_test.py index 82b844691c67..a9a0fb58c356 100644 --- a/paimon-python/pypaimon/tests/daft/daft_pk_distributed_write_test.py +++ b/paimon-python/pypaimon/tests/daft/daft_pk_distributed_write_test.py @@ -34,6 +34,7 @@ daft = pytest.importorskip("daft") from daft.recordbatch.micropartition import MicroPartition +from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon.daft import daft_datasink from pypaimon.daft.daft_paimon import _read_table, _write_table from pypaimon.daft.daft_datasink import PaimonDataSink @@ -235,7 +236,7 @@ def test_postpone_mode_primary_key_write_preserves_existing_path(catalog): assert sum(summary["rows"]) == 2 assert table.snapshot_manager().get_latest_snapshot() is not None - scanner = table.new_read_builder().new_scan().file_scanner + scanner = PythonTableScan(table.new_read_builder().new_scan())._file_scanner() manifests, _ = scanner.manifest_scanner() entries = scanner.manifest_file_manager.read_entries_parallel(manifests) assert {entry.bucket for entry in entries} == {-2} diff --git a/paimon-python/pypaimon/tests/deferred_blob_resolve_test.py b/paimon-python/pypaimon/tests/deferred_blob_resolve_test.py index 57fefb93a672..d6ed4688e9c1 100644 --- a/paimon-python/pypaimon/tests/deferred_blob_resolve_test.py +++ b/paimon-python/pypaimon/tests/deferred_blob_resolve_test.py @@ -27,6 +27,7 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.catalog.table_query_auth import TableQueryAuthResult from pypaimon.read.query_auth_split import QueryAuthSplit from pypaimon.execution.python.read.reader.format_blob_reader import (BlobRecordIterator) @@ -249,7 +250,7 @@ def test_limit_does_not_prefetch_payloads_across_splits(self): table_read = read_builder.new_read() with patch.object( - table_read, + PythonTableRead, "_to_arrow_parallel", side_effect=AssertionError("deferred LIMIT must run serially"), ) as parallel_read: @@ -273,9 +274,10 @@ def test_limit_covering_all_rows_preserves_parallelism(self): table_read = read_builder.new_read() with patch.object( - table_read, + PythonTableRead, "_to_arrow_parallel", - wraps=table_read._to_arrow_parallel, + autospec=True, + side_effect=PythonTableRead._to_arrow_parallel, ) as parallel_read: result = table_read.to_arrow(splits, parallelism=4) diff --git a/paimon-python/pypaimon/tests/global_index_build_test.py b/paimon-python/pypaimon/tests/global_index_build_test.py index a49e3243f7f6..fd3163dab980 100644 --- a/paimon-python/pypaimon/tests/global_index_build_test.py +++ b/paimon-python/pypaimon/tests/global_index_build_test.py @@ -29,6 +29,7 @@ import pyarrow as pa +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.execution.python.globalindex.build_plan import ( filter_non_indexable_splits as _filter_non_indexable_splits, split_by_global_index_shard as _split_by_global_index_shard, @@ -681,7 +682,6 @@ def test_create_vindex_global_index_from_python(self): entry.index_file.file_name))) def test_create_vindex_streaming_failure_cleans_resources(self): - from pypaimon.read.table_read import TableRead schema = pa.schema([('embedding', pa.list_(pa.float32()))]) table = self._create_table(pa_schema=schema, options=dict( @@ -690,7 +690,7 @@ def test_create_vindex_streaming_failure_cleans_resources(self): {'embedding': [[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]]}, schema=schema)) snapshot_id = table.snapshot_manager().get_latest_snapshot().id original_write = VindexVectorIndexWriter.write_batch - original_batches = TableRead._arrow_batch_generator + original_batches = PythonTableRead._arrow_batch_generator temp_paths = [] closed = [] generators = [] @@ -712,7 +712,7 @@ def generate(): return generator with patch.object(VindexVectorIndexWriter, 'write_batch', failing_write), \ - patch.object(TableRead, '_arrow_batch_generator', tracked_batches): + patch.object(PythonTableRead, '_arrow_batch_generator', tracked_batches): with self.assertRaisesRegex(RuntimeError, 'injected write failure'): table.create_global_index('embedding', index_type='ivf-flat', options={ 'ivf-flat.dimension': '2', diff --git a/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py b/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py index 17887a204124..b23894e62655 100644 --- a/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py +++ b/paimon-python/pypaimon/tests/global_index_execution_boundary_test.py @@ -180,7 +180,7 @@ def find_spec(self, fullname, path=None, target=None): from pypaimon.schema.data_types import ArrayType, AtomicType, DataField read_builder = Mock() -read_builder.new_scan.return_value.plan_all_files.return_value.splits.return_value = [] +read_builder.new_scan.return_value.plan_all.return_value.splits.return_value = [] table = SimpleNamespace( options=CoreOptions(Options({ 'bucket': '-1', 'row-tracking.enabled': 'true', 'write.native.enabled': 'false', diff --git a/paimon-python/pypaimon/tests/global_index_test.py b/paimon-python/pypaimon/tests/global_index_test.py index 4e109b7b6265..38ebbd6fa62b 100644 --- a/paimon-python/pypaimon/tests/global_index_test.py +++ b/paimon-python/pypaimon/tests/global_index_test.py @@ -23,6 +23,7 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon.common.options.core_options import CoreOptions, GlobalIndexSearchMode from pypaimon.common.options.options import Options from pypaimon.common.predicate import Predicate @@ -233,6 +234,7 @@ def test_detail_mode_uses_table_data_ranges(self): class GlobalIndexScalarFallbackTest(unittest.TestCase): + @pytest.mark.python_plan def test_full_fallback_groups_are_pruned_before_split_packing(self): from pypaimon.execution.python.read.scanner.file_scanner import (_GlobalIndexPlanningResult) @@ -278,17 +280,17 @@ def test_full_fallback_groups_are_pruned_before_split_packing(self): ) baseline_scan = read_builder.new_scan() - baseline_scan.file_scanner._global_index_result = index_plan - baseline_scan.file_scanner.predicate_for_stats = None + PythonTableScan(baseline_scan)._file_scanner()._global_index_result = index_plan + PythonTableScan(baseline_scan)._file_scanner().predicate_for_stats = None baseline_plan = baseline_scan.plan() scan = read_builder.new_scan() - scan.file_scanner._global_index_result = index_plan + PythonTableScan(scan)._file_scanner()._global_index_result = index_plan plan = scan.plan() stats_scan = read_builder.new_scan() - stats_scan.file_scanner._global_index_result = index_plan - stats_plan, stats = stats_scan.file_scanner.scan_with_stats() + PythonTableScan(stats_scan)._file_scanner()._global_index_result = index_plan + stats_plan, stats = PythonTableScan(stats_scan)._file_scanner().scan_with_stats() self.assertEqual( 10, diff --git a/paimon-python/pypaimon/tests/lazy_read_execution_test.py b/paimon-python/pypaimon/tests/lazy_read_execution_test.py index 6160b0a67477..6444bb4ac166 100644 --- a/paimon-python/pypaimon/tests/lazy_read_execution_test.py +++ b/paimon-python/pypaimon/tests/lazy_read_execution_test.py @@ -42,7 +42,7 @@ def table(tmp_path): return catalog.get_table('default.lazy_execution') -@pytest.mark.parametrize('method', ['plan', 'plan_all_files', 'scan_with_stats']) +@pytest.mark.parametrize('method', ['plan', 'plan_all', 'scan_with_stats']) @pytest.mark.parametrize('unrestricted_auth', [False, True]) def test_native_scan_does_not_construct_python_executors(table, method, unrestricted_auth): with ExitStack() as stack: @@ -80,6 +80,31 @@ def test_python_scanner_is_constructed_only_after_native_preparation_fails(table assert [(r.from_, r.to) for r in scan.file_scanner._row_ranges] == [(1, 4)] +@pytest.mark.parametrize('method', ['plan', 'plan_all', 'scan_with_stats']) +@pytest.mark.parametrize('window', ['100,100', '100,200']) +def test_native_timestamp_scan_delegates_snapshot_selection_and_authorization(table, method, window): + table = table.copy({'incremental-between-timestamp': window, 'query-auth.enabled': 'true'}) + with ExitStack() as stack: + stack.enter_context(patch.object( + table, 'snapshot_manager', side_effect=AssertionError('Python resolved Native snapshots'))) + stack.enter_context(patch.object( + table.catalog_environment, 'table_query_auth', return_value=object())) + stack.enter_context(patch( + 'pypaimon.read.table_scan.TableScan._native_plan_supported', return_value=True)) + stack.enter_context(patch( + 'pypaimon.execution.native.table._native_query_auth_supported', return_value=True)) + stack.enter_context(patch( + 'pypaimon.read.table_scan.authorize', side_effect=AssertionError('Python authorization ran'))) + native = stack.enter_context(patch( + 'pypaimon.execution.native.read.plan.native_plan', return_value=Plan([], 7))) + scan = table.new_read_builder().new_scan() + result = getattr(scan, method)() + plan = result[0] if method == 'scan_with_stats' else result + assert plan.snapshot_id == 7 + native.assert_called_once() + assert 'incremental_range' not in native.call_args.kwargs + + def test_explicit_scanner_access_and_mutation_remain_visible_to_native(table): scan = table.new_read_builder().new_scan() scanner = scan.file_scanner @@ -122,13 +147,20 @@ def batches(): assert events == ['closed'] -def test_native_stream_scan_does_not_construct_python_executors(table): +@pytest.mark.parametrize('native_plan,native_read', [(True, False), (False, True), (True, True)]) +def test_native_stream_scan_does_not_construct_python_executors(table, native_plan, native_read): + table = table.copy({ + 'scan.native-plan.enabled': str(native_plan).lower(), + 'read.native.enabled': str(native_read).lower(), + }) with ExitStack() as stack: for target in ( 'pypaimon.execution.python.read.streaming_table_scan.FileScanner', 'pypaimon.execution.python.read.streaming_table_scan.ManifestListManager', 'pypaimon.execution.python.read.streaming_table_scan.ManifestFileManager'): stack.enter_context(patch(target, side_effect=AssertionError(target))) + stack.enter_context(patch( + 'pypaimon.execution.native.runtime.native_runtime_available', return_value=True)) core = stack.enter_context(patch( 'pypaimon.execution.native.read.streaming_table_scan.native_stream_scan')).return_value expected = Plan([], 7) @@ -139,6 +171,23 @@ def test_native_stream_scan_does_not_construct_python_executors(table): core.plan.assert_called_once() +@pytest.mark.parametrize('runtime_available', [False, True]) +def test_native_read_stream_falls_back_before_planning(table, runtime_available): + from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan + + table = table.copy({'scan.native-plan.enabled': 'false', 'read.native.enabled': 'true'}) + with patch('pypaimon.execution.native.runtime.native_runtime_available', return_value=runtime_available), \ + patch('pypaimon.execution.native.read.streaming_table_scan.native_stream_scan', + side_effect=RuntimeError('prepare')) as native: + scan = table.new_stream_read_builder().new_streaming_scan() + try: + assert isinstance(scan._execution, AsyncStreamingTableScan) + assert scan.plan() is None + finally: + scan.close() + assert native.call_count == int(runtime_available) + + def test_python_read_resources_are_created_after_native_setup_failure(table): from pypaimon.execution.python.read.merge_engine_support import check_supported from pypaimon.execution.python.read.split_read import RawFileSplitRead diff --git a/paimon-python/pypaimon/tests/multimodal_table_test.py b/paimon-python/pypaimon/tests/multimodal_table_test.py index a4d913f8b733..21c6c74eac19 100644 --- a/paimon-python/pypaimon/tests/multimodal_table_test.py +++ b/paimon-python/pypaimon/tests/multimodal_table_test.py @@ -28,6 +28,7 @@ import pyarrow as pa import pytest import pypaimon.multimodal as pmm +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.multimodal import source_col from pypaimon.common.json_util import JSON from pypaimon.common.predicate_builder import PredicateBuilder @@ -995,7 +996,7 @@ def test_blob_store_list_reads_batches_and_stops_at_limit(self): for key in ("other/a", "other/b", "images/a", "images/b", "images/c", "images/d") ]) store = table.blobs(column="image") - original = TableRead._arrow_batch_generator + original = PythonTableRead._arrow_batch_generator batches, closed = [], [] def tracked_read(read, *args, **kwargs): @@ -1009,7 +1010,7 @@ def tracked_read(read, *args, **kwargs): closed.append(True) with patch.object(TableRead, "to_arrow", side_effect=AssertionError("full read")), \ - patch.object(TableRead, "_arrow_batch_generator", tracked_read): + patch.object(PythonTableRead, "_arrow_batch_generator", tracked_read): self.assertEqual([], store.list_objects(limit=0)) self.assertEqual([], batches) listed = store.list_objects(prefix="images/", limit=1, columns=["owner"]) @@ -1549,7 +1550,7 @@ def batches(*args): closed.append(True) with patch( - "pypaimon.read.table_read.TableRead._arrow_batch_generator", + "pypaimon.execution.python.read.table_read.PythonTableRead._arrow_batch_generator", new=batches), patch( "pypaimon.read.read_support." "_RECORD_BATCH_READER_FROM_STREAM", None): @@ -1560,7 +1561,7 @@ def batches(*args): closed.clear() with patch( - "pypaimon.read.table_read.TableRead._arrow_batch_generator", + "pypaimon.execution.python.read.table_read.PythonTableRead._arrow_batch_generator", new=batches): reader = users.scan().select("id").to_arrow_batch_reader() reader.read_next_batch() @@ -2183,7 +2184,7 @@ def test_scan_stream_blobs_closes_underlying_iterator(self): "id": pa.int32(), "image": pa.large_binary(), }), options=dict(_PARQUET_OPTIONS, **{"read.batch-size": "1"})) table.add([{"id": i, "image": b"body"} for i in range(3)]) - original = TableRead._arrow_batch_generator + original = PythonTableRead._arrow_batch_generator closed = [] def tracked_read(read, *args, **kwargs): @@ -2195,7 +2196,7 @@ def tracked_read(read, *args, **kwargs): closed.append(True) with patch.object(TableRead, "_try_native_batches", return_value=None), \ - patch.object(TableRead, "_arrow_batch_generator", tracked_read): + patch.object(PythonTableRead, "_arrow_batch_generator", tracked_read): stream = table.scan().stream_blobs("image") next(stream) stream.close() diff --git a/paimon-python/pypaimon/tests/native_blob_collection_write_test.py b/paimon-python/pypaimon/tests/native_blob_collection_write_test.py index 7d6303b689bc..c6790089237e 100644 --- a/paimon-python/pypaimon/tests/native_blob_collection_write_test.py +++ b/paimon-python/pypaimon/tests/native_blob_collection_write_test.py @@ -289,7 +289,10 @@ def test_native_collection_projection_keeps_selected_blob_keys(tmp_path, project read = read_builder.new_read() splits = read_builder.new_scan().plan().splits() if native: - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): result = read.to_arrow(splits) else: expected = read.to_arrow(splits) diff --git a/paimon-python/pypaimon/tests/native_blob_row_write_test.py b/paimon-python/pypaimon/tests/native_blob_row_write_test.py index cf6ffea3af56..b5581235dfea 100644 --- a/paimon-python/pypaimon/tests/native_blob_row_write_test.py +++ b/paimon-python/pypaimon/tests/native_blob_row_write_test.py @@ -103,7 +103,10 @@ def _table(tmp_path, kind, external=False, extra=None, payload_type=None): def _read(table): builder = table.new_read_builder() read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python read fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + ): return sorted(read.to_arrow(builder.new_scan().plan().splits()).to_pylist(), key=lambda row: row['id']) diff --git a/paimon-python/pypaimon/tests/native_blob_update_test.py b/paimon-python/pypaimon/tests/native_blob_update_test.py index ef248653bef4..4598ab355ac2 100644 --- a/paimon-python/pypaimon/tests/native_blob_update_test.py +++ b/paimon-python/pypaimon/tests/native_blob_update_test.py @@ -78,7 +78,10 @@ def _read(table, descriptors=False, projection=None, resolve_views=True): builder.with_projection(projection) splits = builder.new_scan().plan().splits() read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python read fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + ): return read.to_arrow(splits).sort_by('id').to_pylist() @@ -666,8 +669,11 @@ def test_view_prescan_keeps_inline_descriptor_predicate(tmp_path, native_rest_ca read_builder.with_filter(PredicateBuilder(view.fields).equal('reference', reference)) plan = read_builder.new_scan().plan() read = read_builder.new_read() - with (patch.object(read, '_create_split_read', side_effect=AssertionError('Python read fallback')) - if native else ExitStack()): + guard = (patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + ) if native else ExitStack()) + with guard: rows = read.to_arrow(plan.splits()).to_pylist() assert len(rows) == 1 and rows[0]['id'] == 0 actual = BlobDescriptor.deserialize(rows[0]['payload']) diff --git a/paimon-python/pypaimon/tests/native_blob_write_test.py b/paimon-python/pypaimon/tests/native_blob_write_test.py index bd05437b8f9b..5caff413d5fc 100644 --- a/paimon-python/pypaimon/tests/native_blob_write_test.py +++ b/paimon-python/pypaimon/tests/native_blob_write_test.py @@ -63,8 +63,10 @@ def _read(table, planner, reader): read = builder.new_read() with ExitStack() as stack: if reader: - stack.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) return sorted(read.to_arrow(plan.splits()).to_pylist(), key=lambda row: row['id']) diff --git a/paimon-python/pypaimon/tests/native_branch_write_test.py b/paimon-python/pypaimon/tests/native_branch_write_test.py index 9d2ef8f544f3..03ca2d0e1562 100644 --- a/paimon-python/pypaimon/tests/native_branch_write_test.py +++ b/paimon-python/pypaimon/tests/native_branch_write_test.py @@ -89,11 +89,14 @@ def _rows(table, native): builder = table.copy({'scan.native-plan.enabled': str(native).lower(), 'read.native.enabled': str(native).lower()}).new_read_builder() scan = builder.new_scan() - splits = (scan.plan_all_files() if table.is_primary_key_table + splits = (scan.plan_all() if table.is_primary_key_table and table.options.merge_engine() == MergeEngine.FIRST_ROW else scan.plan()).splits() reader = builder.new_read() if native: - with patch.object(reader, '_create_split_read', side_effect=AssertionError('Python read fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + ): rows = reader.to_arrow(splits).to_pylist() else: rows = reader.to_arrow(splits).to_pylist() diff --git a/paimon-python/pypaimon/tests/native_bucket_selector_test.py b/paimon-python/pypaimon/tests/native_bucket_selector_test.py index b8b54c9b6c93..5d31185f0f3c 100644 --- a/paimon-python/pypaimon/tests/native_bucket_selector_test.py +++ b/paimon-python/pypaimon/tests/native_bucket_selector_test.py @@ -48,8 +48,10 @@ def _selected(table, predicate, native, stats=True, partition_filter=None, limit side_effect=AssertionError('Native scan fell back'))) stack.enter_context(patch.object(scan.file_scanner, 'scan_with_stats', side_effect=AssertionError('Native statistics fell back'))) - stack.enter_context(patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Native read fell back'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native read fell back') + )) if stats: plan, counters = scan.scan_with_stats() assert plan.stats() is counters diff --git a/paimon-python/pypaimon/tests/native_cast_completion_test.py b/paimon-python/pypaimon/tests/native_cast_completion_test.py index 8fbdc0e6368f..845ad33f4861 100644 --- a/paimon-python/pypaimon/tests/native_cast_completion_test.py +++ b/paimon-python/pypaimon/tests/native_cast_completion_test.py @@ -51,8 +51,10 @@ def table_mode(request): @contextmanager def native_only(): - with patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Native read fell back')), \ + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native read fell back') + ), \ patch('pypaimon.execution.python.read.predicate_json_parser._apply_predicate_transform', side_effect=AssertionError('Python executed an authorization transform')), \ patch('pypaimon.execution.python.read.reader.auth_masking_reader._apply_predicate_transform', diff --git a/paimon-python/pypaimon/tests/native_changelog_batch_test.py b/paimon-python/pypaimon/tests/native_changelog_batch_test.py index 9b9736fc4c84..9fae5af3a938 100644 --- a/paimon-python/pypaimon/tests/native_changelog_batch_test.py +++ b/paimon-python/pypaimon/tests/native_changelog_batch_test.py @@ -120,9 +120,10 @@ def _read(table, native, window=(0, 300), shard=None, predicate=None, with ExitStack() as stack: if native: assert all(getattr(split, '_native_split', None) is not None for split in plan.splits()) - stack.enter_context(patch.object( - read, '_create_split_read', - side_effect=AssertionError('batch changelog reading used Python'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('batch changelog reading used Python') + )) rows = read.to_arrow(plan.splits(), parallelism=1).to_pylist() return plan, rows diff --git a/paimon-python/pypaimon/tests/native_commit_test.py b/paimon-python/pypaimon/tests/native_commit_test.py index a51c8f749050..084d63f59d8a 100644 --- a/paimon-python/pypaimon/tests/native_commit_test.py +++ b/paimon-python/pypaimon/tests/native_commit_test.py @@ -649,8 +649,10 @@ def call(self, context): assert all(isinstance(entry, ManifestEntry) for entry in context.commit_entries) assert all(isinstance(entry.partition, GenericRow) for entry in context.commit_entries) # Re-enter Rust's runtime from the Python callback after publication. - with patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('callback read fell back')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('callback read fell back') + ): calls.append((context, _rows(table))) commit = builder.new_commit() diff --git a/paimon-python/pypaimon/tests/native_constructed_evolution_test.py b/paimon-python/pypaimon/tests/native_constructed_evolution_test.py index 2eab7af72686..a9f8ec4b13c8 100644 --- a/paimon-python/pypaimon/tests/native_constructed_evolution_test.py +++ b/paimon-python/pypaimon/tests/native_constructed_evolution_test.py @@ -87,8 +87,10 @@ def test_large_list_constructed_strings_use_native_read( # while its Parquet reader normalizes MAP values to regular List. for native in (False, True): reader = table.copy({'read.native.enabled': str(native).lower()}).new_read_builder().new_read() - guard = patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Native read fell back')) if native else ExitStack() + guard = patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native read fell back') + ) if native else ExitStack() with guard: result = reader.to_arrow(splits).sort_by([('id', 'ascending')]) # Java's constructed casts use STRING for children and ignore the diff --git a/paimon-python/pypaimon/tests/native_cross_partition_write_test.py b/paimon-python/pypaimon/tests/native_cross_partition_write_test.py index 521228603e76..5be045d434c4 100644 --- a/paimon-python/pypaimon/tests/native_cross_partition_write_test.py +++ b/paimon-python/pypaimon/tests/native_cross_partition_write_test.py @@ -52,10 +52,13 @@ def _rows(table, native): builder = table.copy({'scan.native-plan.enabled': str(native).lower(), 'read.native.enabled': str(native).lower()}).new_read_builder() # Java first-row scans hide un-compacted L0 files; a write scan includes them. - splits = builder.new_scan().plan_all_files().splits() + splits = builder.new_scan().plan_all().splits() reader = builder.new_read() if native: - with patch.object(reader, '_create_split_read', side_effect=AssertionError('fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('fallback') + ): rows = reader.to_arrow(splits).to_pylist() else: rows = reader.to_arrow(splits).to_pylist() @@ -224,7 +227,10 @@ def test_cross_partition_internal_row_kinds_and_migration_deletes(tmp_path): read = builder.new_read() read.include_row_kind = True if native: - with patch.object(read, '_create_split_read', side_effect=AssertionError('fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('fallback') + ): rows = read.to_arrow(plan.splits(), parallelism=1).to_pylist() else: rows = read.to_arrow(plan.splits(), parallelism=1).to_pylist() diff --git a/paimon-python/pypaimon/tests/native_data_directory_test.py b/paimon-python/pypaimon/tests/native_data_directory_test.py index 9fe60560c105..579cde26ed79 100644 --- a/paimon-python/pypaimon/tests/native_data_directory_test.py +++ b/paimon-python/pypaimon/tests/native_data_directory_test.py @@ -94,8 +94,10 @@ def _read(table, native_scan, native_read, streaming=False, snapshot=None): read = builder.new_read() with ExitStack() as stack: if native_read: - stack.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) if streaming: reader = read.to_arrow_batch_reader(plan.splits()) try: diff --git a/paimon-python/pypaimon/tests/native_data_evolution_commit_test.py b/paimon-python/pypaimon/tests/native_data_evolution_commit_test.py index f4f2c096fa5b..220413a822a6 100644 --- a/paimon-python/pypaimon/tests/native_data_evolution_commit_test.py +++ b/paimon-python/pypaimon/tests/native_data_evolution_commit_test.py @@ -79,8 +79,10 @@ def _read(table, native=False, metadata=False): read = builder.new_read() with ExitStack() as stack: if native: - stack.enter_context(patch.object( - read, '_create_split_read', side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) return read.to_arrow(builder.new_scan().plan().splits()).sort_by('id').to_pydict() diff --git a/paimon-python/pypaimon/tests/native_decimal_predicate_test.py b/paimon-python/pypaimon/tests/native_decimal_predicate_test.py index 691fa783cea9..9ee484c58501 100644 --- a/paimon-python/pypaimon/tests/native_decimal_predicate_test.py +++ b/paimon-python/pypaimon/tests/native_decimal_predicate_test.py @@ -110,8 +110,10 @@ def _read(table, predicate, native_plan, native_read, projection=('id',)): if native_read: stack.enter_context(patch.object(reader, '_native_fallback', side_effect=AssertionError('Python read fallback'))) - stack.enter_context(patch.object(reader, '_create_split_read', - side_effect=AssertionError('Python split reader'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python split reader') + )) rows = reader.to_arrow(plan.splits(), parallelism=1).to_pylist() return sorted(rows, key=lambda row: row['id']) diff --git a/paimon-python/pypaimon/tests/native_dv_merge_engine_write_test.py b/paimon-python/pypaimon/tests/native_dv_merge_engine_write_test.py index 5efe6809e2e2..cb94f3a5e4d3 100644 --- a/paimon-python/pypaimon/tests/native_dv_merge_engine_write_test.py +++ b/paimon-python/pypaimon/tests/native_dv_merge_engine_write_test.py @@ -86,7 +86,10 @@ def _rows(table, native=True, merge_on_read=True, predicate=None, projection=Non builder.with_projection(projection) read = builder.new_read() if native: - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python read fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + ): result = read.to_arrow(builder.new_scan().plan().splits()) else: result = read.to_arrow(builder.new_scan().plan().splits()) diff --git a/paimon-python/pypaimon/tests/native_dynamic_write_test.py b/paimon-python/pypaimon/tests/native_dynamic_write_test.py index 32e3f385be0d..54bba8d9f043 100644 --- a/paimon-python/pypaimon/tests/native_dynamic_write_test.py +++ b/paimon-python/pypaimon/tests/native_dynamic_write_test.py @@ -73,7 +73,10 @@ def _rows(table, native): splits = builder.new_scan().plan().splits() reader = builder.new_read() if native: - with patch.object(reader, '_create_split_read', side_effect=AssertionError('fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('fallback') + ): rows = reader.to_arrow(splits).to_pylist() else: rows = reader.to_arrow(splits).to_pylist() diff --git a/paimon-python/pypaimon/tests/native_external_write_test.py b/paimon-python/pypaimon/tests/native_external_write_test.py index 758d1caf6c35..2495a88291da 100644 --- a/paimon-python/pypaimon/tests/native_external_write_test.py +++ b/paimon-python/pypaimon/tests/native_external_write_test.py @@ -96,8 +96,10 @@ def _read(table, planner, reader, snapshot=None): read = builder.new_read() with ExitStack() as stack: if reader: - stack.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) result = read.to_arrow(plan.splits()) return sorted(result.to_pylist(), key=lambda row: row['id']) diff --git a/paimon-python/pypaimon/tests/native_file_scan_test.py b/paimon-python/pypaimon/tests/native_file_scan_test.py new file mode 100644 index 000000000000..72e51df1f39f --- /dev/null +++ b/paimon-python/pypaimon/tests/native_file_scan_test.py @@ -0,0 +1,399 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Public file scans dispatch without constructing the other backend.""" + +from contextlib import ExitStack +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import pyarrow as pa +import pytest + +from pypaimon import CatalogFactory, Schema +from pypaimon.catalog.catalog_exception import TableNoPermissionException +from pypaimon.execution.native.read.file_scan import from_native_file_plan +from pypaimon.execution.native.read.plan import NativeTableScan +from pypaimon.execution.native.runtime import native_runtime_available +from pypaimon.execution.python.read.table_scan import PythonTableScan +from pypaimon.read.plan import Plan +from pypaimon.schema.schema_change import SchemaChange +from pypaimon.tests import native_plan_rest_test +from pypaimon.write.commit_message import CommitMessage +from pypaimon.write.commit_message_serializer import serialize_commit_message + + +rest_catalog = native_plan_rest_test.rest_catalog +SCHEMA = pa.schema([('id', pa.int32()), ('pt', pa.string())]) + + +@pytest.fixture +def table(tmp_path): + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + catalog.create_table('default.files', Schema.from_pyarrow_schema( + SCHEMA, partition_keys=['pt'], options={'file.format': 'parquet'}), False) + return catalog.get_table('default.files') + + +def _append(table, rows): + builder = table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.Table.from_pylist(rows, schema=SCHEMA)) + messages = writer.prepare_commit() + commit.commit(messages) + return messages + finally: + writer.close() + commit.close() + + +@pytest.mark.parametrize('native_plan,native_read', [(True, False), (False, True), (True, True)]) +@pytest.mark.parametrize('method', ['scan', 'scan_with_stats', 'plan_files']) +def test_public_file_scanner_does_not_initialize_python(table, method, native_plan, native_read): + table = table.copy({'scan.native-plan.enabled': str(native_plan).lower(), + 'read.native.enabled': str(native_read).lower()}) + with ExitStack() as stack: + for target in ('FileScanner', 'ManifestListManager'): + stack.enter_context(patch('pypaimon.execution.python.read.table_scan.' + target, + side_effect=AssertionError('Python execution constructed'))) + stack.enter_context(patch('pypaimon.execution.native.runtime.native_runtime_available', return_value=True)) + scan = table.new_read_builder().new_scan() + facade = scan.file_scanner + assert facade is scan.file_scanner + native = stack.enter_context(patch('pypaimon.execution.native.read.plan.native_plan', + return_value=Plan([], 7))) + files = stack.enter_context(patch.object(NativeTableScan, 'plan_files', return_value=(['file'], 7))) + result = getattr(facade, method)() + if method == 'plan_files': + assert result == ['file'] and facade._scanned_snapshot_id == 7 + files.assert_called_once_with(False, False) + native.assert_not_called() + else: + assert (result[0] if method == 'scan_with_stats' else result).snapshot_id == 7 + native.assert_called_once() + files.assert_not_called() + + +@pytest.mark.parametrize('skip_level0,only_real', [(False, False), (False, True), (True, False), (True, True)]) +def test_native_raw_file_planning_ignores_query_result_transforms(table, skip_level0, only_real): + scan = table.new_read_builder().with_projection(['id']).with_limit(1).new_scan() + scan.with_shard(0, 3) + scan.file_scanner.with_all_buckets() + native = Mock() + raw = native.new_scan.return_value + raw.with_file_filters.return_value = raw + plan = raw.plan_files.return_value + plan.files.return_value = [] + plan.snapshot_id.return_value = 9 + with patch('pypaimon.execution.native.read.builder._native_read_builder', return_value=native): + assert NativeTableScan(scan).plan_files(skip_level0, only_real) == ([], 9) + raw.with_file_filters.assert_called_once_with(skip_level0, only_real) + raw.plan_files.assert_called_once_with() + native.with_projection.assert_not_called() + native.with_limit.assert_not_called() + raw.with_shard.assert_not_called() + native.with_row_ranges.assert_not_called() + + +def test_raw_file_dispatch_does_not_probe_query_result_capabilities(table): + table = table.copy({'scan.native-plan.enabled': 'true'}) + scan = table.new_read_builder().with_limit(1).new_scan() + scanner = scan.file_scanner.with_chunk_shuffle(1, 2) + with patch.object(scan, '_native_plan_supported', side_effect=AssertionError('Query transform probe')), \ + patch.object(NativeTableScan, 'plan_files', return_value=(['file'], 7)) as native: + assert scanner.plan_files() == ['file'] + native.assert_called_once_with(False, False) + + +def test_file_filters_are_forwarded_only_after_explicit_configuration(table): + scan = table.new_read_builder().new_scan() + facade = scan.file_scanner + with patch('pypaimon.execution.native.read.plan.native_plan', return_value=Plan([], 7)) as native: + scan._try_native_plan() + assert 'file_filters' not in native.call_args[1] + facade.skip_level0 = True + facade.with_all_buckets() + scan._try_native_plan() + assert native.call_args[1]['file_filters'] == (True, False) + + +@pytest.mark.parametrize('assign_directly', [False, True]) +def test_changed_file_predicate_is_used_by_native_planning(table, assign_directly): + scan = table.new_read_builder().new_scan() + predicate = table.new_read_builder().new_predicate_builder().equal('id', 2) + scanner = scan.file_scanner + if assign_directly: + scanner.predicate = predicate + else: + scanner._set_predicate(predicate) + with patch('pypaimon.execution.native.read.plan.native_plan', return_value=Plan([], 7)) as native: + scan._try_native_plan() + assert native.call_args[1]['predicate'] is predicate + + +def test_changed_file_predicate_is_included_in_query_authorization(table): + scan = table.new_read_builder().with_projection(['id']).new_scan() + requested = [] + + def authorize(columns): + requested.append(set(columns)) + return None + + scan._query_auth_fn = authorize + scan.file_scanner._set_predicate(table.new_read_builder().new_predicate_builder().equal('pt', 'a')) + scan.plan() + assert requested == [{'id', 'pt'}] + + +def test_changed_file_partition_selection_is_included_in_query_authorization(table): + scan = table.new_read_builder().with_projection(['id']).new_scan() + requested = [] + + def authorize(columns): + requested.append(set(columns)) + return None + + scan._query_auth_fn = authorize + scanner = scan.file_scanner + scanner.partition_key_predicate = table.new_read_builder().new_predicate_builder().equal('pt', 'b').new_index(0) + scan.plan() + assert requested == [{'id', 'pt'}] + + +@pytest.mark.parametrize('native_auth', [False, True]) +@pytest.mark.parametrize('original_partition_filter', [False, True]) +def test_clearing_partition_selector_restores_derived_pruning_without_native_fallback( + table, native_auth, original_partition_filter): + _append(table, [{'id': 1, 'pt': 'a'}, {'id': 2, 'pt': 'b'}]) + predicates = table.new_read_builder().new_predicate_builder() + predicate = predicates.equal('pt', 'a') + expected = table.new_read_builder().with_filter(predicate).new_scan().plan() + table = table.copy({'scan.native-plan.enabled': 'true', 'read.native.enabled': 'false', + 'query-auth.enabled': str(native_auth).lower()}) + builder = table.new_read_builder() + builder.with_filter(predicate).with_projection(['id']) + if original_partition_filter: + builder.with_partition_filter(predicates.equal('pt', 'b')) + scan = builder.new_scan() + scan.file_scanner.partition_key_predicate = None + request = scan._scan_request() + assert request.partition_key_predicate == predicate.new_index(0) + assert request.explicit_partition_filter is False + assert request.partition_filter is None + requested = [] + + def authorize(columns): + requested.append(set(columns)) + return None + + scan._query_auth_fn = authorize + with patch.object(scan, '_native_plan_supported', return_value=True), \ + patch('pypaimon.execution.native.read.plan.native_plan', + return_value=expected) as native, \ + patch('pypaimon.execution.python.read.table_scan.FileScanner', + side_effect=AssertionError('Native planning initialized Python')): + plan = scan.plan() + native.assert_called_once() + assert native.call_args[1]['predicate'] == predicate + assert 'partition_predicate' not in native.call_args[1] + assert requested == ([] if native_auth else [{'id', 'pt'}]) + assert {split.partition.values[0] for split in plan.splits()} == {'a'} + assert builder.new_read().to_arrow(plan.splits()).to_pylist() == [{'id': 1}] + + +@pytest.mark.parametrize('original_partition_filter', [False, True]) +@pytest.mark.parametrize('selected_partition', [None, 'b']) +def test_raw_file_partition_selection_overrides_the_derived_filter( + table, original_partition_filter, selected_partition): + table = table.copy({'scan.native-plan.enabled': 'false', 'read.native.enabled': 'false'}) + _append(table, [{'id': 1, 'pt': 'a'}, {'id': 2, 'pt': 'b'}]) + builder = table.new_read_builder() + predicates = builder.new_predicate_builder() + predicate = predicates.and_predicates([ + predicates.greater_than('id', 0), predicates.equal('pt', 'a')]) + builder.with_filter(predicate) + if original_partition_filter: + builder.with_partition_filter(predicates.equal('pt', 'b')) + scan = builder.new_scan() + scanner = scan.file_scanner + scanner.partition_key_predicate = ( + None if selected_partition is None else predicates.equal('pt', selected_partition).new_index(0)) + expected = ['a'] if selected_partition is None else ['b'] + assert sorted(entry.partition.values[0] for entry in scanner.plan_files()) == expected + assert scan._scan_request().read_predicate == predicate + rows = builder.new_read().to_arrow(scan.plan().splits()).to_pylist() + assert rows == ([{'id': 1, 'pt': 'a'}] if selected_partition is None else []) + + native = Mock() + native.with_filter.return_value = native + native.with_partition_filter.return_value = native + raw = native.new_scan.return_value + raw.with_file_filters.return_value = raw + with patch('pypaimon.execution.native.read.builder._native_read_builder', return_value=native), \ + patch('pypaimon.execution.native.read.file_scan.from_native_file_plan', return_value=([], 1)): + assert NativeTableScan(scan).plan_files(False, False) == ([], 1) + native.with_filter.assert_called_once_with({ + 'method': 'and', 'children': [ + {'method': 'greaterThan', 'field': 'id', 'literals': [0]}, + {'method': 'equal', 'field': 'pt', 'literals': ['a']}]}) + if selected_partition is None: + native.with_partition_filter.assert_not_called() + else: + native.with_partition_filter.assert_called_once_with({ + 'method': 'equal', 'field': 'pt', 'literals': ['b']}) + raw.plan_files.assert_called_once_with() + + with patch('pypaimon.execution.native.read.plan.native_plan', return_value=Plan([], 1)) as plan: + result = NativeTableScan(scan).plan() + assert result.snapshot_id == 1 + assert plan.call_args[1]['predicate'] == predicate + if selected_partition is not None: + assert plan.call_args[1]['partition_predicate'] == scanner.partition_key_predicate + else: + assert 'partition_predicate' not in plan.call_args[1] + + +@pytest.mark.parametrize('original_partition_filter', [False, True]) +@pytest.mark.parametrize('selected_partition', [None, 'b']) +def test_file_partition_selection_is_independent_of_later_row_filters( + table, original_partition_filter, selected_partition): + table = table.copy({'scan.native-plan.enabled': 'false', 'read.native.enabled': 'false'}) + _append(table, [{'id': 1, 'pt': 'a'}, {'id': 2, 'pt': 'b'}]) + builder = table.new_read_builder() + predicates = builder.new_predicate_builder() + if original_partition_filter: + builder.with_partition_filter(predicates.equal('pt', 'b')) + scan = builder.new_scan() + scanner = scan.file_scanner + selector = None if selected_partition is None else predicates.equal('pt', selected_partition).new_index(0) + scanner.partition_key_predicate = selector + scanner.predicate = predicates.equal('pt', 'a') + assert scanner.partition_key_predicate == ( + predicates.equal('pt', 'a').new_index(0) if selector is None else selector) + assert scanner._explicit_partition_filter is (selector is not None) + expected = ['a'] if selected_partition is None else ['b'] + assert sorted(entry.partition.values[0] for entry in scanner.plan_files()) == expected + + +def test_incremental_facade_retains_level_zero_after_native_preparation_fails(tmp_path): + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + catalog.create_table('default.files', Schema.from_pyarrow_schema( + SCHEMA, primary_keys=['id'], options={'bucket': '1', 'merge-engine': 'first-row'}), False) + table = catalog.get_table('default.files') + _append(table, [{'id': 1, 'pt': 'a'}]) + selected = table.copy({'scan.native-plan.enabled': 'true', + 'incremental-between-timestamp': '0,9223372036854775807'}) + scan = selected.new_read_builder().new_scan() + assert scan.file_scanner.skip_level0 is False + with patch.object(scan, '_native_plan_supported', return_value=True), \ + patch('pypaimon.execution.native.read.plan.native_plan', return_value=None): + plan = scan.plan() + assert len(plan.splits()) == 1 + assert all(file.level == 0 for split in plan.splits() for file in split.files) + + +def test_python_file_scanner_configuration_can_change_after_first_plan(table): + _append(table, [{'id': 1, 'pt': 'a'}]) + _append(table, [{'id': 2, 'pt': 'b'}]) + scan = table.new_read_builder().new_scan() + scanner = scan.file_scanner + assert len(scanner.plan_files()) == 2 + scanner._set_predicate(table.new_read_builder().new_predicate_builder().equal('pt', 'a')) + entries = scanner.plan_files() + assert len(entries) == 1 and entries[0].partition.values == ['a'] + assert len(scanner.scan().splits()) == 1 + assert scanner.scan_with_stats()[0].snapshot_id == 2 + + +def test_raw_file_planning_preserves_all_files_before_limit_and_shard(table): + for i in range(3): + _append(table, [{'id': i, 'pt': str(i)}]) + scanner = table.new_read_builder().with_limit(1).new_scan().with_shard(0, 3).file_scanner + assert len(scanner.plan_files()) == 3 + + +def test_native_file_metadata_bridge_keeps_partition_sequence_and_path(table): + messages = _append(table, [{'id': 1, 'pt': '甲🙂'}]) + entry_message = messages[0] + payload = serialize_commit_message(CommitMessage( + entry_message.partition, entry_message.bucket, [entry_message.new_files[0]]), + table.partition_keys_fields) + entry = SimpleNamespace(kind=0, bucket=entry_message.bucket, total_buckets=7, + _to_commit_message=lambda: SimpleNamespace(serialize=lambda: payload)) + plan = SimpleNamespace(files=lambda: [entry], snapshot_id=lambda: 3) + entries, snapshot = from_native_file_plan(table, plan) + assert snapshot == 3 and len(entries) == 1 + decoded = entries[0] + assert decoded.partition.values == ['甲🙂'] and decoded.total_buckets == 7 + assert decoded.file.file_path == entry_message.new_files[0].file_path + assert decoded.file.max_sequence_number == entry_message.new_files[0].max_sequence_number + assert decoded.file.schema_id == entry_message.new_files[0].schema_id + + +@pytest.mark.parametrize('native_failure', [False, True]) +def test_raw_metadata_never_falls_back_past_restricted_authorization(table, native_failure): + table = table.copy({'scan.native-plan.enabled': str(native_failure).lower()}) + scan = table.new_read_builder().new_scan() + scan._query_auth_fn = object() + with patch.object(scan, '_native_plan_supported', return_value=True), \ + patch('pypaimon.execution.native.table._native_query_auth_supported', return_value=True), \ + patch.object(NativeTableScan, 'plan_files', return_value=None), \ + patch('pypaimon.read.table_scan.authorize', return_value=object()) as authorize, \ + patch.object(PythonTableScan, '_file_scanner', side_effect=AssertionError('Raw metadata leaked')): + with pytest.raises(TableNoPermissionException): + scan.file_scanner.plan_files() + assert authorize.call_args[0][2] is None # permission covers every column + + +@pytest.mark.native_plan +@pytest.mark.skipif(not native_runtime_available(), reason='pypaimon-rust not installed') +@pytest.mark.parametrize('policy', ['filter', 'mask', 'denied', 'unrestricted']) +def test_rest_raw_file_authorization_matches_both_backends(rest_catalog, policy): + import json + from pypaimon.api.api_response import AuthTableQueryResponse, ErrorResponse + catalog, server = rest_catalog + catalog.create_table('default.files', Schema.from_pyarrow_schema(SCHEMA), False) + table = catalog.get_table('default.files') + _append(table, [{'id': 1, 'pt': 'a'}]) + catalog.alter_table(table.identifier, [SchemaChange.set_option('query-auth.enabled', 'true')]) + table = catalog.get_table('default.files') + if policy == 'filter': + rule = {'kind': 'LEAF', 'transform': {'name': 'FIELD_REF', + 'fieldRef': {'index': 0, 'name': 'id', 'type': 'INT'}}, + 'function': 'GREATER_THAN', 'literals': [0]} + response = AuthTableQueryResponse(filter=[json.dumps(rule)]) + elif policy == 'mask': + response = AuthTableQueryResponse(column_masking={ + 'pt': json.dumps({'name': 'UPPER', 'inputs': [{'index': 1, 'name': 'pt', 'type': 'STRING'}]})}) + else: + response = AuthTableQueryResponse() + server.table_query_auth_store[table.identifier.get_full_name()] = response + with ExitStack() as stack: + if policy == 'denied': + denied = server._mock_response(ErrorResponse('TABLE', 'files', 'permission denied', 403), 403) + stack.enter_context(patch.object(server, '_table_query_auth_handle', return_value=denied)) + for native in (False, True): + selected = table.copy({'scan.native-plan.enabled': str(native).lower()}) + scanner = selected.new_read_builder().with_projection(['id']).new_scan().file_scanner + if policy == 'unrestricted': + assert len(scanner.plan_files()) == 1 + else: + with pytest.raises(Exception, match='[Pp]ermission|denied|unrestricted|403'): + scanner.plan_files() diff --git a/paimon-python/pypaimon/tests/native_full_startup_test.py b/paimon-python/pypaimon/tests/native_full_startup_test.py index 6e204453661b..960fb0795010 100644 --- a/paimon-python/pypaimon/tests/native_full_startup_test.py +++ b/paimon-python/pypaimon/tests/native_full_startup_test.py @@ -90,7 +90,10 @@ def test_full_batch_startup_reads(source, native, mode, method, filtered): with ExitStack() as stack: if native: stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Python scan'))) - stack.enter_context(patch.object(read, '_create_split_read', side_effect=AssertionError('Python read'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read') + )) plan = scan.plan() if method == 'arrow': rows = read.to_arrow(plan.splits()).to_pylist() @@ -153,7 +156,10 @@ def test_full_startup_empty_table(rest_catalog, native, mode): with ExitStack() as stack: if native: stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Python scan'))) - stack.enter_context(patch.object(read, '_create_split_read', side_effect=AssertionError('Python read'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read') + )) plan = scan.plan() assert plan.snapshot_id is None assert plan.splits() == [] @@ -277,7 +283,7 @@ def test_merge_authorized_source_failures_never_modify_target(rest_catalog, nati stack.enter_context(patch.object(server, '_table_query_auth_handle', return_value=denied)) with pytest.raises(Exception) as caught: # No value column is used by this MERGE, but a mask on that column - # still prevents plan_all_files from exposing the source table. + # still prevents plan_all from exposing the source table. builder.new_update().merge_into(source, on=['id'], when_matched=[WhenMatched.delete()]) assert not isinstance(caught.value, AssertionError) assert target.snapshot_manager().get_latest_snapshot().id == before diff --git a/paimon-python/pypaimon/tests/native_iterator_test.py b/paimon-python/pypaimon/tests/native_iterator_test.py index ca396fda3902..f7f87fedd3a0 100644 --- a/paimon-python/pypaimon/tests/native_iterator_test.py +++ b/paimon-python/pypaimon/tests/native_iterator_test.py @@ -101,7 +101,10 @@ def test_iterator_uses_native_read(tmp_path, mode, projection, limit): python.with_filter(predicate) expected = _rows(python.new_read(), splits) read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python read fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + ): actual = _rows(read, splits) assert [row for row, _ in actual] == [row for row, _ in expected] assert all(kind == RowKind.INSERT for _, kind in actual) @@ -128,7 +131,10 @@ def test_native_arrow_modes_share_limit_and_materialized_keeps_split_prefix(tmp_ if limit is not None: limited.with_limit(limit) read = limited.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python read fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + ): result = getattr(read, method_name)(splits, parallelism=parallelism) if method_name != 'to_arrow': try: @@ -196,7 +202,10 @@ def test_iterator_is_lazy_and_close_stops_native_input(tmp_path, consume): read, splits = _planned_read(tmp_path) batches = _Batches([_batch([1, 2]), RuntimeError('must not fetch next batch')]) with patch('pypaimon.execution.native.read.table_read.native_read', return_value=batches) as prepare, \ - patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): iterator = read.to_iterator(splits) prepare.assert_not_called() if consume: @@ -217,7 +226,10 @@ def test_iterator_closes_native_input_on_failure_without_replaying_python(tmp_pa if failure == 'schema' else _batch([2], ['invalid'])) batches = _Batches([_batch([1]), bad]) with patch('pypaimon.execution.native.read.table_read.native_read', return_value=batches), \ - patch.object(read, '_create_split_read', side_effect=AssertionError('Python replay')): + patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python replay') + ): iterator = read.to_iterator(splits) assert next(iterator).get_field(0) == 1 with pytest.raises((RuntimeError, TypeError, ValueError)): @@ -233,7 +245,10 @@ def test_iterator_delegates_ordered_global_limit_in_one_native_read(tmp_path): patch( 'pypaimon.execution.native.read.split_adapter.native_split_from_python', side_effect=lambda split, **kwargs: split), \ - patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): result = _rows(read, splits + splits) assert result == [((value,), RowKind.INSERT) for value in [1, 2, 3]] native.assert_called_once() @@ -287,7 +302,10 @@ def test_iterator_nested_projection_and_filter_only_inputs(tmp_path, mode, proje expected = _rows(table.copy({'read.native.enabled': 'false'}).new_read_builder() .with_projection(projection).with_filter(predicate).new_read(), splits) read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): actual = _rows(read, splits) assert actual == expected assert actual and all(len(values) == len(projection) for values, _ in actual) @@ -300,7 +318,10 @@ def test_duplicate_arrow_outputs_align_metadata_by_position(tmp_path, mode, proj builder = table.new_read_builder().with_projection(projection) splits = builder.new_scan().plan().splits() read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): for result in (read.to_arrow(splits), read.to_arrow_batch_reader(splits).read_all()): assert result.schema.names == projection assert result.column(0).to_pylist() == result.column(1 if len(projection) == 2 else 2).to_pylist() @@ -324,7 +345,10 @@ def test_iterator_schema_evolution_keeps_current_nested_types(tmp_path, mode): read = builder.new_read() expected = _rows(table.copy({'read.native.enabled': 'false'}).new_read_builder() .with_projection(['profile', 'v', 'id']).new_read(), splits) - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): assert _rows(read, splits) == expected @@ -345,7 +369,10 @@ def test_iterator_blob_accessors_survive_projection_and_close(tmp_path, descript builder = table.new_read_builder().with_projection(projection) read, splits = builder.new_read(), builder.new_scan().plan().splits() result = [] - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): iterator = read.to_iterator(splits) for row in iterator: result.append((row.get_field(2), [row.get_blob(index) for index in [0, 1] + ([3] if duplicate else [])])) @@ -369,7 +396,10 @@ def test_iterator_pk_blob_respects_payload_mode_and_limit(tmp_path, descriptors) ]) builder = table.new_read_builder().with_projection(['id', 'output']).with_limit(1) read, splits = builder.new_read(), builder.new_scan().plan().splits() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): iterator = read.to_iterator(splits) row = next(iterator) assert row.get_field(0) == 1 @@ -392,7 +422,10 @@ def test_iterator_reads_actual_incremental_changelog_kinds(tmp_path, native_writ 'read.native.enabled': 'true', 'scan.native-plan.enabled': 'true'}) builder = table.new_read_builder().with_projection(projection) read, splits = builder.new_read(), builder.new_scan().plan().splits() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): rows = _rows(read, splits) assert sorted((values, kind.value) for values, kind in rows) == sorted([ ((index, 1) if projection else (), kind.value) @@ -418,7 +451,10 @@ def test_iterator_user_row_kind_column_does_not_collide_with_transport(tmp_path, python.with_projection(projection) read, splits = builder.new_read(), builder.new_scan().plan().splits() expected = _rows(python.new_read(), splits) - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): assert _rows(read, splits) == expected @@ -432,7 +468,10 @@ def test_iterator_reads_parquet_vector_sidecars_and_keeps_typed_accessors(tmp_pa builder = table.new_read_builder().with_projection(projection) read, splits = builder.new_read(), builder.new_scan().plan().splits() result = [] - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): for row in read.to_iterator(splits): id_ = row.get_field(1) result.append(id_) @@ -471,7 +510,10 @@ def test_iterator_keeps_plan_authorization_across_splits_and_fallback(rest_catal server.table_query_auth_store[table.identifier.get_full_name()] = AuthTableQueryResponse() read = builder.new_read() if mode == 'native': - context = patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')) + context = patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ) else: failure = RuntimeError('setup failed') if mode == 'wire-fallback': @@ -501,7 +543,10 @@ def test_iterator_data_evolution_applies_partial_updates_and_deletion_vectors(tm read, splits = builder.new_read(), builder.new_scan().plan().splits() expected = _rows(table.copy({'read.native.enabled': 'false'}).new_read_builder() .with_projection(projection).new_read(), splits) - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): actual = _rows(read, splits) assert actual == expected assert {row[0]: row[1] for row, _ in actual} == ( @@ -526,8 +571,10 @@ def test_iterator_map_blob_leaf_keeps_blob_accessor(tmp_path, descriptors, dupli values = [] with ExitStack() as stack: if native: - stack.enter_context(patch.object( - read, '_create_split_read', side_effect=AssertionError('Python fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + )) for row in read.to_iterator(splits): assert len(row) == len(projection) blob = row.get_blob(0) diff --git a/paimon-python/pypaimon/tests/native_managed_blob_write_test.py b/paimon-python/pypaimon/tests/native_managed_blob_write_test.py index 8f5071530d35..a44383395484 100644 --- a/paimon-python/pypaimon/tests/native_managed_blob_write_test.py +++ b/paimon-python/pypaimon/tests/native_managed_blob_write_test.py @@ -70,7 +70,10 @@ def _read(table, predicate=None, projection=None, limit=None, descriptors=False) builder.with_limit(limit) splits = builder.new_scan().plan().splits() read = builder.new_read() - guard = patch.object(read, '_create_split_read', side_effect=AssertionError('Python read fallback')) + guard = patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + ) with guard: return read.to_arrow(splits, blob_parallelism=2).to_pylist() diff --git a/paimon-python/pypaimon/tests/native_manifest_compression_test.py b/paimon-python/pypaimon/tests/native_manifest_compression_test.py index 400a972151c9..32f6c3fa81d7 100644 --- a/paimon-python/pypaimon/tests/native_manifest_compression_test.py +++ b/paimon-python/pypaimon/tests/native_manifest_compression_test.py @@ -62,8 +62,10 @@ def _read(table, native, snapshot_id=None): plan = scan.plan() if native: assert all(getattr(split, '_native_split', None) is not None for split in plan.splits()) - stack.enter_context(patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) return builder.new_read().to_arrow(plan.splits()).sort_by('id').to_pylist() diff --git a/paimon-python/pypaimon/tests/native_map_shredding_write_test.py b/paimon-python/pypaimon/tests/native_map_shredding_write_test.py index 1ef72a9a883d..e067e67d96b0 100644 --- a/paimon-python/pypaimon/tests/native_map_shredding_write_test.py +++ b/paimon-python/pypaimon/tests/native_map_shredding_write_test.py @@ -82,7 +82,10 @@ def _read(table, native, projection=None): builder.with_projection(projection) reader = builder.new_read() splits = builder.new_scan().plan().splits() - guard = patch.object(reader, '_create_split_read', side_effect=AssertionError('Python reader fallback')) + guard = patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python reader fallback') + ) with guard if native else nullcontext(): return sorted(reader.to_arrow(splits).to_pylist(), key=lambda row: row['id']) diff --git a/paimon-python/pypaimon/tests/native_merge_read_test.py b/paimon-python/pypaimon/tests/native_merge_read_test.py index 7b86a7a8d140..e5170c9ad34f 100644 --- a/paimon-python/pypaimon/tests/native_merge_read_test.py +++ b/paimon-python/pypaimon/tests/native_merge_read_test.py @@ -71,7 +71,10 @@ def test_native_merge_reads_configurations_outside_python_subset(tmp_path, optio builder = table.new_read_builder() splits = builder.new_scan().plan().splits() read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): if streaming: batches = read.to_arrow_batch_reader(splits) try: @@ -114,7 +117,9 @@ def test_unsupported_python_merge_fails_before_raw_split_dispatch( else: failure = patch('pypaimon.execution.native.read.table_read._prepare_native_read', side_effect=RuntimeError('native setup failed')) - with failure, patch.object(read, '_create_split_read') as dispatch: + with failure, patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read' + ) as dispatch: with pytest.raises(NotImplementedError, match='sequence-group'): if api == 'arrow': read.to_arrow(splits, parallelism=parallelism) diff --git a/paimon-python/pypaimon/tests/native_merge_source_metadata_test.py b/paimon-python/pypaimon/tests/native_merge_source_metadata_test.py index 977cf5e555d5..fc15dfa8349d 100644 --- a/paimon-python/pypaimon/tests/native_merge_source_metadata_test.py +++ b/paimon-python/pypaimon/tests/native_merge_source_metadata_test.py @@ -63,7 +63,7 @@ def _new_table(catalog, name, evolution=True): def _rows(table, metadata=()): builder = table.new_read_builder().with_projection(list(dict.fromkeys(table.field_names + list(metadata)))) - return builder.new_read().to_arrow(builder.new_scan().plan_all_files().splits()).to_pylist() + return builder.new_read().to_arrow(builder.new_scan().plan_all().splits()).to_pylist() @pytest.mark.parametrize('native', _ENGINES, ids=['python', 'native']) diff --git a/paimon-python/pypaimon/tests/native_nested_predicate_test.py b/paimon-python/pypaimon/tests/native_nested_predicate_test.py index 2b30d8c1df23..52ddda8e0945 100644 --- a/paimon-python/pypaimon/tests/native_nested_predicate_test.py +++ b/paimon-python/pypaimon/tests/native_nested_predicate_test.py @@ -120,8 +120,10 @@ def _read(table, native, stream, case, projection, limit, compound): read = builder.new_read() with ExitStack() as stack: if native: - stack.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) result = read.to_arrow_batch_reader(splits).read_all() return result.sort_by('id').to_pylist() @@ -232,8 +234,10 @@ def test_nested_predicate_alias_collisions( if native: stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Python scan fallback'))) - stack.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) rows = read.to_arrow_batch_reader(scan.plan().splits()).read_all().to_pylist() expected = {11} if nested_filter else {12, 13} if compound == 'and': @@ -308,8 +312,10 @@ def test_nested_projection_keeps_physical_filter_only_inputs( if native: stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Python scan fallback'))) - stack.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) splits = scan.plan().splits() for result in (read.to_arrow(splits), read.to_arrow_batch_reader(splits).read_all()): assert result.schema.names == schema_names @@ -349,8 +355,10 @@ def test_nested_projection_keeps_synthesized_row_id_filter_input(tmp_path, nativ if native: stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Python scan fallback'))) - stack.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) splits = scan.plan().splits() for result in (read.to_arrow(splits), read.to_arrow_batch_reader(splits).read_all()): assert result.to_pylist() == [{'_ROW_ID': 100, 'id': 1}] diff --git a/paimon-python/pypaimon/tests/native_nested_update_test.py b/paimon-python/pypaimon/tests/native_nested_update_test.py index 042c08acacc1..9ae2aaf385ba 100644 --- a/paimon-python/pypaimon/tests/native_nested_update_test.py +++ b/paimon-python/pypaimon/tests/native_nested_update_test.py @@ -93,8 +93,10 @@ def _check_rows(table, values): read = builder.new_read() with ExitStack() as stack: if native: - stack.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) actual = read.to_arrow(builder.new_scan().plan().splits()).sort_by('id') assert actual['value'].to_pylist() == values assert actual['untouched'].to_pylist() == list(range(10, 10 + len(values))) diff --git a/paimon-python/pypaimon/tests/native_nullable_primary_key_test.py b/paimon-python/pypaimon/tests/native_nullable_primary_key_test.py index 12617ea28fa0..14eb81282885 100644 --- a/paimon-python/pypaimon/tests/native_nullable_primary_key_test.py +++ b/paimon-python/pypaimon/tests/native_nullable_primary_key_test.py @@ -91,8 +91,11 @@ def _read(table, native, null_field=None, all_files=False): if native: if not all_files: stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Python scan'))) - stack.enter_context(patch.object(read, '_create_split_read', side_effect=AssertionError('Python read'))) - splits = (scan.plan_all_files() if all_files else scan.plan()).splits() + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read') + )) + splits = (scan.plan_all() if all_files else scan.plan()).splits() rows = read.to_arrow(splits).to_pylist() # All values in these fixtures are non-negative except the seeded MERGE target. return sorted((tuple(row[name] for name in _SCHEMA.names) for row in rows), @@ -204,7 +207,10 @@ def test_nullable_scalar_key_distinguishes_null_from_default_value(rest_catalog, with ExitStack() as stack: if native: stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Python scan'))) - stack.enter_context(patch.object(read, '_create_split_read', side_effect=AssertionError('Python read'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read') + )) rows = read.to_arrow(scan.plan().splits()).to_pylist() assert sorted(rows, key=lambda row: row['id'] is not None) == [ {'id': None, 'value': 30}, {'id': zero, 'value': 200}] diff --git a/paimon-python/pypaimon/tests/native_partition_blob_read_test.py b/paimon-python/pypaimon/tests/native_partition_blob_read_test.py index 9943947f8bac..ffc28b7adf6f 100644 --- a/paimon-python/pypaimon/tests/native_partition_blob_read_test.py +++ b/paimon-python/pypaimon/tests/native_partition_blob_read_test.py @@ -82,8 +82,10 @@ def _read(table, splits, native, projection, predicate=None, limit=None): reader = builder.new_read() with ExitStack() as guard: if native: - guard.enter_context(patch.object( - reader, '_create_split_read', side_effect=AssertionError('Python partition-BLOB read fallback'))) + guard.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python partition-BLOB read fallback') + )) return reader.to_arrow(splits, parallelism=1) diff --git a/paimon-python/pypaimon/tests/native_partition_filter_test.py b/paimon-python/pypaimon/tests/native_partition_filter_test.py index c7e82e26505f..3fb00ca31b34 100644 --- a/paimon-python/pypaimon/tests/native_partition_filter_test.py +++ b/paimon-python/pypaimon/tests/native_partition_filter_test.py @@ -91,8 +91,10 @@ def _read(table, native, selector, predicate=None, limit=None, selector_first=Fa if native: stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Native plan fell back'))) - stack.enter_context(patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Native read fell back'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native read fell back') + )) plan = scan.plan() rows = builder.new_read().to_arrow(plan.splits()).to_pylist() if native: @@ -146,6 +148,32 @@ def test_replacing_and_clearing_partition_selector(source): assert sorted(row['id'] for row in builder.new_read().to_arrow(plan.splits()).to_pylist()) == [1, 3] +def test_clearing_file_selector_restores_derived_pruning_in_both_backends(source): + table, _, _ = source + predicates = PredicateBuilder(table.fields) + for native in (False, True): + selected = table.copy({'scan.native-plan.enabled': str(native).lower(), + 'read.native.enabled': str(native).lower()}) + builder = selected.new_read_builder().with_projection(['id']) + builder.with_filter(predicates.equal('p', 'a')) + builder.with_partition_filter(predicates.equal('region', 'west')) + scan = builder.new_scan() + scanner = scan.file_scanner + # None removes the explicit selection; p=a still prunes partitions. + scanner.partition_key_predicate = None + with ExitStack() as stack: + if native: + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_scan.FileScanner', + side_effect=AssertionError('Native planning initialized Python'))) + plan = scan.plan() + assert {tuple(split.partition.values) for split in plan.splits()} == { + ('east', 'a'), ('west', 'a')} + assert sorted(row['id'] for row in builder.new_read().to_arrow(plan.splits()).to_pylist()) == [1, 3] + assert {tuple(entry.partition.values) for entry in scanner.plan_files()} == { + ('east', 'a'), ('west', 'a')} + + def test_unmasked_partition_selector_with_a_masked_row_predicate(source): table, server, auth = source if not auth: diff --git a/paimon-python/pypaimon/tests/native_pk_blob_limit_test.py b/paimon-python/pypaimon/tests/native_pk_blob_limit_test.py index 523d1f4eeb46..53b25265e523 100644 --- a/paimon-python/pypaimon/tests/native_pk_blob_limit_test.py +++ b/paimon-python/pypaimon/tests/native_pk_blob_limit_test.py @@ -88,7 +88,10 @@ def _read(table, projection, predicate=None, limit=None, parallelism=1, streamin builder.with_limit(limit) splits = builder.new_scan().plan().splits() if splits is None else splits reader = builder.new_read() - with patch.object(reader, '_create_split_read', side_effect=AssertionError('Python read fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + ): if streaming: with reader.to_arrow_batch_reader(splits, parallelism=parallelism, blob_parallelism=2) as batches: return pa.Table.from_batches(list(batches), schema=batches.schema).to_pylist() @@ -249,15 +252,18 @@ def test_streaming_pk_blob_row_kinds_resolve_only_selected_payload(tmp_path, pay dict(id=2, tag=1, payload=missing, output=missing)]) builder = table.new_stream_read_builder().with_projection(['id', 'payload']) scan = builder.new_streaming_scan() - if table.options.native_plan_enabled(): + try: scan.restore(table.snapshot_manager().get_latest_snapshot().id) splits = scan.plan().splits() - else: - splits = scan._create_delta_plan(table.snapshot_manager().get_latest_snapshot()).splits() + finally: + scan.close() assert splits and all(split.is_streaming for split in splits) predicate = builder.new_predicate_builder().equal('payload', b'selected') if payload_filter else None reader = TableRead(table, predicate, builder.read_type(), include_row_kind=True, limit=1) - with patch.object(reader, '_create_split_read', side_effect=AssertionError('Python read fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + ): assert reader.to_arrow(splits).to_pylist() == [ {'_row_kind': '+I', 'id': 1, 'payload': b'selected'}] @@ -273,11 +279,11 @@ def test_pk_row_kind_limit_is_global_across_snapshot_and_delta_splits(tmp_path): splits = table.new_read_builder().new_scan().plan().splits() _write(table, [dict(id=2, tag=1, payload=missing, output=missing, pt='b')]) scan = table.new_stream_read_builder().new_streaming_scan() - if table.options.native_plan_enabled(): + try: scan.restore(table.snapshot_manager().get_latest_snapshot().id) splits += scan.plan().splits() - else: - splits += scan._create_delta_plan(table.snapshot_manager().get_latest_snapshot()).splits() + finally: + scan.close() assert len(splits) == 2 and not splits[0].is_streaming and splits[1].is_streaming # Exercise the Rust quota directly: TableRead's outer truncation must not # conceal extra rows or payload reads in the core row-kind path. diff --git a/paimon-python/pypaimon/tests/native_plan_capabilities_test.py b/paimon-python/pypaimon/tests/native_plan_capabilities_test.py index 6b2837fab38c..54616c644366 100644 --- a/paimon-python/pypaimon/tests/native_plan_capabilities_test.py +++ b/paimon-python/pypaimon/tests/native_plan_capabilities_test.py @@ -137,7 +137,7 @@ def _assert_parity(self, table, expected_rows, snapshot_id, getattr(split, '_native_split', None) is not None for split in plan.splits())) read_guard = patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'native capability read fell back to Python')) else: diff --git a/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py b/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py index 88e1d7c5956f..967bd0ebda94 100644 --- a/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py +++ b/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py @@ -160,7 +160,7 @@ def _chunks(table, seed, chunk_size=3, shard=None, predicate=None, projection=No if native_read_enabled: assert native_runtime_available() with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')): rows = read_builder.new_read().to_arrow([split]).to_pylist() else: diff --git a/paimon-python/pypaimon/tests/native_plan_distribution_test.py b/paimon-python/pypaimon/tests/native_plan_distribution_test.py index 06078bfa4cf2..28582efb4270 100644 --- a/paimon-python/pypaimon/tests/native_plan_distribution_test.py +++ b/paimon-python/pypaimon/tests/native_plan_distribution_test.py @@ -115,8 +115,9 @@ def _read(self, table, native, shard=None, slice_=None, row_ranges=None, stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError( 'distributed native plan fell back to Python'))) if table.is_primary_key_table and not table.options.global_index_enabled(): - stack.enter_context(patch.object( - scan.file_scanner, '_apply_push_down_limit', side_effect=AssertionError( + stack.enter_context(patch( + 'pypaimon.execution.python.read.scanner.file_scanner.FileScanner._apply_push_down_limit', + side_effect=AssertionError( 'PK shard LIMIT was applied in Python'))) plan = scan.plan() if native: @@ -124,7 +125,7 @@ def _read(self, table, native, shard=None, slice_=None, row_ranges=None, getattr(split, '_native_split', None) is not None for split in plan.splits())) read_guard = patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'distributed native read fell back to Python')) else: diff --git a/paimon-python/pypaimon/tests/native_plan_dynamic_bucket_test.py b/paimon-python/pypaimon/tests/native_plan_dynamic_bucket_test.py index 5740945f15b1..f09d8a6d3c9c 100644 --- a/paimon-python/pypaimon/tests/native_plan_dynamic_bucket_test.py +++ b/paimon-python/pypaimon/tests/native_plan_dynamic_bucket_test.py @@ -68,7 +68,7 @@ def read(table, native, predicate=None, shard=None, limit=None): assert all(getattr(split, '_native_split', None) is not None for split in plan.splits()) read_guard = patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('dynamic-bucket native read fell back')) else: read_guard = ExitStack() @@ -148,7 +148,7 @@ def read(table, native, predicate=None, shard=None, limit=None): read = builder.new_read() read.include_row_kind = True with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'dynamic incremental native read fell back')): rows = read.to_arrow(plan.splits()).to_pylist() diff --git a/paimon-python/pypaimon/tests/native_plan_expanded_test.py b/paimon-python/pypaimon/tests/native_plan_expanded_test.py index 8df9ae5fc30f..bb3d7b174c4e 100644 --- a/paimon-python/pypaimon/tests/native_plan_expanded_test.py +++ b/paimon-python/pypaimon/tests/native_plan_expanded_test.py @@ -86,7 +86,7 @@ def read(table, native, predicate=None, shard=None, limit=None, result=None, ran plan = scan.plan() if native: read_guard = patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('expanded native read fell back')) else: read_guard = ExitStack() diff --git a/paimon-python/pypaimon/tests/native_plan_incremental_test.py b/paimon-python/pypaimon/tests/native_plan_incremental_test.py index 385e6ba2c467..8a65aa34d56c 100644 --- a/paimon-python/pypaimon/tests/native_plan_incremental_test.py +++ b/paimon-python/pypaimon/tests/native_plan_incremental_test.py @@ -113,10 +113,9 @@ def _read(table, native, window, predicate=None, limit=None, shard=None, scan.with_row_ranges(row_ranges) with ExitStack() as stack: if native: - for method in ('scan', 'scan_with_stats'): - stack.enter_context(patch.object( - scan.file_scanner, method, side_effect=AssertionError( - 'incremental native plan fell back to Python'))) + stack.enter_context(patch.object( + scan, '_create_file_scanner', side_effect=AssertionError( + 'incremental native plan fell back to Python'))) plan = scan.scan_with_stats()[0] if with_stats else scan.plan() with ExitStack() as stack: if native: @@ -124,7 +123,7 @@ def _read(table, native, window, predicate=None, limit=None, shard=None, getattr(split, '_native_split', None) is not None for split in plan.splits()) stack.enter_context(patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'incremental native read fell back to Python'))) result = builder.new_read().to_arrow( @@ -333,7 +332,7 @@ def test_incremental_does_not_evaluate_endpoint_global_indexes(catalog, native): def test_incremental_rejects_manifest_delete_before_reconciliation(catalog, native): - from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan + from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan table = _table(catalog, 'invalid_delta', True) _write(table, 100, [{'k': 1, 'v': 'old'}]) _write(table, 200, [{'k': 1, 'v': 'new'}], overwrite=True) @@ -353,7 +352,7 @@ def test_incremental_rejects_manifest_delete_before_reconciliation(catalog, nati def test_incremental_reader_preserves_all_physical_row_kinds(catalog, native): import pyarrow.parquet as pq - from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan + from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan table = _table(catalog, 'row_kinds', True, {'bucket': '1'}) scan = AsyncStreamingTableScan(table, prefetch_enabled=False) expected = [] @@ -387,7 +386,7 @@ def test_incremental_reader_preserves_all_physical_row_kinds(catalog, native): read = builder.new_read() read.include_row_kind = True with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('native row-kind read fell back')): arrow = read.to_arrow(plan.splits(), parallelism=2) assert sorted(zip( @@ -444,7 +443,7 @@ def test_stream_read_builder_combines_native_nested_projection_and_row_kind( scan.restore(snapshot.id) plan = scan.plan() with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('streaming nested native read fell back')): rows = builder.new_read().to_arrow(plan.splits()).to_pylist() @@ -502,7 +501,7 @@ async def first_two_frames(): for plan in plans for split in plan.splits() for file in split.files) with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'streaming changelog native read fell back to Python')): rows = [ @@ -579,7 +578,7 @@ def test_streaming_reader_honors_explicit_split_deletion_vector(catalog, native, with ExitStack() as stack: if native: stack.enter_context(patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('explicit DV native read fell back'))) result = read_table.new_read_builder().new_read().to_arrow([dv_split]).to_pylist() assert result == [{'k': 1, 'v': '1'}, {'k': 3, 'v': '3'}] @@ -597,7 +596,7 @@ def test_reverse_window_is_rejected_when_snapshots_exist(catalog): table = _table(catalog, 'reverse_window') _write(table, 100, [{'k': 1, 'v': 'base'}]) with pytest.raises(ValueError, match='Ending timestamp'): - table.copy({'incremental-between-timestamp': '200,100'}).new_read_builder().new_scan() + table.copy({'incremental-between-timestamp': '200,100'}).new_read_builder().new_scan().plan() def test_equal_window_is_empty_for_populated_tables(native, history): @@ -606,6 +605,24 @@ def test_equal_window_is_empty_for_populated_tables(native, history): assert plan.snapshot_id is None +def test_timestamp_selection_is_deferred_until_planning(native, catalog): + table = _table(catalog, 'deferred_window') + builder = table.copy({ + 'scan.native-plan.enabled': str(native).lower(), + 'read.native.enabled': str(native).lower(), + 'incremental-between-timestamp': '0,200', + }).new_read_builder() + scan = builder.new_scan() + _write(table, 100, [{'k': 1, 'v': 'selected'}]) + with ExitStack() as stack: + if native: + stack.enter_context(patch.object( + scan, '_create_file_scanner', side_effect=AssertionError('Python scanner ran'))) + plan = scan.plan() + assert plan.snapshot_id == 1 + assert builder.new_read().to_arrow(plan.splits()).to_pylist() == [{'k': 1, 'v': 'selected'}] + + def test_timestamp_auto_reads_physical_changelog(native, catalog): table = _table(catalog, 'batch_changelog', True, {'bucket': '1', 'changelog-producer': 'input'}) _write(table, 100, [{'k': 1, 'v': 'old'}]) diff --git a/paimon-python/pypaimon/tests/native_plan_integration_test.py b/paimon-python/pypaimon/tests/native_plan_integration_test.py index 519af2fbd346..9fc818cace32 100644 --- a/paimon-python/pypaimon/tests/native_plan_integration_test.py +++ b/paimon-python/pypaimon/tests/native_plan_integration_test.py @@ -104,7 +104,7 @@ def _assert_matches(self, name, expect_native=True): @staticmethod def _native_rows(builder, plan): with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')): return builder.new_read().to_arrow(plan.splits()).to_pylist() @@ -206,7 +206,7 @@ def test_native_read_bypasses_python_split_reader(self): splits = [pickle.loads(pickle.dumps(split)) for split in plan.splits()] with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')): read = builder.new_read() rows = read.to_arrow(splits).to_pylist() @@ -244,7 +244,7 @@ def test_native_read_bridges_python_planned_splits(self): wraps=native_split_from_python) as bridge, patch( 'pypaimon.execution.native.read.table_read.native_read', wraps=native_read) as rust_read, patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')): rows = builder.new_read().to_arrow(plan.splits()).to_pylist() @@ -272,7 +272,7 @@ def test_native_read_bridges_python_indexed_data_evolution_split(self): plan = scan.plan() with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')), patch( 'pypaimon.execution.native.read.table_read.native_read', wraps=native_read) as rust_read: @@ -358,7 +358,7 @@ def test_native_read_uses_effective_split_parallelism(self): with patch('pypaimon.execution.native.read.table_read._prepare_native_read', wraps=_prepare_native_read) as prepare, \ patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')): rows = builder.new_read().to_arrow(plan.splits()).to_pylist() @@ -373,7 +373,7 @@ def test_native_read_uses_effective_split_parallelism(self): with patch('pypaimon.execution.native.read.table_read._prepare_native_read', wraps=_prepare_native_read) as prepare, \ patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')): streamed = builder.new_read().to_arrow_batch_reader( plan.splits()).read_all().to_pylist() @@ -400,7 +400,7 @@ def test_native_read_primary_key_matches_python(self): builder = table.new_read_builder() plan = builder.new_scan().plan() with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')): native_read_instance = builder.new_read() native = sorted( @@ -509,7 +509,7 @@ def test_native_read_nested_rows_and_map_keys_across_formats(self): projection) plan = builder.new_scan().plan() with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'nested native read fell back to Python')): actual = builder.new_read().to_arrow( @@ -554,7 +554,7 @@ def test_native_read_nested_projection_after_primary_key_merge(self): ['id', 'payload.score']) plan = builder.new_scan().plan() with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('nested PK native read fell back')): actual = sorted( builder.new_read().to_arrow(plan.splits()).to_pylist(), @@ -605,7 +605,7 @@ def test_native_nested_projection_after_partial_update_merge(self): ['id', 'payload.score']) plan = builder.new_scan().plan() with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'nested partial-update native read fell back')): actual = sorted( @@ -681,7 +681,7 @@ def write(table, schema, rows): builder.with_filter(predicate) plan = builder.new_scan().plan() with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'evolved nested native read fell back to Python')): actual = sorted( @@ -700,7 +700,7 @@ def write(table, schema, rows): ['payload.score', 'id']).with_limit(2) limited_plan = limited.new_scan().plan() with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'limited nested native read fell back to Python')): limited_rows = limited.new_read().to_arrow( @@ -930,7 +930,7 @@ def test_native_read_data_evolution_blob_parallelism(self): )) with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')): table_result = builder.new_read().to_arrow( plan.splits(), parallelism=1, blob_parallelism=3) @@ -986,7 +986,7 @@ def test_native_read_data_evolution_nested_blobs(self): )) with patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')): result = builder.new_read().to_arrow( plan.splits(), parallelism=1, blob_parallelism=3) @@ -1030,7 +1030,7 @@ def test_native_read_pruning_limit_defers_blob_payload_io(self): with patch( 'pypaimon.execution.native.read.table_read.native_read', wraps=native_read) as native, patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader used')): result = builder.new_read().to_arrow( plan.splits(), parallelism=1) @@ -1148,7 +1148,7 @@ def test_native_read_dynamic_blob_as_descriptor(self): splits = builder.new_scan().plan().splits() with patch('pypaimon.execution.native.read.table_read.native_read', wraps=native_read) as native, patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader used')): result = builder.new_read().to_arrow(splits) native.assert_called_once() @@ -1162,7 +1162,7 @@ def test_native_read_dynamic_blob_as_descriptor(self): 'blob-as-descriptor': 'true', }).new_read_builder() splits = builder.new_scan().plan().splits() - with patch('pypaimon.read.table_read.TableRead._create_split_read', + with patch('pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader used')): self.assertEqual(builder.new_read().to_arrow(splits).to_pydict(), {'id': [1], 'payload': [descriptor]}) @@ -1213,7 +1213,7 @@ def test_native_read_pruning_limit_defers_descriptor_blob_payload_io(self): with patch( 'pypaimon.execution.native.read.table_read.native_read', wraps=native_read) as native, patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader used')): result = builder.new_read().to_arrow(plan.splits()) @@ -1256,7 +1256,7 @@ def test_native_read_limit_filters_before_descriptor_payload_io(self): for split in plan.splits())) with patch('pypaimon.execution.native.read.table_read.native_read', wraps=native_read) as native: - with patch('pypaimon.read.table_read.TableRead._create_split_read', + with patch('pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader used')): result = builder.new_read().to_arrow(plan.splits()) @@ -1298,7 +1298,7 @@ def test_native_read_supports_precision_zero_timestamps(self): with patch( 'pypaimon.execution.native.read.table_read.native_read', wraps=native_read) as native, patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('Python reader was used')): result = builder.new_read().to_arrow(plan.splits()) diff --git a/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py b/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py index afc5dfe5f901..93227d3a82da 100644 --- a/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py +++ b/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py @@ -106,7 +106,7 @@ def test_clustered_materialized_dv_files_use_native_raw_splits(tmp_path, engine, assert all(getattr(split, '_native_split', None) is not None for split in plan.splits()) read_guard = patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'materialized PK native read fell back')) else: @@ -204,7 +204,7 @@ def test_first_row_level_zero_merges_before_filtering(tmp_path, target_size, com assert all(getattr(split, '_native_split', None) is not None for split in plan.splits()) read_guard = patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError( 'first-row native read fell back')) else: @@ -283,7 +283,7 @@ def test_pk_merge_engines_read_deletion_vectors_natively(tmp_path, engine, merge plan = scan.plan() assert all(getattr(split, '_native_split', None) is not None for split in plan.splits()) - guard = patch('pypaimon.read.table_read.TableRead._create_split_read', + guard = patch('pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('native read fell back')) else: plan = scan.plan() @@ -332,8 +332,9 @@ def test_first_row_sorted_materialized_and_l0_merge_natively(tmp_path, batch_siz if native: guards.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Native plan fell back'))) - guards.enter_context(patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Native read fell back'))) + guards.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native read fell back'))) plan = scan.plan() assert any(not split.raw_convertible for split in plan.splits()) rows = builder.new_read().to_arrow(plan.splits()).sort_by([('id', 'ascending')]).to_pylist() diff --git a/paimon-python/pypaimon/tests/native_plan_resolved_schema_test.py b/paimon-python/pypaimon/tests/native_plan_resolved_schema_test.py index 91bce088e94f..6bd3ca00ced9 100644 --- a/paimon-python/pypaimon/tests/native_plan_resolved_schema_test.py +++ b/paimon-python/pypaimon/tests/native_plan_resolved_schema_test.py @@ -105,7 +105,7 @@ def _read(table, native, predicate=None, projection=None): assert all(getattr(split, '_native_split', None) is not None for split in plan.splits()) read_guard = patch( - 'pypaimon.read.table_read.TableRead._create_split_read', + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('resolved-schema native read fell back')) else: read_guard = ExitStack() diff --git a/paimon-python/pypaimon/tests/native_plan_rest_test.py b/paimon-python/pypaimon/tests/native_plan_rest_test.py index cad8cf22f668..6ebb049042bb 100644 --- a/paimon-python/pypaimon/tests/native_plan_rest_test.py +++ b/paimon-python/pypaimon/tests/native_plan_rest_test.py @@ -124,7 +124,7 @@ def test_dynamic_branch_uses_native_catalog(rest_source, rest_catalog, from_tag) assert _resolved_rest_table_response(branch) is None plan = native_plan(branch) assert plan.snapshot_id == (1 if from_tag else None) - with patch('pypaimon.read.table_read.TableRead._create_split_read', + with patch('pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('native read fell back')): rows = branch.new_read_builder().new_read().to_arrow(plan.splits()).to_pylist() assert rows == ([{'id': 1, 'value': 'old'}] if from_tag else []) @@ -270,7 +270,7 @@ def test_rest_blob_view_limit_filters_before_resolving_unselected_view(rest_cata plan = scan.plan() assert all(getattr(split, '_native_split', None) is not None for split in plan.splits()) - with patch('pypaimon.read.table_read.TableRead._create_split_read', + with patch('pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('native view read fell back')): assert builder.new_read().to_arrow(plan.splits()).to_pylist() == [ {'id': 11, 'payload': b'selected'}] @@ -433,7 +433,7 @@ def test_rest_blob_view_dependency_read_via(rest_blob_views, native, nested): guard, requests = _require_blob_view_read_via(server, root.identifier) with guard, ExitStack() as stack: if native: - stack.enter_context(patch('pypaimon.read.table_read.TableRead._create_split_read', + stack.enter_context(patch('pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('native Blob view read fell back'))) builder = root.new_read_builder() rows = builder.new_read().to_arrow(builder.new_scan().plan().splits()).sort_by('id').to_pylist() @@ -582,8 +582,9 @@ def token(method, identifier): guard, dependency_requests = _require_blob_view_read_via(rest_server, view.identifier) with guard, ExitStack() as stack: if native: - stack.enter_context(patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('token view read fell back'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('token view read fell back'))) builder = view.new_read_builder() result = builder.new_read().to_arrow(builder.new_scan().plan().splits()).sort_by('id') assert result.to_pylist() == [dict(id=0, payload=b'actual'), dict(id=1, payload=None)] @@ -635,7 +636,7 @@ def test_native_rest_blob_views_group_upstream_fields_by_id(rest_catalog): # Drop an ordinary column so BLOB field IDs differ from their positions. catalog.alter_table('default.upstream', [SchemaChange.drop_column('id')]) guard, requests = _require_blob_view_read_via(server, view.identifier) - with guard, patch('pypaimon.read.table_read.TableRead._create_split_read', + with guard, patch('pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('multi-field view read fell back')): builder = view.new_read_builder() rows = builder.new_read().to_arrow(builder.new_scan().plan().splits()).sort_by('id').to_pylist() @@ -674,7 +675,7 @@ def test_rest_blob_view_keeps_branch_and_explicit_outer_context(rest_blob_views, guard, requests = _require_blob_view_read_via(server, expected) with guard, ExitStack() as stack: if native: - stack.enter_context(patch('pypaimon.read.table_read.TableRead._create_split_read', + stack.enter_context(patch('pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('branch view read fell back'))) builder = view.new_read_builder() result = builder.new_read().to_arrow(builder.new_scan().plan().splits()).sort_by('id') @@ -695,7 +696,7 @@ def deny_upstream(method, path, parameters, data, headers): return route(method, path, parameters, data, headers) with patch.object(server, '_route_request', side_effect=deny_upstream), \ - patch('pypaimon.read.table_read.TableRead._create_split_read', + patch('pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', side_effect=AssertionError('denied dependency fell back')): builder = root.new_read_builder() with pytest.raises(Exception, match='upstream denied'): diff --git a/paimon-python/pypaimon/tests/native_plan_test.py b/paimon-python/pypaimon/tests/native_plan_test.py index 0a7c6ad436b4..7c2636b22284 100644 --- a/paimon-python/pypaimon/tests/native_plan_test.py +++ b/paimon-python/pypaimon/tests/native_plan_test.py @@ -86,6 +86,7 @@ def _scan(native_enabled, file_scanner): file_scanner.is_streaming = False file_scanner.skip_level0 = False file_scanner.only_read_real_buckets = False # not postpone bucket + file_scanner._explicit_partition_filter = False scan.file_scanner = file_scanner scan.predicate = None scan.partition_predicate = None @@ -1341,16 +1342,15 @@ def test_deletion_vectors_use_native_plan_for_bucket_layouts(self): native.assert_called_once() fs.scan.assert_not_called() - def test_incremental_range_is_forwarded_without_timestamp_reinterpretation(self): + def test_incremental_timestamp_options_are_resolved_in_core(self): fs = Mock(partition_key_predicate=None) scan = _scan(True, fs) scan.table.options.options = Options({'incremental-between-timestamp': '100,200'}) scan.table._applied_dynamic_options = {'incremental-between-timestamp': '100,200'} - scan._incremental_snapshot_range = (2, 4) with patch('pypaimon.execution.native.read.plan.native_plan', return_value=Plan([], 4)) as native: self.assertEqual(scan.plan().snapshot_id, 4) - self.assertEqual(native.call_args[1]['incremental_range'], (2, 4)) + self.assertNotIn('incremental_range', native.call_args[1]) fs.scan.assert_not_called() def test_native_plan_forwards_explicit_incremental_mode(self): @@ -1375,16 +1375,15 @@ def test_native_plan_forwards_explicit_incremental_mode(self): with self.assertRaisesRegex(ValueError, 'incremental_range'): native_plan(table, incremental_mode='changelog') - def test_incremental_window_outside_snapshots_is_terminal_empty(self): + def test_empty_incremental_window_still_runs_core_planning(self): fs = Mock(partition_key_predicate=None) scan = _scan(True, fs) scan.table.options.options = Options({'incremental-between-timestamp': '100,200'}) - scan._incremental_snapshot_range = None - with patch('pypaimon.execution.native.read.plan.native_plan') as native: + with patch('pypaimon.execution.native.read.plan.native_plan', return_value=Plan([])) as native: plan = scan.plan() self.assertEqual(plan.splits(), []) self.assertIsNone(plan.snapshot_id) - native.assert_not_called() + native.assert_called_once() fs.scan.assert_not_called() def test_primary_key_shard_and_limit_are_planned_in_rust(self): diff --git a/paimon-python/pypaimon/tests/native_postpone_fixed_write_test.py b/paimon-python/pypaimon/tests/native_postpone_fixed_write_test.py index 4eb3c8171b00..6aa6e5c92ae0 100644 --- a/paimon-python/pypaimon/tests/native_postpone_fixed_write_test.py +++ b/paimon-python/pypaimon/tests/native_postpone_fixed_write_test.py @@ -22,6 +22,7 @@ import pyarrow as pa import pytest +from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon import CatalogFactory, Schema from pypaimon.table.row.generic_row import GenericRow from pypaimon.write.write_dispatch import NativePostponeFixedBucketTableWrite @@ -456,7 +457,7 @@ def test_pending_and_real_buckets_coexist_without_rewriting_history( assert table.file_io.exists(file.file_path) # Ordinary reads expose real buckets; committed pending rows await assignment. assert _rows(table) == [dict(id=1, p='a', v=10)] - scanner = table.new_read_builder().new_scan().file_scanner + scanner = PythonTableScan(table.new_read_builder().new_scan())._file_scanner() manifests, _ = scanner.manifest_scanner() entries = scanner.with_all_buckets().read_manifest_entries(manifests) assert {entry.bucket for entry in entries} == {-2, fixed_messages[0].bucket} diff --git a/paimon-python/pypaimon/tests/native_precomputed_bucket_test.py b/paimon-python/pypaimon/tests/native_precomputed_bucket_test.py index 5c327b37ee57..0ccf77a953aa 100644 --- a/paimon-python/pypaimon/tests/native_precomputed_bucket_test.py +++ b/paimon-python/pypaimon/tests/native_precomputed_bucket_test.py @@ -399,7 +399,7 @@ def test_empty_restore_does_not_apply_select_policy_to_python_writes(tmp_path, o authorize.assert_not_called() # Only the explicit whole-table read requires an unrestricted grant. with pytest.raises(TableNoPermissionException): - table.new_read_builder().new_scan().plan_all_files() + table.new_read_builder().new_scan().plan_all() builder.new_commit().commit(messages) finally: writer.close() diff --git a/paimon-python/pypaimon/tests/native_preplanned_row_id_read_test.py b/paimon-python/pypaimon/tests/native_preplanned_row_id_read_test.py index d5844fc19d6d..c1a072223121 100644 --- a/paimon-python/pypaimon/tests/native_preplanned_row_id_read_test.py +++ b/paimon-python/pypaimon/tests/native_preplanned_row_id_read_test.py @@ -84,8 +84,10 @@ def _read(table, splits, predicate, native, limit=None): read = builder.new_read() with ExitStack() as guards: if native: - guards.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + guards.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) data = read.to_arrow(splits, parallelism=1) assert data.column_names == ['id'] return sorted(data['id'].to_pylist()) diff --git a/paimon-python/pypaimon/tests/native_query_auth_test.py b/paimon-python/pypaimon/tests/native_query_auth_test.py index 7d8f33ff90cd..5d2eea692052 100644 --- a/paimon-python/pypaimon/tests/native_query_auth_test.py +++ b/paimon-python/pypaimon/tests/native_query_auth_test.py @@ -87,8 +87,10 @@ def _read(table, native, projection=None, predicate=None, limit=None): if native: stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Native plan fell back'))) - stack.enter_context(patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Native read fell back'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native read fell back') + )) plan = scan.plan() return builder.new_read().to_arrow(plan.splits()) diff --git a/paimon-python/pypaimon/tests/native_read_path_completion_test.py b/paimon-python/pypaimon/tests/native_read_path_completion_test.py index c171094ba0b7..7e6b1b3db954 100644 --- a/paimon-python/pypaimon/tests/native_read_path_completion_test.py +++ b/paimon-python/pypaimon/tests/native_read_path_completion_test.py @@ -268,8 +268,10 @@ def test_authorized_python_and_serialized_splits_use_native_reader(guarded_table for split in splits] assert all(getattr(split, '_native_split', None) is None for split in splits) builder = table.copy({'read.native.enabled': 'true'}).new_read_builder().with_projection(['id', 'value']) - with patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Native read fell back')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native read fell back') + ): rows = builder.new_read().to_arrow(splits).sort_by([('id', 'ascending')]).to_pylist() assert rows == [{'id': 2, 'value': 'ABCABC'}, {'id': 3, 'value': ''}, {'id': 4, 'value': None}] @@ -342,7 +344,7 @@ def table_query_auth(self, options, identifier): with patch('pypaimon.execution.native.read.plan.native_plan', return_value=Plan([])) as planner, \ patch('pypaimon.execution.native.runtime.native_runtime_available', return_value=True): with pytest.raises(TableNoPermissionException): - scan.plan_all_files() + scan.plan_all() planner.assert_not_called() diff --git a/paimon-python/pypaimon/tests/native_read_test.py b/paimon-python/pypaimon/tests/native_read_test.py index c3f0b9f5693c..f8bbe3833536 100644 --- a/paimon-python/pypaimon/tests/native_read_test.py +++ b/paimon-python/pypaimon/tests/native_read_test.py @@ -587,7 +587,9 @@ def test_native_batch_reader_setup_fallback_preserves_python_blob_worker_cap(): schema = pa.schema([('id', pa.int32())]) with patch('pypaimon.execution.native.read.table_read.native_read', side_effect=RuntimeError('setup failed')) as native, \ - patch.object(read, '_arrow_batch_generator', return_value=iter([_id_batch([1])])) as python_read: + patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._arrow_batch_generator', + return_value=iter([_id_batch([1])])) as python_read: result = read.to_arrow_batch_reader(splits, parallelism=16, blob_parallelism=16).read_all() assert result.to_pydict() == {'id': [1]} assert native.call_args.kwargs['parallelism'] == 16 @@ -636,7 +638,10 @@ def batches(): closed.append(True) with patch('pypaimon.execution.native.read.table_read.native_read', return_value=batches()) as native, \ - patch.object(read, '_create_split_read', side_effect=AssertionError('Python replay')): + patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python replay') + ): with pytest.raises(RuntimeError, match='stream failed'): result = getattr(read, method_name)(splits) if method_name != 'to_arrow': diff --git a/paimon-python/pypaimon/tests/native_read_type_test.py b/paimon-python/pypaimon/tests/native_read_type_test.py index bff791b310e9..18a63ba6f091 100644 --- a/paimon-python/pypaimon/tests/native_read_type_test.py +++ b/paimon-python/pypaimon/tests/native_read_type_test.py @@ -125,7 +125,10 @@ def test_nested_map_variant_share_read_type(tmp_path, mode, shredded, stream): if shredded: for row in expected: del row['ratio'] - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): for parallelism in (1, 2): assert read.to_arrow(splits, parallelism=parallelism).sort_by('id').to_pylist() == expected result = read.to_arrow_batch_reader(splits, parallelism=parallelism).read_all() @@ -145,7 +148,10 @@ def test_list_output_order_and_nullable_parents(tmp_path, native, mode): {'profile_detail_score': None, 'attrs_overflow': None, 'id': 2}, {'profile_detail_score': None, 'attrs_overflow': None, 'id': 3}] if native: - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): result = read.to_arrow(splits).sort_by('id').to_pylist() else: # ROW fallback in DE remains deliberately unsupported in the existing @@ -219,7 +225,10 @@ def test_nested_variant_and_map_types_are_structured_native_requests(tmp_path, m 'paths': ['$.ratio'], 'target_type': pa.float32(), 'fail_on_error': True}}) read_builder = table.new_read_builder().with_read_type(requested) read = read_builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): result = read.to_arrow(read_builder.new_scan().plan().splits()).sort_by('id') assert result.to_pylist() == [ {'outer': {'payload': {'0': 1.25}, 'attrs': {'wanted': 10}}, 'id': 1}, @@ -229,7 +238,10 @@ def test_nested_variant_and_map_types_are_structured_native_requests(tmp_path, m 'ratio': "variant_get(outer.payload, '$.ratio', 'float')", 'wanted': "outer.attrs['wanted']", 'id': 'id'}) read = named.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): assert read.to_arrow(named.new_scan().plan().splits()).sort_by('id').to_pylist() == [ {'ratio': 1.25, 'wanted': 10, 'id': 1}, {'ratio': None, 'wanted': None, 'id': 2}, {'ratio': None, 'wanted': None, 'id': 3}] @@ -243,7 +255,10 @@ def test_shredded_variant_projection_retains_fallbacks_and_nulls(tmp_path, mode) 'wrong_case': "try_variant_get(payload, '$.Ratio', 'float')", 'id': 'id'}) read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): assert read.to_arrow(builder.new_scan().plan().splits()).sort_by('id').to_pylist() == [ {'ratio': values[0]['payload']['ratio'], 'wrong_case': None, 'id': 1}, {'ratio': None, 'wrong_case': None, 'id': 2}, @@ -437,7 +452,10 @@ def test_clipped_variant_reads_only_selected_paths_for_all_json_kinds(tmp_path): 'ratio': "%s(payload, '$.ratio', 'float')" % function, 'wanted': "attrs['wanted']", 'id': 'id'}) read = reader.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): result = read.to_arrow(reader.new_scan().plan().splits()).sort_by('id').to_pylist() assert result == [{'ratio': 1.25, 'wanted': None, 'id': 1}, {'ratio': None, 'wanted': None, 'id': 2}, @@ -469,7 +487,10 @@ def test_map_aggregation_projects_keys_after_merging_original_values(tmp_path, d commit.close() builder = table.new_read_builder().with_projection({'a': "attrs['a']", 'b': "attrs['b']"}) read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): assert read.to_arrow(builder.new_scan().plan().splits()).to_pylist() == [{'a': 7, 'b': 9}] @@ -506,12 +527,18 @@ def test_reversed_nested_variant_fields_keep_their_own_paths(tmp_path): for name, path in [('v2', '$.b'), ('v1', '$.a')]}) builder = table.new_read_builder().with_read_type(requested) read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): assert read.to_arrow(builder.new_scan().plan().splits()).to_pylist() == [ {'outer': {'inner': {'v2': {'0': 9.0}, 'v1': {'0': 7.0}}, 'ordinary': 5}}] builder.with_projection({'second': "variant_get(outer.inner.v2, '$.b', 'float')", 'first': "variant_get(outer.inner.v1, '$.a', 'float')", 'ordinary': 'outer.ordinary'}) read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): assert read.to_arrow(builder.new_scan().plan().splits()).to_pylist() == [ {'second': 9.0, 'first': 7.0, 'ordinary': 5}] diff --git a/paimon-python/pypaimon/tests/native_row_sidecar_test.py b/paimon-python/pypaimon/tests/native_row_sidecar_test.py index cd5d8b300012..7e91ccea0d07 100644 --- a/paimon-python/pypaimon/tests/native_row_sidecar_test.py +++ b/paimon-python/pypaimon/tests/native_row_sidecar_test.py @@ -110,7 +110,10 @@ def _read(table, projection, row_id, native=True): splits = builder.new_scan().plan().splits() if native: # Fail if a successful assertion actually used Python's ROW decoder. - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): return read.to_arrow(splits) return read.to_arrow(splits) @@ -379,7 +382,10 @@ def test_bitmap_and_limit_select_expected_source_without_requested_row_ranges(tm else: builder.with_limit(1) read = builder.new_read() - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): actual = read.to_arrow(builder.new_scan().plan().splits()) expected_id = 5 if mode == 'bitmap' else 0 assert actual.to_pydict() == {'id': [expected_id], 'value': ['v%d' % expected_id]} diff --git a/paimon-python/pypaimon/tests/native_scan_stats_test.py b/paimon-python/pypaimon/tests/native_scan_stats_test.py index 3e79d9af5f9c..212c6fb66b98 100644 --- a/paimon-python/pypaimon/tests/native_scan_stats_test.py +++ b/paimon-python/pypaimon/tests/native_scan_stats_test.py @@ -371,6 +371,8 @@ def test_native_explain_preserves_the_authorized_plan_and_captured_policy(guarde assert plan.splits() assert all(getattr(split, '_native_split', None) is not None for split in plan.splits()) server.table_query_auth_store[table.identifier.get_full_name()] = AuthTableQueryResponse() - with patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Native authorized read fell back')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native authorized read fell back') + ): assert reader.new_read().to_arrow(plan.splits()).to_pylist() == [{'id': 2, 'value': 'ab'}] diff --git a/paimon-python/pypaimon/tests/native_schema_evolution_cast_test.py b/paimon-python/pypaimon/tests/native_schema_evolution_cast_test.py index 533bc978cf24..edf838fbf82b 100644 --- a/paimon-python/pypaimon/tests/native_schema_evolution_cast_test.py +++ b/paimon-python/pypaimon/tests/native_schema_evolution_cast_test.py @@ -81,8 +81,10 @@ def _read(table, native, predicate=None, projection=None): if native: stack.enter_context(patch.object(scan.file_scanner, 'scan', side_effect=AssertionError('Native plan fell back'))) - stack.enter_context(patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Native read fell back'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native read fell back') + )) plan = scan.plan() result = builder.new_read().to_arrow(plan.splits()) return result.sort_by([('id', 'ascending')]) @@ -110,8 +112,10 @@ def test_large_utf8_old_parquet_reads_as_integer_without_native_fallback(rest_ca meta.file_size = os.stat(path).st_size assert pq.read_schema(path).field('v').type == pa.large_string() reader = table.copy({'read.native.enabled': 'true'}).new_read_builder().new_read() - with patch('pypaimon.read.table_read.TableRead._create_split_read', - side_effect=AssertionError('Native read fell back')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native read fell back') + ): result = reader.to_arrow(splits).sort_by([('id', 'ascending')]) assert result.column('v').to_pylist() == [1, -2, None] diff --git a/paimon-python/pypaimon/tests/native_sequence_test.py b/paimon-python/pypaimon/tests/native_sequence_test.py index cc873af9e50e..791f0f55a210 100644 --- a/paimon-python/pypaimon/tests/native_sequence_test.py +++ b/paimon-python/pypaimon/tests/native_sequence_test.py @@ -87,8 +87,10 @@ def _read(table, native, predicate=None, projection=None): read = builder.new_read() with ExitStack() as stack: if native: - stack.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) return read.to_arrow(splits).sort_by('id').to_pylist() diff --git a/paimon-python/pypaimon/tests/native_sliced_split_read_test.py b/paimon-python/pypaimon/tests/native_sliced_split_read_test.py index 456e9fbed5e2..3de22c889750 100644 --- a/paimon-python/pypaimon/tests/native_sliced_split_read_test.py +++ b/paimon-python/pypaimon/tests/native_sliced_split_read_test.py @@ -119,8 +119,10 @@ def _read(table, splits, native, projection, limit=2, predicate=None): read = builder.with_filter(predicate).new_read() with ExitStack() as guard: if native: - guard.enter_context(patch.object( - read, '_create_split_read', side_effect=AssertionError('Python sliced-reader fallback'))) + guard.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python sliced-reader fallback') + )) return read.to_arrow(splits, parallelism=1) diff --git a/paimon-python/pypaimon/tests/native_stream_consumer_branch_test.py b/paimon-python/pypaimon/tests/native_stream_consumer_branch_test.py index 907872489cb2..794afa906ec7 100644 --- a/paimon-python/pypaimon/tests/native_stream_consumer_branch_test.py +++ b/paimon-python/pypaimon/tests/native_stream_consumer_branch_test.py @@ -28,7 +28,7 @@ from pypaimon.common.identifier import Identifier from pypaimon.consumer.consumer import Consumer from pypaimon.execution.native.runtime import (native_runtime_available) -from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan, StreamTableScan +from pypaimon.read.streaming_table_scan import StreamTableScan from pypaimon.tests import native_plan_rest_test @@ -69,12 +69,12 @@ def _tables(catalog, native, branch_snapshot): def _read_and_acknowledge(table, native): builder = table.new_stream_read_builder().with_consumer_id('job').with_poll_interval_ms(1) scan = builder.new_streaming_scan() - assert isinstance(scan, StreamTableScan if native else AsyncStreamingTableScan) + assert isinstance(scan, StreamTableScan) if not native: # Keep this regression independent of the optional prefetch worker. - scan._prefetch_executor.shutdown(wait=True) - scan._prefetch_executor = None - scan._prefetch_enabled = False + scan._execution._prefetch_executor.shutdown(wait=True) + scan._execution._prefetch_executor = None + scan._execution._prefetch_enabled = False async def consume(): iterator = scan.stream() @@ -83,8 +83,10 @@ async def consume(): reader = builder.new_read() with ExitStack() as guards: if native: - guards.enter_context(patch.object(reader, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + guards.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) rows = reader.to_arrow(plan.splits()).to_pylist() # Resuming acknowledges the preceding plan before polling for data. with pytest.raises(asyncio.TimeoutError): diff --git a/paimon-python/pypaimon/tests/native_stream_query_auth_test.py b/paimon-python/pypaimon/tests/native_stream_query_auth_test.py index e0915e9dd20e..33391d81777e 100644 --- a/paimon-python/pypaimon/tests/native_stream_query_auth_test.py +++ b/paimon-python/pypaimon/tests/native_stream_query_auth_test.py @@ -28,7 +28,7 @@ from pypaimon.api.api_response import AuthTableQueryResponse, ErrorResponse from pypaimon.common.predicate_builder import PredicateBuilder from pypaimon.consumer.consumer_manager import ConsumerManager -from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan, StreamTableScan +from pypaimon.read.streaming_table_scan import StreamTableScan from pypaimon.schema.data_types import PyarrowFieldParser from pypaimon.schema.schema_change import SchemaChange from pypaimon.table.row.generic_row import GenericRow @@ -101,8 +101,10 @@ def _rows(builder, plan, native): reader = builder.new_read() with ExitStack() as stack: if native: - stack.enter_context(patch.object(reader, '_create_split_read', - side_effect=AssertionError('Native read fell back'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native read fell back') + )) result = reader.to_arrow(plan.splits()).to_pylist() return sorted(result, key=repr) @@ -111,18 +113,13 @@ def _frame(builder, native, snapshot=None): with patch('pypaimon.execution.python.read.streaming_table_scan.AsyncStreamingTableScan', side_effect=AssertionError('Native stream fell back')) if native else ExitStack(): scan = builder.new_streaming_scan() - if native: + try: assert isinstance(scan, StreamTableScan) if snapshot is not None: scan.restore(snapshot.id) return scan.plan() - try: - if snapshot is None: - snapshot = builder.table.snapshot_manager().get_latest_snapshot() - return scan._create_initial_plan(snapshot) - return scan._create_follow_up_plan(snapshot) finally: - scan._prefetch_executor.shutdown(wait=True) + scan.close() @pytest.mark.parametrize('kinds', [False, True]) @@ -220,11 +217,33 @@ def test_combined_timestamp_events_apply_auth_and_masked_filter_before_limit(sou reader = builder.new_read() with ExitStack() as stack: if native: - stack.enter_context(patch.object(reader, '_create_split_read', - side_effect=AssertionError('Native incremental read fell back'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Native incremental read fell back') + )) assert reader.to_arrow(plan.splits()).to_pylist() == [{'id': 5}] +@pytest.mark.parametrize('method', ['plan', 'plan_all', 'scan_with_stats']) +@pytest.mark.parametrize('window', ['equal', 'future']) +def test_empty_timestamp_windows_cannot_bypass_authorization(source, method, window): + _, table, server, _ = source + timestamp = table.snapshot_manager().get_latest_snapshot().time_millis + bounds = (timestamp, timestamp) if window == 'equal' else (timestamp + 1000, timestamp + 2000) + denied = server._mock_response(ErrorResponse('TABLE', 't', 'permission denied', 403), 403) + for native in (False, True): + selected = table.copy({ + 'scan.native-plan.enabled': str(native).lower(), + 'read.native.enabled': str(native).lower(), + 'incremental-between-timestamp': '{},{}'.format(*bounds), + }) + with patch.object(server, '_table_query_auth_handle', return_value=denied): + scan = selected.new_read_builder().new_scan() + with patch.object(scan, '_create_file_scanner', side_effect=AssertionError('Python scanner ran')): + with pytest.raises(Exception, match='403|permission|denied'): + getattr(scan, method)() + + @pytest.mark.parametrize('custom_policy', ['subclass', 'instance']) def test_custom_authorization_environment_keeps_its_python_policy(source, custom_policy): from pypaimon.catalog.catalog_environment import CatalogEnvironment @@ -245,12 +264,11 @@ def table_query_auth(self, options, identifier): selected.catalog_environment = custom builder = selected.new_stream_read_builder().with_projection(['id']) scan = builder.new_streaming_scan() - assert isinstance(scan, AsyncStreamingTableScan) + assert isinstance(scan, StreamTableScan) try: - snapshot = selected.snapshot_manager().get_latest_snapshot() - assert _rows(builder, scan._create_initial_plan(snapshot), False) == [{'id': 2}] + assert _rows(builder, scan.plan(), False) == [{'id': 2}] finally: - scan._prefetch_executor.shutdown(wait=True) + scan.close() @pytest.mark.parametrize('native', [False, True]) diff --git a/paimon-python/pypaimon/tests/native_stream_table_scan_test.py b/paimon-python/pypaimon/tests/native_stream_table_scan_test.py index 82ae2c97e9a7..1e3b7713e380 100644 --- a/paimon-python/pypaimon/tests/native_stream_table_scan_test.py +++ b/paimon-python/pypaimon/tests/native_stream_table_scan_test.py @@ -83,29 +83,27 @@ def _frame(table, snapshot, mode, native, bucket_filter=None, projection=None, p stack.enter_context(patch( 'pypaimon.execution.python.read.streaming_table_scan.ManifestFileManager', side_effect=AssertionError('Python incremental planning'))) - if native: - if mode != 'initial': - scan.restore(snapshot.id) - plan = scan.plan() - else: - plan = getattr(scan, '_create_{}_plan'.format(mode))(snapshot) + if mode != 'initial': + scan.restore(snapshot.id) + plan = scan.plan() if plan is None: - assert native and mode != 'initial' + assert mode != 'initial' assert scan.checkpoint() == snapshot.id + 1 return None, [] reader = builder.new_read() with ExitStack() as stack: if native: - stack.enter_context(patch.object( - reader, '_create_split_read', side_effect=AssertionError('Python read'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read') + )) rows = reader.to_arrow(plan.splits()).to_pylist() assert plan.snapshot_id == snapshot.id if native: assert all(getattr(split, '_native_split', None) is not None for split in plan.splits()) return plan, sorted(rows, key=lambda row: (row.get('id', 0), repr(row))) finally: - if getattr(scan, '_prefetch_executor', None) is not None: - scan._prefetch_executor.shutdown(wait=True) + scan.close() @pytest.mark.parametrize('engine', ['deduplicate', 'first-row']) diff --git a/paimon-python/pypaimon/tests/native_system_columns_test.py b/paimon-python/pypaimon/tests/native_system_columns_test.py index 89bbd17f4e95..02cc1e63d8cf 100644 --- a/paimon-python/pypaimon/tests/native_system_columns_test.py +++ b/paimon-python/pypaimon/tests/native_system_columns_test.py @@ -76,8 +76,10 @@ def _read(table, native, projection, predicate=None, limit=None): reader = builder.new_read() with ExitStack() as stack: if native: - stack.enter_context(patch.object(reader, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) return reader.to_arrow(builder.new_scan().plan().splits(), parallelism=1) diff --git a/paimon-python/pypaimon/tests/native_timestamp_bucket_write_test.py b/paimon-python/pypaimon/tests/native_timestamp_bucket_write_test.py index dcada8aeb089..b282f6b13817 100644 --- a/paimon-python/pypaimon/tests/native_timestamp_bucket_write_test.py +++ b/paimon-python/pypaimon/tests/native_timestamp_bucket_write_test.py @@ -33,7 +33,6 @@ from pypaimon.schema.data_types import AtomicType, DataField from pypaimon.schema.table_schema import TableSchema from pypaimon.table.row.generic_row import GenericRow, GenericRowDeserializer, GenericRowSerializer -from pypaimon.read.table_read import TableRead from pypaimon.tests import native_plan_rest_test from pypaimon.write.write_dispatch import NativeTableWrite from pypaimon.execution.python.write.row_key_extractor import FixedBucketRowKeyExtractor @@ -274,8 +273,9 @@ def test_timestamp_row_and_sliced_arrow_writes_agree_with_precomputed_java_bucke commit.close() with ExitStack() as guards: if native: - guards.enter_context(patch.object( - TableRead, '_create_split_read', side_effect=AssertionError('Python read fallback'))) + guards.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback'))) read = table.new_read_builder() rows = {} for split in read.new_scan().plan().splits(): diff --git a/paimon-python/pypaimon/tests/native_update_container_casts_test.py b/paimon-python/pypaimon/tests/native_update_container_casts_test.py index 1814e35c53ae..12e86897c981 100644 --- a/paimon-python/pypaimon/tests/native_update_container_casts_test.py +++ b/paimon-python/pypaimon/tests/native_update_container_casts_test.py @@ -156,8 +156,10 @@ def _check(table, expected): reader = builder.new_read() with ExitStack() as stack: if native: - stack.enter_context(patch.object(reader, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) rows = reader.to_arrow(builder.new_scan().plan().splits()).sort_by('id') assert rows['value'].to_pylist() == expected + [None] * (5 - len(expected)) assert rows['keep'].to_pylist() == list(range(10, 15)) diff --git a/paimon-python/pypaimon/tests/native_variant_update_test.py b/paimon-python/pypaimon/tests/native_variant_update_test.py index 765910852941..93c546581918 100644 --- a/paimon-python/pypaimon/tests/native_variant_update_test.py +++ b/paimon-python/pypaimon/tests/native_variant_update_test.py @@ -91,8 +91,10 @@ def _read(table, native, projection=None): read = builder.new_read() with ExitStack() as stack: if native: - stack.enter_context(patch.object(read, '_create_split_read', - side_effect=AssertionError('Python read fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python read fallback') + )) data = read.to_arrow(splits).sort_by('id') rows = data.to_pylist() for row in rows: diff --git a/paimon-python/pypaimon/tests/native_video_read_test.py b/paimon-python/pypaimon/tests/native_video_read_test.py index dbed48934cbc..dc6d4fd7a341 100644 --- a/paimon-python/pypaimon/tests/native_video_read_test.py +++ b/paimon-python/pypaimon/tests/native_video_read_test.py @@ -81,7 +81,10 @@ def _read(table, native, projection=None, filtered=False, limit=None): read = builder.new_read() splits = builder.new_scan().plan().splits() if native: - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python video read fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python video read fallback') + ): return read.to_arrow(splits), read.to_arrow_batch_reader(splits).read_all() return read.to_arrow(splits), read.to_arrow_batch_reader(splits).read_all() @@ -167,7 +170,10 @@ def test_native_video_placeholder_falls_back_but_null_stops_older_frames(tmp_pat total_buckets=split.total_buckets) expected = [dict(row) for row in before] expected[1]['left'] = None - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): for result in (read.to_arrow([combined]), read.to_arrow_batch_reader([combined]).read_all()): assert result.to_pylist() == expected diff --git a/paimon-python/pypaimon/tests/native_write_capabilities_test.py b/paimon-python/pypaimon/tests/native_write_capabilities_test.py index ab282e7303d4..5b954fe4e5c2 100644 --- a/paimon-python/pypaimon/tests/native_write_capabilities_test.py +++ b/paimon-python/pypaimon/tests/native_write_capabilities_test.py @@ -48,10 +48,13 @@ def _rows(table, native_read, all_files=False, chunk_shuffle=False): scan = builder.new_scan() if chunk_shuffle: scan.with_chunk_shuffle(seed=7, chunk_size=1) - splits = (scan.plan_all_files() if all_files else scan.plan()).splits() + splits = (scan.plan_all() if all_files else scan.plan()).splits() read = builder.new_read() if native_read: - with patch.object(read, '_create_split_read', side_effect=AssertionError('Python fallback')): + with patch( + 'pypaimon.execution.python.read.table_read.PythonTableRead._create_split_read', + side_effect=AssertionError('Python fallback') + ): rows = read.to_arrow(splits).to_pylist() else: rows = read.to_arrow(splits).to_pylist() diff --git a/paimon-python/pypaimon/tests/public_streaming_table_scan_test.py b/paimon-python/pypaimon/tests/public_streaming_table_scan_test.py new file mode 100644 index 000000000000..853d128e60c9 --- /dev/null +++ b/paimon-python/pypaimon/tests/public_streaming_table_scan_test.py @@ -0,0 +1,196 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Direct streaming constructors and builders share backend selection.""" + +import asyncio +from contextlib import ExitStack +from inspect import Parameter, signature +from unittest.mock import Mock, patch + +import pyarrow as pa +import pytest + +from pypaimon import CatalogFactory, Schema +from pypaimon.execution.native.read.streaming_table_scan import StreamTableScan as NativeStreamScan +from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan as PythonStreamScan +from pypaimon.read.plan import Plan +from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan, StreamTableScan + + +pytestmark = [pytest.mark.python_write, pytest.mark.python_commit] + + +@pytest.fixture +def table(tmp_path): + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + catalog.create_table('default.stream_facade', Schema.from_pyarrow_schema( + pa.schema([('id', pa.int32())]), options={ + 'scan.native-plan.enabled': 'false', 'read.native.enabled': 'false', + 'write.native.enabled': 'false', 'commit.native.enabled': 'false', + }), False) + return catalog.get_table('default.stream_facade') + + +def _create(table, entrypoint): + if entrypoint == 'builder': + return table.new_stream_read_builder().new_streaming_scan() + return entrypoint(table) + + +@pytest.mark.parametrize('public,execution', [ + (AsyncStreamingTableScan, PythonStreamScan), (StreamTableScan, NativeStreamScan), +]) +def test_standard_constructor_arguments_and_defaults_remain_stable(public, execution): + def parameters(cls): + return [parameter.replace(annotation=Parameter.empty) + for parameter in signature(cls).parameters.values() + if parameter.name != 'follow_up_scanner'] + assert parameters(public) == parameters(execution) + + +@pytest.mark.parametrize('entrypoint', [AsyncStreamingTableScan, StreamTableScan, 'builder']) +@pytest.mark.parametrize('native_plan,native_read', [(True, False), (False, True), (True, True)]) +def test_all_streaming_entrypoints_select_native_without_python_resources( + table, entrypoint, native_plan, native_read): + selected = table.copy({'scan.native-plan.enabled': str(native_plan).lower(), + 'read.native.enabled': str(native_read).lower()}) + core = Mock() + expected = Plan([], 7) + with ExitStack() as stack: + stack.enter_context(patch( + 'pypaimon.execution.native.runtime.native_runtime_available', return_value=True)) + for resource in ('FileScanner', 'ManifestListManager', 'ManifestFileManager'): + target = 'pypaimon.execution.python.read.streaming_table_scan.' + resource + stack.enter_context(patch(target, side_effect=AssertionError(target))) + stack.enter_context(patch( + 'pypaimon.execution.native.read.streaming_table_scan.native_stream_scan', return_value=core)) + stack.enter_context(patch( + 'pypaimon.execution.native.read.streaming_table_scan._from_native_plan', return_value=expected)) + scan = _create(selected, entrypoint) + try: + assert isinstance(scan._execution, NativeStreamScan) + assert scan.plan() is expected + finally: + scan.close() + core.plan.assert_called_once() + + +@pytest.mark.parametrize('entrypoint', [AsyncStreamingTableScan, StreamTableScan, 'builder']) +@pytest.mark.parametrize('failure', ['disabled', 'unavailable', 'preparation']) +def test_streaming_fallback_is_selected_before_planning(table, entrypoint, failure): + selected = table.copy({'read.native.enabled': str(failure != 'disabled').lower()}) + with patch('pypaimon.execution.native.runtime.native_runtime_available', + return_value=failure == 'preparation') as runtime, patch( + 'pypaimon.execution.native.read.streaming_table_scan.native_stream_scan', + side_effect=ImportError('optional runtime unavailable')) as native: + scan = _create(selected, entrypoint) + try: + assert isinstance(scan._execution, PythonStreamScan) + assert scan.plan() is None + finally: + scan.close() + assert native.call_count == int(failure == 'preparation') + assert runtime.call_count == int(failure != 'disabled') + + +@pytest.mark.parametrize('entrypoint', [AsyncStreamingTableScan, StreamTableScan, 'builder']) +def test_planning_errors_never_switch_to_python(table, entrypoint): + failure = OSError('native frame unavailable') + core = Mock() + core.plan.side_effect = failure + with patch('pypaimon.execution.native.read.streaming_table_scan.native_stream_scan', return_value=core), \ + patch('pypaimon.execution.python.read.streaming_table_scan.AsyncStreamingTableScan', + side_effect=AssertionError('late Python fallback')): + scan = _create(table.copy({'scan.native-plan.enabled': 'true'}), entrypoint) + try: + with pytest.raises(OSError) as caught: + scan.plan() + assert caught.value is failure + finally: + scan.close() + core.plan.assert_called_once() + + +def test_public_streaming_does_not_expose_scanner_injection(table): + with pytest.raises(TypeError, match='follow_up_scanner'): + AsyncStreamingTableScan(table, follow_up_scanner=object()) + + +def test_python_constructor_controls_do_not_force_python_execution(table): + core = Mock() + with patch('pypaimon.execution.native.read.streaming_table_scan.native_stream_scan', return_value=core): + scan = AsyncStreamingTableScan( + table.copy({'scan.native-plan.enabled': 'true'}), prefetch_enabled=False, diff_threshold=3) + try: + assert isinstance(scan._execution, NativeStreamScan) + finally: + scan.close() + + +@pytest.mark.parametrize('public', [AsyncStreamingTableScan, StreamTableScan]) +@pytest.mark.parametrize('synchronous', [False, True]) +def test_polling_honors_public_control_overrides_and_cancellation(public, synchronous): + acknowledgements = [] + + class CustomScan(public): + position = 0 + + def plan(self): + self.position += 1 + return Plan([], self.position) + + def checkpoint(self): + return self.position + + def notify_checkpoint_complete(self, next_snapshot): + acknowledgements.append(next_snapshot) + + execution = Mock() + with patch('pypaimon.read.read_dispatch.create_streaming_scan', return_value=execution): + scan = CustomScan(object()) + + async def consume(): + iterator = scan.stream() + try: + assert (await iterator.__anext__()).snapshot_id == 1 + assert acknowledgements == [] + assert (await iterator.__anext__()).snapshot_id == 2 + finally: + await iterator.aclose() + + try: + if synchronous: + iterator = scan.stream_sync() + try: + assert next(iterator).snapshot_id == 1 + assert acknowledgements == [] + assert next(iterator).snapshot_id == 2 + finally: + iterator.close() + else: + asyncio.run(consume()) + assert acknowledgements == [1] + assert scan._pending_consumer_snapshot is None + execution.plan.assert_not_called() + execution.checkpoint.assert_not_called() + execution.notify_checkpoint_complete.assert_not_called() + execution.cancel_prefetch.assert_called_once() + finally: + scan.close() + execution.close.assert_called_once() diff --git a/paimon-python/pypaimon/tests/pushdown_bucket_test.py b/paimon-python/pypaimon/tests/pushdown_bucket_test.py index 0ad5f40239b6..d2d72bb27cbf 100644 --- a/paimon-python/pypaimon/tests/pushdown_bucket_test.py +++ b/paimon-python/pypaimon/tests/pushdown_bucket_test.py @@ -52,6 +52,7 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon.common.predicate_builder import PredicateBuilder from pypaimon.manifest.manifest_list_manager import ManifestListManager from pypaimon.manifest.schema.manifest_file_meta import ManifestFileMeta @@ -729,7 +730,7 @@ def test_value_only_predicate_falls_back_to_full_scan(self): # rows we picked. rb = table.new_read_builder().with_filter(pred) scan = rb.new_scan() - self.assertIsNone(scan.file_scanner._bucket_selector, + self.assertIsNone(PythonTableScan(scan)._file_scanner()._bucket_selector, "value-only predicate must NOT produce a selector") def test_range_on_pk_falls_back_to_full_scan(self): @@ -787,7 +788,7 @@ def test_manifest_bucket_pruning_skips_file_reads(self): def test_manifest_bucket_pruning_keeps_unknown_metadata(self): table = self._create_pk_table('manifest_bucket_fallback') predicate = table.new_read_builder().new_predicate_builder().equal('id', 0) - scanner = table.new_read_builder().with_filter(predicate).new_scan().file_scanner + scanner = PythonTableScan(table.new_read_builder().with_filter(predicate).new_scan())._file_scanner() selected = next(iter(self._expected_buckets(table, [0]))) other = (selected + 1) % self.NUM_BUCKETS meta = ManifestFileMeta('manifest', 1, 1, 0, SimpleStats.empty_stats(), 0, @@ -861,7 +862,7 @@ def test_init_bucket_selector_fails_open_when_bucket_keys_raises(self): rb = table.new_read_builder().with_filter( table.new_read_builder().new_predicate_builder().equal('id', 1)) - scanner = rb.new_scan().file_scanner + scanner = PythonTableScan(rb.new_scan())._file_scanner() # Must NOT raise: the broken option falls back to "no pruning", # and the scan still finds the row. self.assertIsNone(scanner._init_bucket_selector()) diff --git a/paimon-python/pypaimon/tests/py36/rest_ao_read_write_test.py b/paimon-python/pypaimon/tests/py36/rest_ao_read_write_test.py index 52e5ccf41dca..2fccf2dad7ac 100644 --- a/paimon-python/pypaimon/tests/py36/rest_ao_read_write_test.py +++ b/paimon-python/pypaimon/tests/py36/rest_ao_read_write_test.py @@ -28,6 +28,7 @@ import pandas as pd import pyarrow as pa +from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon import CatalogFactory, Schema from pypaimon.common.options import Options from pypaimon.catalog.catalog_context import CatalogContext @@ -252,10 +253,11 @@ def test_full_data_types(self): # to test GenericRow ability latest_snapshot = table.snapshot_manager().get_latest_snapshot() - manifest_files = table_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot) - manifest_entries = table_scan.file_scanner.manifest_file_manager.read( + file_scanner = PythonTableScan(table_scan)._file_scanner() + manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot) + manifest_entries = file_scanner.manifest_file_manager.read( manifest_files[0].file_name, - lambda row: table_scan.file_scanner._filter_manifest_entry(row), + lambda row: file_scanner._filter_manifest_entry(row), drop_stats=False) # Python write does not produce value stats if stats_enabled: diff --git a/paimon-python/pypaimon/tests/query_auth_filter_test.py b/paimon-python/pypaimon/tests/query_auth_filter_test.py index 74a75f3438b3..ab2e3bb7759f 100644 --- a/paimon-python/pypaimon/tests/query_auth_filter_test.py +++ b/paimon-python/pypaimon/tests/query_auth_filter_test.py @@ -22,9 +22,10 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_scan import PythonTableScan +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.catalog.table_query_auth import TableQueryAuthResult from pypaimon.common.predicate_builder import PredicateBuilder -from pypaimon.read.table_read import TableRead from pypaimon.table.row.generic_row import GenericRow from pypaimon.table.row.row_kind import RowKind @@ -155,7 +156,7 @@ def test_limit_is_applied_after_masked_filter(table): assert read(table, predicate, projection=['id'], limit=1) == [{'id': 2}] -def test_plan_all_files_authorizes_every_column_before_exposing_inventory(table): +def test_plan_all_authorizes_every_column_before_exposing_inventory(table): from pypaimon.catalog.catalog_exception import TableNoPermissionException table = table.copy({'scan.native-plan.enabled': 'false', 'read.native.enabled': 'false'}) selected = [] @@ -170,7 +171,7 @@ def auth(select): scan = table.new_read_builder().with_projection(['id']).new_scan() assert scan.plan().splits() with pytest.raises(TableNoPermissionException): - scan.plan_all_files() + scan.plan_all() assert selected == [['id'], None] @@ -220,14 +221,14 @@ def test_or_arms_match_independently(table, masked_value, expected): def test_unmasked_conjunct_is_pushed_to_split_reads(table, monkeypatch): authorize(table, rules()) pushed = [] - create_split_read = TableRead._create_split_read + create_split_read = PythonTableRead._create_split_read def recording(self, *args, **kwargs): split_read = create_split_read(self, *args, **kwargs) pushed.append(split_read.predicate) return split_read - monkeypatch.setattr(TableRead, '_create_split_read', recording) + monkeypatch.setattr(PythonTableRead, '_create_split_read', recording) predicates = PredicateBuilder(table.fields) unmasked = predicates.greater_than('id', 1) for combine, expected_push, expected_ids in [ @@ -262,8 +263,8 @@ def test_unmasked_conjunct_keeps_its_pushdown(table): predicate = PredicateBuilder.and_predicates([unmasked, predicates.is_null('secret')]) scan = table.new_read_builder().with_filter(predicate).new_scan() scan.plan() - assert scan.file_scanner.predicate == unmasked - assert scan.file_scanner.auth_has_non_partition_filter + assert PythonTableScan(scan)._file_scanner().predicate == unmasked + assert PythonTableScan(scan)._file_scanner().auth_has_non_partition_filter def test_same_scan_refreshes_masked_filter_rules(table): @@ -289,14 +290,45 @@ def test_explain_keeps_masked_filter_candidates(table): @pytest.mark.parametrize('table', ['partitioned-append', 'partitioned-primary-key'], indirect=True) -def test_explicit_partition_filter_on_masked_key_is_rejected(table): +@pytest.mark.parametrize('file_scanner_selector', [False, True]) +def test_explicit_partition_filter_on_masked_key_is_rejected(table, file_scanner_selector): authorize(table, rules()) predicate = PredicateBuilder(table.fields).equal('secret', 'alpha') - scan = table.new_read_builder().with_partition_filter(predicate).new_scan() + builder = table.new_read_builder() + if file_scanner_selector: + scan = builder.new_scan() + scan.file_scanner.partition_key_predicate = predicate.new_index(0) + else: + scan = builder.with_partition_filter(predicate).new_scan() with pytest.raises(ValueError, match='partition filters cannot reference masked columns'): scan.plan() +@pytest.mark.parametrize('table', ['partitioned-append', 'partitioned-primary-key'], indirect=True) +@pytest.mark.parametrize('derived_partition', [False, True]) +@pytest.mark.parametrize('selected_partition', [None, 'beta']) +def test_file_partition_selection_survives_masked_predicate_pruning( + table, derived_partition, selected_partition): + authorize(table, TableQueryAuthResult(None, {'salt': json.dumps({'name': 'NULL'})})) + predicates = PredicateBuilder(table.fields) + predicate = predicates.is_null('salt') + if derived_partition: + predicate = predicates.and_predicates([predicate, predicates.equal('secret', 'alpha')]) + builder = table.new_read_builder().with_filter(predicate).with_projection(['id']) + scan = builder.new_scan() + scan.file_scanner.partition_key_predicate = ( + None if selected_partition is None else predicates.equal('secret', selected_partition).new_index(0)) + plan = scan.plan() + expected = ({'alpha'} if derived_partition else {'alpha', 'beta', 'gamma'}) + if selected_partition is not None: + expected = {'beta'} + assert {split.partition.values[0] for split in plan.splits()} == expected + expected_ids = [1] if derived_partition else [1, 2, 3] + if selected_partition is not None: + expected_ids = [] if derived_partition else [2] + assert sorted(row['id'] for row in builder.new_read().to_arrow(plan.splits()).to_pylist()) == expected_ids + + @pytest.mark.parametrize('phase', ['initial', 'follow_up', 'catch_up']) def test_streaming_user_filter_runs_after_masking(table, phase): authorize(table, rules(field_ref('salt', 2))) @@ -320,9 +352,9 @@ def test_write_scan_restores_filter_after_rules_are_removed(table): scan = builder.new_scan() scan.plan() state[0] = None - splits = scan.plan_all_files().splits() - assert scan.file_scanner.predicate == predicate - assert not scan.file_scanner.auth_has_non_partition_filter + splits = scan.plan_all().splits() + assert PythonTableScan(scan)._file_scanner().predicate == predicate + assert not PythonTableScan(scan)._file_scanner().auth_has_non_partition_filter assert arrow_rows(builder.new_read().to_arrow(splits)) == [] diff --git a/paimon-python/pypaimon/tests/query_auth_validation_test.py b/paimon-python/pypaimon/tests/query_auth_validation_test.py index f776d6602fcc..d0c93557a7ca 100644 --- a/paimon-python/pypaimon/tests/query_auth_validation_test.py +++ b/paimon-python/pypaimon/tests/query_auth_validation_test.py @@ -19,12 +19,14 @@ import json import time import unittest +from unittest.mock import patch from types import SimpleNamespace import pyarrow as pa import pytest +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.common.options import CoreOptions, Options from pypaimon.read.query_auth_split import QueryAuthSplit from pypaimon.read.reader.iface.record_batch_reader import RecordBatchReader @@ -32,8 +34,9 @@ from pypaimon.schema.data_types import PyarrowFieldParser from pypaimon.catalog.table_query_auth import TableQueryAuthResult from pypaimon.read.table_scan import TableScan, validate_auth_rules -from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan +from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan from pypaimon.read.plan import Plan +from pypaimon.read.scan_request import ScanRequest from pypaimon.schema.data_types import AtomicType, DataField, RowType from pypaimon.table.special_fields import SpecialFields @@ -114,6 +117,9 @@ def _make_scan(auth_result, read_type, table): scan._query_auth_fn = lambda select: auth_result scan._read_type = read_type scan.table = table + scan.limit = None + scan._file_scanner = None + scan._scan_configuration = ScanRequest() return scan @@ -486,7 +492,7 @@ def test_read_rejects_nested_projection_of_mask_target(self): [_string_field(0, "id"), _string_field(2, "s_a")], [["id"], ["s", "a"]]) with self.assertRaises(RuntimeError) as ctx: - read._TableRead__authed_reader(None, auth) + PythonTableRead(read)._authed_reader(None, auth) self.assertIn("partial column", str(ctx.exception)) def test_read_rejects_nested_projection_of_filter_operand(self): @@ -497,7 +503,7 @@ def test_read_rejects_nested_projection_of_filter_operand(self): [_string_field(0, "id"), _string_field(2, "s_a")], [["id"], ["s", "a"]]) with self.assertRaises(RuntimeError) as ctx: - read._TableRead__authed_reader(None, auth) + PythonTableRead(read)._authed_reader(None, auth) self.assertIn("partial column", str(ctx.exception)) @@ -544,7 +550,7 @@ def test_a_cross_mask_on_a_physical_system_looking_column_is_rejected(self): self.assertIn("masked too", str(ctx.exception)) def test_streaming_scan_validates_like_the_batch_scan(self): - from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan + from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan auth = TableQueryAuthResult(None, {"email": _null_mask()}) fields = [_string_field(0, "id"), _string_field(1, "mail")] scan = AsyncStreamingTableScan.__new__(AsyncStreamingTableScan) @@ -821,8 +827,11 @@ def _reader(self, table, fields): batch = pa.RecordBatch.from_pydict( {"id": ["1"], "email": ["SECRET"]}, schema=PyarrowFieldParser.from_paimon_schema(fields)) - read._create_split_read = lambda split, **kw: SimpleNamespace( - create_reader=lambda: _OneBatch(batch)) + storage = patch.object( + PythonTableRead, '_create_split_read', side_effect=lambda split, **kw: SimpleNamespace( + create_reader=lambda: _OneBatch(batch))) + storage.start() + self.addCleanup(storage.stop) return read def test_a_read_asks_the_catalog_once_across_workers_and_rule_objects(self): @@ -881,7 +890,9 @@ def storage(split, **kwargs): {f.name: data[f.name] for f in requested}, schema=schema) return SimpleNamespace(create_reader=lambda: _OneBatch(batch)) - read._create_split_read = storage + stub = patch.object(PythonTableRead, '_create_split_read', side_effect=storage) + stub.start() + self.addCleanup(stub.stop) return read def _rows(self, read, auth): diff --git a/paimon-python/pypaimon/tests/raw_file_scan_auth_scope_test.py b/paimon-python/pypaimon/tests/raw_file_scan_auth_scope_test.py new file mode 100644 index 000000000000..d1cbe980f8f4 --- /dev/null +++ b/paimon-python/pypaimon/tests/raw_file_scan_auth_scope_test.py @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Raw file planning uses its own authorization scope on cached scanners.""" + +import json +from unittest.mock import patch + +import pyarrow as pa +import pytest + +from pypaimon import CatalogFactory, Schema +from pypaimon.catalog.catalog_exception import TableNoPermissionException +from pypaimon.catalog.table_query_auth import TableQueryAuthResult +from pypaimon.execution.native.read.plan import NativeTableScan + + +@pytest.fixture +def table(tmp_path): + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + arrow_schema = pa.schema([('id', pa.int32()), ('pt', pa.string())]) + catalog.create_table('default.files', Schema.from_pyarrow_schema( + arrow_schema, partition_keys=['pt'], options={'file.format': 'parquet'}), False) + table = catalog.get_table('default.files') + builder = table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.Table.from_pylist( + [{'id': 1, 'pt': 'a'}, {'id': 2, 'pt': 'b'}], schema=arrow_schema)) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + return table + + +def _partition_filter(): + return json.dumps({ + 'kind': 'LEAF', + 'transform': {'name': 'FIELD_REF', + 'fieldRef': {'index': 1, 'name': 'pt', 'type': 'STRING'}}, + 'function': 'EQUAL', 'literals': ['a'], + }) + + +@pytest.mark.parametrize('native_fallback', [False, True]) +@pytest.mark.parametrize('restriction', ['row-filter', 'mask']) +def test_raw_scope_clears_cached_query_rules(table, restriction, native_fallback): + builder = table.new_read_builder() + if restriction == 'mask': + predicate = builder.new_predicate_builder().equal('pt', 'a') + builder.with_filter(predicate) + rules = TableQueryAuthResult(None, {'pt': json.dumps({'name': 'NULL'})}) + else: + rules = TableQueryAuthResult([_partition_filter()], None) + scan = builder.new_scan() + state = {'rules': rules} + selects = [] + + def authorize(select): + selects.append(select) + return state['rules'] + + scan._query_auth_fn = authorize + scan.plan() + source = scan._python_file_scanner + state['rules'] = None + with patch.object(scan, '_native_requested', return_value=native_fallback), \ + patch.object(NativeTableScan, 'plan_files', return_value=None) as native: + entries = scan.file_scanner.plan_files() + assert {tuple(entry.partition.values) for entry in entries} == ( + {('a',)} if restriction == 'mask' else {('a',), ('b',)}) + assert scan._python_file_scanner is source + assert source.auth_partition_predicate is None + assert source.predicate == scan.predicate + assert selects[-1] is None, 'Raw files require whole-table authorization' + assert native.call_count == int(native_fallback) + + +def test_restricted_raw_scope_does_not_clear_cached_rules(table): + scan = table.new_read_builder().new_scan() + scan._query_auth_fn = lambda select: TableQueryAuthResult([_partition_filter()], None) + scan.plan() + source = scan._python_file_scanner + prior_partition_filter = source.auth_partition_predicate + assert prior_partition_filter is not None + with patch.object(source, 'plan_files', side_effect=AssertionError('Unauthorized file access')): + with pytest.raises(TableNoPermissionException): + scan.file_scanner.plan_files() + assert source.auth_partition_predicate is prior_partition_filter diff --git a/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py b/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py index 825774d3c1c6..fd1a63d2f3d3 100644 --- a/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py +++ b/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py @@ -123,7 +123,7 @@ def _compact_all_data_files(self, table): read_builder = table.new_read_builder().with_projection( list(table.field_names) + [SpecialFields.ROW_ID.name] ) - plan = read_builder.new_scan().plan_all_files() + plan = read_builder.new_scan().plan_all() old_files = [ file for split in plan.splits() for file in split.files ] @@ -2516,7 +2516,7 @@ def test_self_merge_update_bypasses_routing_shuffle(self): self._write(target, self._source(ids=(1, 2))) self._write(target, self._source(ids=(3, 4))) table = self.catalog.get_table(target) - packed_splits = table.new_read_builder().new_scan().plan_all_files().splits() + packed_splits = table.new_read_builder().new_scan().plan_all().splits() self.assertEqual(len(packed_splits), 1) with patch.object( diff --git a/paimon-python/pypaimon/tests/read_execution_boundary_test.py b/paimon-python/pypaimon/tests/read_execution_boundary_test.py index b8d22d4ef6eb..4f1236e7bcb2 100644 --- a/paimon-python/pypaimon/tests/read_execution_boundary_test.py +++ b/paimon-python/pypaimon/tests/read_execution_boundary_test.py @@ -68,7 +68,7 @@ def find_spec(self, fullname, path=None, target=None): with patch('pypaimon.execution.native.runtime.native_runtime_available', return_value=True), \ patch('pypaimon.execution.native.read.plan.native_plan', return_value=Plan([], 7)): assert builder.new_scan().plan().snapshot_id == 7 - assert builder.new_scan().plan_all_files().snapshot_id == 7 + assert builder.new_scan().plan_all().snapshot_id == 7 assert builder.new_scan().scan_with_stats()[0].snapshot_id == 7 reader = builder.new_read() assert reader.to_arrow([]).num_rows == 0 @@ -305,30 +305,34 @@ def close(self): source = Source() read = SimpleNamespace( - _check_python_merge_supported=lambda: None, - _TableRead__create_reader_for_split=lambda *args, **kwargs: source, + table=SimpleNamespace(is_primary_key_table=False), _begin_auth_read=lambda splits: None, - _serialize_blob_views_as_descriptors=lambda batch, reader: batch, + _blob_as_descriptor=False, + _blob_view_output_indices=(), _project_batch_to_output=lambda batch: batch, output_projection=None, _adapter_read_type=[DataField(0, 'id', AtomicType('INT'))], include_row_kind=False, limit=None, ) - return read, source, schema + from pypaimon.execution.python.read.table_read import PythonTableRead + + execution = PythonTableRead(read) + execution._create_reader_for_split = lambda *args, **kwargs: source + return execution, source, schema def _consume_python_read_path(path, read, schema): from concurrent.futures import ThreadPoolExecutor - from pypaimon.execution.python.read.table_read import PythonTableRead, _RemainingRows + from pypaimon.execution.python.read.table_read import _RemainingRows if path == 'rows': - return list(PythonTableRead(read).read_rows([object()])) + return list(read.read_rows([object()])) if path == 'serial': - return list(PythonTableRead(read)._arrow_batch_generator([object()], schema)) + return list(read._arrow_batch_generator([object()], schema)) with ThreadPoolExecutor(max_workers=1) as executor: return executor.submit( - PythonTableRead(read)._read_one_split_to_batches, + read._read_one_split_to_batches, object(), schema, _RemainingRows(None), ).result(timeout=5) @@ -370,14 +374,12 @@ def test_python_batch_conversion_keeps_primary_failure_when_close_also_fails(pat @pytest.mark.parametrize('path', ['rows', 'serial']) @pytest.mark.parametrize('close_error', [False, True]) def test_python_read_explicit_generator_close_keeps_cleanup_error_visible(path, close_error): - from pypaimon.execution.python.read.table_read import PythonTableRead - cleanup = OSError('explicit close failure') if close_error else None read, source, schema = _python_read_source(cleanup=cleanup) if path == 'rows': - iterator = PythonTableRead(read).read_rows([object()]) + iterator = read.read_rows([object()]) else: - iterator = PythonTableRead(read)._arrow_batch_generator([object()], schema) + iterator = read._arrow_batch_generator([object()], schema) next(iterator) if close_error: with pytest.raises(OSError) as caught: diff --git a/paimon-python/pypaimon/tests/read_execution_protocol_test.py b/paimon-python/pypaimon/tests/read_execution_protocol_test.py index 24dfa5b5cd87..d6cfe3d74cbb 100644 --- a/paimon-python/pypaimon/tests/read_execution_protocol_test.py +++ b/paimon-python/pypaimon/tests/read_execution_protocol_test.py @@ -19,7 +19,7 @@ from inspect import signature from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import Mock, patch import pyarrow as pa import pytest @@ -30,6 +30,7 @@ from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon.read.plan import Plan +from pypaimon.read.scan_request import ScanRequest pytestmark = [pytest.mark.python_plan, pytest.mark.python_read, pytest.mark.python_write, pytest.mark.python_commit] @@ -40,7 +41,7 @@ def test_bound_read_executors_have_the_same_call_contract(method): assert signature(getattr(PythonTableRead, method)) == signature(getattr(NativeTableRead, method)) -@pytest.mark.parametrize('method', ['plan', 'plan_all_files', 'scan_with_stats']) +@pytest.mark.parametrize('method', ['plan', 'plan_all', 'scan_with_stats']) def test_bound_scan_executors_have_the_same_call_contract(method): assert signature(getattr(PythonTableScan, method)) == signature(getattr(NativeTableScan, method)) @@ -68,7 +69,7 @@ def table(tmp_path): @pytest.mark.parametrize('backend', ['python', 'native']) @pytest.mark.parametrize('method', ['read_rows', 'read_batches']) -def test_bound_read_uses_live_configuration_and_preserves_public_reader_hooks(table, backend, method): +def test_bound_read_uses_live_configuration_and_selected_executor(table, backend, method): splits = table.new_read_builder().new_scan().plan().splits() if backend == 'native': table = table.copy({'read.native.enabled': 'true'}) @@ -95,8 +96,8 @@ def batches(): return batches() with patch('pypaimon.execution.native.read.table_read.native_read', side_effect=native_batches) as native, \ - patch.object(read, '_TableRead__create_reader_for_split', - wraps=read._TableRead__create_reader_for_split) as python: + patch.object(PythonTableRead, '_create_reader_for_split', autospec=True, + side_effect=PythonTableRead._create_reader_for_split) as python: if method == 'read_rows': values = [row.get_field(0) for row in execution.read_rows(splits)] else: @@ -145,16 +146,16 @@ def scan(): return expected scanner.scan = scan - execution = PythonTableScan(SimpleNamespace(file_scanner=scanner, _query_auth_fn=None)) + execution = PythonTableScan(SimpleNamespace(_python_file_scanner=scanner, _query_auth_fn=None)) if failure: with pytest.raises(OSError, match='planning failed'): - execution.plan_all_files() + execution.plan_all() else: - assert execution.plan_all_files() is expected + assert execution.plan_all() is expected assert scanner.skip_level0 is True -@pytest.mark.parametrize('method', ['plan', 'plan_all_files', 'scan_with_stats']) +@pytest.mark.parametrize('method', ['plan', 'plan_all', 'scan_with_stats']) def test_native_bound_scan_uses_live_configuration_without_creating_a_python_scanner(table, method): scan = table.new_read_builder().new_scan() execution = NativeTableScan(scan) @@ -165,11 +166,11 @@ def test_native_bound_scan_uses_live_configuration_without_creating_a_python_sca result = getattr(execution, method)() assert (result[0] if method == 'scan_with_stats' else result).snapshot_id == 9 assert native.call_args[1]['limit'] == 1 - assert bool(native.call_args[1].get('all_files')) is (method == 'plan_all_files') + assert bool(native.call_args[1].get('all_files')) is (method == 'plan_all') assert scan._file_scanner is None -@pytest.mark.parametrize('method', ['plan', 'plan_all_files', 'scan_with_stats']) +@pytest.mark.parametrize('method', ['plan', 'plan_all', 'scan_with_stats']) def test_native_bound_scan_does_not_turn_preparation_failure_into_an_empty_plan(table, method): scan = table.new_read_builder().new_scan() execution = NativeTableScan(scan) @@ -189,3 +190,38 @@ def test_native_bound_batches_preserve_the_requested_order_mode(table, ordered): assert list(execution.read_batches([split], schema, parallelism=3, ordered=ordered)) == [] assert native.call_args[1]['parallelism'] == 3 assert native.call_args[1]['ordered'] is ordered + + +def test_python_scanner_uses_private_cache_and_captured_configuration(table): + request = ScanRequest(idx_of_this_subtask=1, number_of_para_subtasks=3, + row_ranges=((7, 9),)) + configure = Mock(side_effect=lambda scanner: setattr(scanner, 'skip_level0', False)) + + class PublicScanner: + _is_dispatch_file_scanner = True + _configure_python_scanner = staticmethod(configure) + + class Scan: + _python_file_scanner = None + _file_scanner = PublicScanner() + _query_auth_fn = None + + @property + def file_scanner(self): + raise AssertionError('Python execution entered public scanner dispatch') + + def _scan_request(self): + return request + + scan = Scan() + scan.table = table + scanner = Mock(skip_level0=True) + with patch.object(PythonTableScan, '_create_file_scanner', return_value=scanner) as create: + assert PythonTableScan(scan)._file_scanner() is scanner + assert PythonTableScan(scan)._scanner(None) is scanner + create.assert_called_once() + configure.assert_called_once_with(scanner) + scanner.with_shard.assert_called_once_with(1, 3) + ranges, = scanner.with_row_ranges.call_args.args + assert [(value.from_, value.to) for value in ranges] == [(7, 9)] + assert scan._python_file_scanner is scanner and not scanner.skip_level0 diff --git a/paimon-python/pypaimon/tests/reader_base_test.py b/paimon-python/pypaimon/tests/reader_base_test.py index cddd66bec8f4..1f60dc382051 100644 --- a/paimon-python/pypaimon/tests/reader_base_test.py +++ b/paimon-python/pypaimon/tests/reader_base_test.py @@ -29,7 +29,9 @@ import pytest from parameterized import parameterized +from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.manifest.manifest_file_manager import ManifestFileManager from pypaimon.manifest.schema.data_file_meta import DataFileMeta from pypaimon.manifest.schema.manifest_entry import ManifestEntry @@ -301,9 +303,10 @@ def test_full_data_types(self, stats_mode): # to test GenericRow ability latest_snapshot = table.snapshot_manager().get_latest_snapshot() - manifest_files = table_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot) - manifest_entries = table_scan.file_scanner.manifest_file_manager.read( - manifest_files[0].file_name, lambda row: table_scan.file_scanner._filter_manifest_entry(row), False) + file_scanner = PythonTableScan(table_scan)._file_scanner() + manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot) + manifest_entries = file_scanner.manifest_file_manager.read( + manifest_files[0].file_name, lambda row: file_scanner._filter_manifest_entry(row), False) if stats_mode == 'full': self.assertEqual(manifest_entries[0].file.value_stats_cols, None) @@ -651,10 +654,11 @@ def test_primary_key_value_stats_excludes_system_fields(self): pk_read_builder = pk_table.new_read_builder() pk_table_scan = pk_read_builder.new_scan() latest_snapshot = pk_table.snapshot_manager().get_latest_snapshot() - pk_manifest_files = pk_table_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot) - pk_manifest_entries = pk_table_scan.file_scanner.manifest_file_manager.read( + pk_file_scanner = PythonTableScan(pk_table_scan)._file_scanner() + pk_manifest_files = pk_file_scanner.manifest_list_manager.read_all(latest_snapshot) + pk_manifest_entries = pk_file_scanner.manifest_file_manager.read( pk_manifest_files[0].file_name, - lambda row: pk_table_scan.file_scanner._filter_manifest_entry(row), + lambda row: pk_file_scanner._filter_manifest_entry(row), False ) @@ -668,7 +672,7 @@ def test_primary_key_value_stats_excludes_system_fields(self): f"table.fields should NOT contain system fields, but got: {pk_table_field_names}") if pk_file_meta.value_stats_cols is None: - pk_value_stats_fields = pk_table_scan.file_scanner.manifest_file_manager._get_value_stats_fields( + pk_value_stats_fields = pk_file_scanner.manifest_file_manager._get_value_stats_fields( {'_VALUE_STATS_COLS': None}, pk_table.fields ) @@ -726,10 +730,11 @@ def test_value_stats_empty_when_stats_disabled(self): read_builder = table.new_read_builder() table_scan = read_builder.new_scan() latest_snapshot = table.snapshot_manager().get_latest_snapshot() - manifest_files = table_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot) - manifest_entries = table_scan.file_scanner.manifest_file_manager.read( + file_scanner = PythonTableScan(table_scan)._file_scanner() + manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot) + manifest_entries = file_scanner.manifest_file_manager.read( manifest_files[0].file_name, - lambda row: table_scan.file_scanner._filter_manifest_entry(row), + lambda row: file_scanner._filter_manifest_entry(row), False ) @@ -1239,10 +1244,11 @@ def test_primary_key_value_stats(self): read_builder = table.new_read_builder() table_scan = read_builder.new_scan() latest_snapshot = table.snapshot_manager().get_latest_snapshot() - manifest_files = table_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot) - manifest_entries = table_scan.file_scanner.manifest_file_manager.read( + file_scanner = PythonTableScan(table_scan)._file_scanner() + manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot) + manifest_entries = file_scanner.manifest_file_manager.read( manifest_files[0].file_name, - lambda row: table_scan.file_scanner._filter_manifest_entry(row), + lambda row: file_scanner._filter_manifest_entry(row), False ) @@ -1295,7 +1301,7 @@ def test_primary_key_value_stats(self): self.assertFalse(is_system_field, f"value_stats_cols should not contain system field: {field_name}") - value_stats_fields = table_scan.file_scanner.manifest_file_manager._get_value_stats_fields( + value_stats_fields = file_scanner.manifest_file_manager._get_value_stats_fields( {'_VALUE_STATS_COLS': file_meta.value_stats_cols}, table.fields ) @@ -1517,7 +1523,7 @@ def test_read_batch_size_config(self): if splits: # Use _create_split_read to create reader - split_read = table_read._create_split_read(splits[0]) + split_read = PythonTableRead(table_read)._create_split_read(splits[0]) reader = split_read.create_reader() batch_count = 0 total_rows = 0 diff --git a/paimon-python/pypaimon/tests/reader_parallel_test.py b/paimon-python/pypaimon/tests/reader_parallel_test.py index 3e99e6fb77a5..32d6ade20d3f 100644 --- a/paimon-python/pypaimon/tests/reader_parallel_test.py +++ b/paimon-python/pypaimon/tests/reader_parallel_test.py @@ -27,6 +27,7 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.read.table_read import TableRead from pypaimon.execution.python.read.table_read import _RemainingRows @@ -248,7 +249,8 @@ def test_default_none_auto_takes_parallel_path_when_multicore(self): if (os.cpu_count() or 1) < 2: self.skipTest("single-core runner: auto resolves to serial") with mock.patch.object( - read, '_to_arrow_parallel', wraps=read._to_arrow_parallel + PythonTableRead, '_to_arrow_parallel', autospec=True, + side_effect=PythonTableRead._to_arrow_parallel ) as spy: read.to_arrow(splits) spy.assert_called_once() @@ -260,7 +262,7 @@ def test_default_none_auto_takes_parallel_path_when_multicore(self): def test_method_arg_overrides_option_to_serial(self): # option=4 but caller passes 1: should disable parallelism. read = self.table_opt_4.new_read_builder().new_read() - with mock.patch.object(read, '_to_arrow_parallel') as patched: + with mock.patch.object(PythonTableRead, '_to_arrow_parallel') as patched: patched.side_effect = AssertionError( "_to_arrow_parallel should not be called when arg=1 overrides option") splits = self._scan_splits(self.table_opt_4.new_read_builder()) @@ -272,7 +274,8 @@ def test_method_arg_overrides_option_to_parallel(self): read = self.table_opt_1.new_read_builder().new_read() splits = self._scan_splits(self.table_opt_1.new_read_builder()) with mock.patch.object( - read, '_to_arrow_parallel', wraps=read._to_arrow_parallel + PythonTableRead, '_to_arrow_parallel', autospec=True, + side_effect=PythonTableRead._to_arrow_parallel ) as spy: result = read.to_arrow(splits, parallelism=4) spy.assert_called_once() @@ -359,7 +362,7 @@ def test_parallel_reader_error_propagates(self): splits = self._scan_splits(rb) self.assertGreaterEqual(len(splits), 2) - original_create = TableRead._create_split_read + original_create = PythonTableRead._create_split_read call_counter = {'n': 0} lock = threading.Lock() @@ -371,7 +374,7 @@ def flaky(self_, split, blob_parallelism=1): raise RuntimeError("simulated reader failure") return original_create(self_, split, blob_parallelism) - with mock.patch.object(TableRead, '_create_split_read', flaky): + with mock.patch.object(PythonTableRead, '_create_split_read', flaky): read = rb.new_read() with self.assertRaises(RuntimeError) as ctx: read.to_pandas(splits, parallelism=4) diff --git a/paimon-python/pypaimon/tests/reader_primary_key_test.py b/paimon-python/pypaimon/tests/reader_primary_key_test.py index 9d81586c5571..fa79f4950c50 100644 --- a/paimon-python/pypaimon/tests/reader_primary_key_test.py +++ b/paimon-python/pypaimon/tests/reader_primary_key_test.py @@ -24,6 +24,7 @@ import pyarrow as pa +from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon import CatalogFactory, Schema from pypaimon.schema.arrow_schema import read_arrow_schema from pypaimon.common.options.core_options import CoreOptions @@ -549,10 +550,11 @@ def test_manifest_creation_time_timestamp(self): latest_snapshot = snapshot_manager.get_latest_snapshot() read_builder = table.new_read_builder() table_scan = read_builder.new_scan() - manifest_list_manager = table_scan.file_scanner.manifest_list_manager + file_scanner = PythonTableScan(table_scan)._file_scanner() + manifest_list_manager = file_scanner.manifest_list_manager manifest_files = manifest_list_manager.read_all(latest_snapshot) - manifest_file_manager = table_scan.file_scanner.manifest_file_manager + manifest_file_manager = file_scanner.manifest_file_manager creation_times_found = [] for manifest_file_meta in manifest_files: entries = manifest_file_manager.read(manifest_file_meta.file_name, drop_stats=False) diff --git a/paimon-python/pypaimon/tests/scan_request_test.py b/paimon-python/pypaimon/tests/scan_request_test.py index 155646b97dbb..5e47c0a3e361 100644 --- a/paimon-python/pypaimon/tests/scan_request_test.py +++ b/paimon-python/pypaimon/tests/scan_request_test.py @@ -61,7 +61,6 @@ def _scan(): scan.limit = 10 scan.partition_predicate = None scan._read_type = [DataField(0, 'pt', AtomicType('INT'))] - scan._incremental_snapshot_range = (2, 4) return scan @@ -86,7 +85,6 @@ def test_request_recaptures_external_configuration_and_authorization_changes(): assert not before.auth_has_non_partition_filter and after.auth_has_non_partition_filter assert (after.idx_of_this_subtask, after.number_of_para_subtasks) == (1, 3) assert len(before.read_type) == 1 and len(after.read_type) == 2 - assert after.incremental_snapshot_range == (2, 4) def test_native_prepare_observes_the_latest_public_scanner_configuration(): diff --git a/paimon-python/pypaimon/tests/scanner/file_scanner_schema_fields_test.py b/paimon-python/pypaimon/tests/scanner/file_scanner_schema_fields_test.py index 59a6eae7cd69..a58fecf1ec0c 100644 --- a/paimon-python/pypaimon/tests/scanner/file_scanner_schema_fields_test.py +++ b/paimon-python/pypaimon/tests/scanner/file_scanner_schema_fields_test.py @@ -21,6 +21,7 @@ import pyarrow as pa import pytest +from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon import CatalogFactory, Schema from pypaimon.manifest.manifest_list_manager import ManifestListManager from pypaimon.execution.python.read.scanner.file_scanner import (FileScanner) @@ -121,7 +122,7 @@ def test_transported_query_auth_files_retain_snapshot_statistics(tmp_path, guard table = table.copy({'query-auth.enabled': str(guarded).lower()}) snapshot = table.snapshot_manager().get_latest_snapshot() manifests = ManifestListManager(table).read_all(snapshot) - scanner = table.new_read_builder().new_scan().file_scanner + scanner = PythonTableScan(table.new_read_builder().new_scan())._file_scanner() entries = scanner.read_manifest_entries(manifests) assert len(entries) == 1 # Core validates the complete transported metadata against the snapshot. diff --git a/paimon-python/pypaimon/tests/schema_evolution_pk_read_test.py b/paimon-python/pypaimon/tests/schema_evolution_pk_read_test.py index 614c3c5d1fe3..28f2adbb7bbb 100644 --- a/paimon-python/pypaimon/tests/schema_evolution_pk_read_test.py +++ b/paimon-python/pypaimon/tests/schema_evolution_pk_read_test.py @@ -247,7 +247,7 @@ def test_pk_first_row_add_column(self): builder = table.new_read_builder() scan = builder.new_scan() self.assertEqual(scan.plan().splits(), []) - rows = builder.new_read().to_arrow(scan.plan_all_files().splits()).to_pylist() + rows = builder.new_read().to_arrow(scan.plan_all().splits()).to_pylist() self.assertEqual(rows, [ {'id': 1, 'v': 'first', 'w': None}]) diff --git a/paimon-python/pypaimon/tests/schema_evolution_read_test.py b/paimon-python/pypaimon/tests/schema_evolution_read_test.py index 144129cb5231..b9308979fa4c 100644 --- a/paimon-python/pypaimon/tests/schema_evolution_read_test.py +++ b/paimon-python/pypaimon/tests/schema_evolution_read_test.py @@ -25,6 +25,7 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon.schema.data_types import AtomicType from pypaimon.schema.schema_change import Move, SchemaChange @@ -432,14 +433,14 @@ def test_schema_evolution_with_scan_filter(self): schema_manager.commit(TableSchema.from_schema(schema_id=0, schema=schema)) schema_manager.commit(TableSchema.from_schema(schema_id=1, schema=schema2)) # scan filter for schema evolution - latest_snapshot = table1.new_read_builder().new_scan().file_scanner.snapshot_manager.get_latest_snapshot() + latest_snapshot = table1.snapshot_manager().get_latest_snapshot() table2.table_path = table1.table_path new_read_buidler = table2.new_read_builder() predicate_builder = new_read_buidler.new_predicate_builder() predicate = predicate_builder.less_than('user_id', 3) new_scan = new_read_buidler.with_filter(predicate).new_scan() - manifest_files = new_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot) - entries = new_scan.file_scanner.read_manifest_entries(manifest_files) + manifest_files = PythonTableScan(new_scan)._file_scanner().manifest_list_manager.read_all(latest_snapshot) + entries = PythonTableScan(new_scan)._file_scanner().read_manifest_entries(manifest_files) self.assertEqual(1, len(entries)) # verify scan filter success for schema evolution @pytest.mark.python_plan diff --git a/paimon-python/pypaimon/tests/stream_execution_protocol_test.py b/paimon-python/pypaimon/tests/stream_execution_protocol_test.py index 6ea26fc2c197..4eeea379d0db 100644 --- a/paimon-python/pypaimon/tests/stream_execution_protocol_test.py +++ b/paimon-python/pypaimon/tests/stream_execution_protocol_test.py @@ -278,7 +278,7 @@ def test_prefetch_failure_is_not_hidden_or_advanced(tmp_path): future = Future() failure = OSError('prefetch manifest unavailable') future.set_exception(failure) - scan._prefetch_future = future + scan._execution._prefetch_future = future try: with pytest.raises(OSError) as caught: scan.plan() @@ -303,7 +303,7 @@ def result(): return original_result(timeout=5) future.result = result - scan._prefetch_future = future + scan._execution._prefetch_future = future try: with ThreadPoolExecutor(max_workers=1) as executor: planning = executor.submit(scan.plan) @@ -314,7 +314,7 @@ def result(): planning.result(timeout=5) assert scan.checkpoint() == 7 assert scan.watermark() is None - assert scan._prefetch_future is None + assert scan._execution._prefetch_future is None finally: scan.close() diff --git a/paimon-python/pypaimon/tests/stream_initial_level_filter_test.py b/paimon-python/pypaimon/tests/stream_initial_level_filter_test.py index 15c309d82ead..d6140bd02359 100644 --- a/paimon-python/pypaimon/tests/stream_initial_level_filter_test.py +++ b/paimon-python/pypaimon/tests/stream_initial_level_filter_test.py @@ -23,6 +23,7 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan from pypaimon.manifest.manifest_file_manager import ManifestFileManager from pypaimon.manifest.manifest_list_manager import ManifestListManager @@ -77,7 +78,7 @@ def test_initial_frame_selects_only_published_changelog_levels(tmp_path, produce table, snapshot = _initial_level_table(tmp_path, [0, 1, 2, 5, 2147483646]) table = table.copy(dict(options, **{'changelog-producer': producer})) builder = table.new_stream_read_builder() - scan = builder.new_streaming_scan() + scan = AsyncStreamingTableScan(table) try: plan = scan._create_initial_plan(snapshot) assert plan.snapshot_id == snapshot.id @@ -85,7 +86,7 @@ def test_initial_frame_selects_only_published_changelog_levels(tmp_path, produce assert sorted(row['id'] for row in builder.new_read().to_arrow(plan.splits()).to_pylist()) == expected assert all(not split.is_streaming for split in plan.splits()) finally: - scan._prefetch_executor.shutdown(wait=True) + scan.close() @pytest.mark.parametrize('producer', ['lookup', 'full-compaction']) @@ -93,11 +94,11 @@ def test_initial_frame_with_only_level_zero_is_empty(tmp_path, producer): table, snapshot = _initial_level_table(tmp_path, [0]) table = table.copy({'changelog-producer': producer}) builder = table.new_stream_read_builder() - scan = builder.new_streaming_scan() + scan = AsyncStreamingTableScan(table) try: plan = scan._create_initial_plan(snapshot) assert plan.snapshot_id == snapshot.id assert plan.splits() == [] assert builder.new_read().to_arrow(plan.splits()).num_rows == 0 finally: - scan._prefetch_executor.shutdown(wait=True) + scan.close() diff --git a/paimon-python/pypaimon/tests/stream_read_builder_test.py b/paimon-python/pypaimon/tests/stream_read_builder_test.py index 0e4f74b5a0f7..a0ab43fc7e43 100644 --- a/paimon-python/pypaimon/tests/stream_read_builder_test.py +++ b/paimon-python/pypaimon/tests/stream_read_builder_test.py @@ -24,7 +24,8 @@ from pypaimon.read.read_type import OutputProjection from pypaimon.read.stream_read_builder import StreamReadBuilder -from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan +from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan as PythonStreamScan +from pypaimon.read.streaming_table_scan import StreamTableScan from pypaimon.schema.data_types import AtomicType, DataField @@ -41,6 +42,8 @@ def mock_table(): table = MagicMock() table.fields = [] table.options.row_tracking_enabled.return_value = False + table.options.native_plan_enabled.return_value = False + table.options.native_read_enabled.return_value = False return table @@ -59,6 +62,9 @@ def mock_scan_table(): table.table_path = "/tmp/test" table.fields = [] table.options.row_tracking_enabled.return_value = False + table.options.native_plan_enabled.return_value = False + table.options.native_read_enabled.return_value = False + table.catalog_environment.table_query_auth.return_value = None return table @@ -126,7 +132,7 @@ class TestAsyncStreamingTableScanFiltering: def test_filter_with_bucket_filter(self, mock_scan_table): """Test _filter_entries_for_shard with custom bucket filter.""" - scan = AsyncStreamingTableScan( + scan = PythonStreamScan( table=mock_scan_table, bucket_filter=lambda b: b % 2 == 0 ) @@ -136,7 +142,7 @@ def test_filter_with_bucket_filter(self, mock_scan_table): def test_filter_no_filter_returns_all(self, mock_scan_table): """Test _filter_entries_for_shard with no filter returns all entries.""" - scan = AsyncStreamingTableScan(table=mock_scan_table) + scan = PythonStreamScan(table=mock_scan_table) entries = [MockEntry(b) for b in range(8)] filtered = scan._filter_entries_for_shard(entries) assert [e.bucket for e in filtered] == list(range(8)) @@ -153,7 +159,7 @@ def test_new_streaming_scan_passes_consumer_id(self, mock_scan_table): scan = builder.new_streaming_scan() assert scan._consumer_id == "test-consumer" - assert scan._consumer_manager is not None + assert scan._execution._consumer_manager is not None def test_new_streaming_scan_no_consumer_by_default(self, mock_scan_table): """Without with_consumer_id(), scan should have no consumer.""" @@ -162,7 +168,7 @@ def test_new_streaming_scan_no_consumer_by_default(self, mock_scan_table): scan = builder.new_streaming_scan() assert scan._consumer_id is None - assert scan._consumer_manager is None + assert scan._execution._consumer_manager is None if __name__ == '__main__': @@ -175,11 +181,14 @@ def test_optional_native_initialization_falls_back(self, mock_scan_table): builder = StreamReadBuilder(mock_scan_table) with patch('pypaimon.execution.native.read.streaming_table_scan.StreamTableScan', side_effect=ImportError('optional native runtime unavailable')): - assert isinstance(builder.new_streaming_scan(), AsyncStreamingTableScan) + scan = builder.new_streaming_scan() + assert isinstance(scan, StreamTableScan) + assert isinstance(scan._execution, PythonStreamScan) def test_native_planning_failure_is_not_retried_with_python(self, mock_scan_table): from pypaimon.read.streaming_table_scan import StreamTableScan failure = LookupError('native planning failed') + mock_scan_table.options.native_plan_enabled.return_value = True native = MagicMock() native.plan.side_effect = failure with patch('pypaimon.execution.native.read.streaming_table_scan.native_stream_scan', return_value=native): diff --git a/paimon-python/pypaimon/tests/streaming_scan_import_boundary_test.py b/paimon-python/pypaimon/tests/streaming_scan_import_boundary_test.py index a54c437b5025..cf583342b838 100644 --- a/paimon-python/pypaimon/tests/streaming_scan_import_boundary_test.py +++ b/paimon-python/pypaimon/tests/streaming_scan_import_boundary_test.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Importing a public streaming class loads only its own execution backend.""" +"""Public streaming classes do not load an execution backend on import.""" import subprocess import sys @@ -23,9 +23,6 @@ import pytest -pytestmark = pytest.mark.skipif( - sys.version_info < (3, 7), reason='Module-level lazy exports require Python 3.7') - def _run(source, *args): result = subprocess.run( @@ -35,42 +32,27 @@ def _run(source, *args): assert result.returncode == 0, result.stdout + result.stderr -@pytest.mark.parametrize('export, backend, forbidden', [ - ('StreamTableScan', 'native', 'python'), - ('AsyncStreamingTableScan', 'python', 'native'), -]) -def test_public_streaming_class_imports_only_its_backend(export, backend, forbidden): +@pytest.mark.parametrize('export', ['StreamTableScan', 'AsyncStreamingTableScan']) +def test_public_streaming_class_imports_without_either_backend(export): _run(""" import importlib import importlib.abc - import inspect import pickle import sys - export, backend, forbidden = sys.argv[1:] - blocked = 'pypaimon.execution.' + forbidden + export = sys.argv[1] - class RejectOtherBackend(importlib.abc.MetaPathFinder): + class RejectBackends(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path=None, target=None): - if fullname == blocked or fullname.startswith(blocked + '.'): - raise AssertionError('Unexpected backend import: ' + fullname) + if fullname.startswith(('pypaimon.execution.python', 'pypaimon.execution.native')): + raise AssertionError('Unexpected execution import: ' + fullname) - sys.meta_path.insert(0, RejectOtherBackend()) - if backend == 'native': - from pypaimon.read.streaming_table_scan import StreamTableScan as public_class - else: - from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan as public_class + sys.meta_path.insert(0, RejectBackends()) module = importlib.import_module('pypaimon.read.streaming_table_scan') - direct = importlib.import_module( - 'pypaimon.execution.' + backend + '.read.streaming_table_scan') - canonical_class = getattr(direct, export) - assert public_class is canonical_class - assert getattr(module, export) is canonical_class - assert inspect.signature(public_class) == inspect.signature(canonical_class) - assert pickle.loads(pickle.dumps(public_class)) is canonical_class - assert not any(name == blocked or name.startswith(blocked + '.') - for name in sys.modules) - """, export, backend, forbidden) + public_class = getattr(module, export) + assert public_class.__module__ == module.__name__ + assert pickle.loads(pickle.dumps(public_class)) is public_class + """, export) def test_streaming_exports_are_discoverable_without_loading_either_backend(): @@ -90,7 +72,7 @@ def find_spec(self, fullname, path=None, target=None): exports = ['AsyncStreamingTableScan', 'StreamTableScan'] assert module.__all__ == exports assert all(name in dir(module) for name in exports) - assert all(name not in module.__dict__ for name in exports) + assert all(getattr(module, name).__module__ == module.__name__ for name in exports) try: module.unknown_export except AttributeError as error: diff --git a/paimon-python/pypaimon/tests/streaming_table_scan_test.py b/paimon-python/pypaimon/tests/streaming_table_scan_test.py index f782895899ca..7a6cb235e586 100644 --- a/paimon-python/pypaimon/tests/streaming_table_scan_test.py +++ b/paimon-python/pypaimon/tests/streaming_table_scan_test.py @@ -23,7 +23,7 @@ from pypaimon.common.options.core_options import ChangelogProducer from pypaimon.read.plan import Plan -from pypaimon.read.streaming_table_scan import AsyncStreamingTableScan +from pypaimon.execution.python.read.streaming_table_scan import AsyncStreamingTableScan from pypaimon.snapshot.snapshot import Snapshot diff --git a/paimon-python/pypaimon/tests/table_query_auth_test.py b/paimon-python/pypaimon/tests/table_query_auth_test.py index 42c9d5646df1..2fde85b57356 100644 --- a/paimon-python/pypaimon/tests/table_query_auth_test.py +++ b/paimon-python/pypaimon/tests/table_query_auth_test.py @@ -390,7 +390,7 @@ def test_no_restriction_returns_plan_unchanged(self): plain_plan = Plan([_FakeSplit()], snapshot_id=5) scan = _FakeScan(plain_plan, auth_result=None) - result = TableScan.plan_all_files(scan) + result = TableScan.plan_all(scan) self.assertIs(result, plain_plan) def test_row_filter_restriction_raises(self): @@ -400,7 +400,7 @@ def test_row_filter_restriction_raises(self): plan = Plan([_FakeSplit()], snapshot_id=5) scan = _FakeScan(plan, auth_result=auth) with self.assertRaises(TableNoPermissionException): - TableScan.plan_all_files(scan) + TableScan.plan_all(scan) def test_column_masking_restriction_raises(self): from pypaimon.read.table_scan import TableScan @@ -409,7 +409,7 @@ def test_column_masking_restriction_raises(self): plan = Plan([_FakeSplit()], snapshot_id=5) scan = _FakeScan(plan, auth_result=auth) with self.assertRaises(TableNoPermissionException): - TableScan.plan_all_files(scan) + TableScan.plan_all(scan) def test_empty_table_with_auth_still_raises(self): from pypaimon.read.table_scan import TableScan @@ -418,14 +418,14 @@ def test_empty_table_with_auth_still_raises(self): empty_plan = Plan([], snapshot_id=5) scan = _FakeScan(empty_plan, auth_result=auth) with self.assertRaises(TableNoPermissionException): - TableScan.plan_all_files(scan) + TableScan.plan_all(scan) def test_no_restrictions_allows_write(self): from pypaimon.read.table_scan import TableScan plan = Plan([_FakeSplit()], snapshot_id=5) scan = _FakeScan(plan, auth_result=None) - result = TableScan.plan_all_files(scan) + result = TableScan.plan_all(scan) self.assertIs(result, plan) diff --git a/paimon-python/pypaimon/tests/table_read_chunked_overflow_test.py b/paimon-python/pypaimon/tests/table_read_chunked_overflow_test.py index cc13b8dbb4ac..c1cc8ac788fd 100644 --- a/paimon-python/pypaimon/tests/table_read_chunked_overflow_test.py +++ b/paimon-python/pypaimon/tests/table_read_chunked_overflow_test.py @@ -17,24 +17,24 @@ ################################################################################ import unittest +from types import SimpleNamespace from unittest.mock import patch import pyarrow as pa -from pypaimon.read import table_read -from pypaimon.read.table_read import ROW_KIND_COLUMN, TableRead +from pypaimon.execution.python.read import table_read +from pypaimon.read.constants import ROW_KIND_COLUMN +from pypaimon.execution.python.read.table_read import PythonTableRead # Patching table_read.pyarrow.array also rebinds pa.array (same module object), # so keep a handle to the genuine implementation for use inside the fakes. _REAL_ARRAY = pa.array -def _make_table_read(include_row_kind: bool) -> TableRead: +def _make_table_read(include_row_kind: bool) -> PythonTableRead: # The conversion method only depends on self.include_row_kind, so build a # bare instance instead of standing up a full catalog/table. - tr = TableRead.__new__(TableRead) - tr.include_row_kind = include_row_kind - return tr + return PythonTableRead(SimpleNamespace(include_row_kind=include_row_kind)) class TableReadChunkedOverflowTest(unittest.TestCase): diff --git a/paimon-python/pypaimon/tests/table_update_test.py b/paimon-python/pypaimon/tests/table_update_test.py index 4f832bdb82c4..c225b6db54ff 100644 --- a/paimon-python/pypaimon/tests/table_update_test.py +++ b/paimon-python/pypaimon/tests/table_update_test.py @@ -378,7 +378,7 @@ def test_update_by_predicate_processes_one_file_group_at_a_time(self): pa.Table.from_pydict({'_ROW_ID': [0], 'age': [26]}), ['age'], ) - splits = table.new_read_builder().new_scan().plan_all_files().splits() + splits = table.new_read_builder().new_scan().plan_all().splits() self.assertEqual(1, len(splits)) group_sizes = [] diff --git a/paimon-python/pypaimon/tests/table_upsert_by_key_test.py b/paimon-python/pypaimon/tests/table_upsert_by_key_test.py index 8ffcfb5dd355..824b04b89c47 100644 --- a/paimon-python/pypaimon/tests/table_upsert_by_key_test.py +++ b/paimon-python/pypaimon/tests/table_upsert_by_key_test.py @@ -295,7 +295,7 @@ def _compact_all_data_files(self, table): read_builder = table.new_read_builder().with_projection( list(table.field_names) + [SpecialFields.ROW_ID.name] ) - plan = read_builder.new_scan().plan_all_files() + plan = read_builder.new_scan().plan_all() old_files = [ file for split in plan.splits() diff --git a/paimon-python/pypaimon/tests/test_first_row_e2e.py b/paimon-python/pypaimon/tests/test_first_row_e2e.py index d8c9910624dd..8db4c72eda33 100644 --- a/paimon-python/pypaimon/tests/test_first_row_e2e.py +++ b/paimon-python/pypaimon/tests/test_first_row_e2e.py @@ -84,7 +84,7 @@ def _read(self, table): rb = table.new_read_builder() scan = rb.new_scan() self.assertEqual(scan.plan().splits(), []) - splits = scan.plan_all_files().splits() + splits = scan.plan_all().splits() if not splits: return [] return sorted( diff --git a/paimon-python/pypaimon/tests/test_limit_pushdown.py b/paimon-python/pypaimon/tests/test_limit_pushdown.py index b3b941d8a0be..035f977f7bc9 100644 --- a/paimon-python/pypaimon/tests/test_limit_pushdown.py +++ b/paimon-python/pypaimon/tests/test_limit_pushdown.py @@ -31,6 +31,7 @@ import pyarrow as pa from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_read import PythonTableRead class LimitPushdownTest(unittest.TestCase): @@ -223,7 +224,7 @@ def test_append_only_split_read_creates_limited_batch_reader(self): splits = rb.new_scan().plan().splits() self.assertGreater(len(splits), 0) for split in splits: - split_read = table_read._create_split_read(split) + split_read = PythonTableRead(table_read)._create_split_read(split) self.assertEqual(split_read.limit, 3) reader = split_read.create_reader() self.assertIsInstance(reader, LimitedRecordBatchReader, @@ -242,7 +243,7 @@ def test_append_only_split_read_limit_truncates_within_split(self): table_read = rb.new_read() splits = rb.new_scan().plan().splits() self.assertEqual(len(splits), 1) - split_read = table_read._create_split_read(splits[0]) + split_read = PythonTableRead(table_read)._create_split_read(splits[0]) reader = split_read.create_reader() # Drain the reader directly, bypassing TableRead-level control total_rows = 0 diff --git a/paimon-python/pypaimon/tests/test_partial_update_e2e.py b/paimon-python/pypaimon/tests/test_partial_update_e2e.py index e82cff1440ae..076afc583c3f 100644 --- a/paimon-python/pypaimon/tests/test_partial_update_e2e.py +++ b/paimon-python/pypaimon/tests/test_partial_update_e2e.py @@ -286,7 +286,7 @@ def test_first_row_engine_keeps_first(self): self.assertEqual(self._read(table), []) builder = table.new_read_builder() self.assertEqual( - builder.new_read().to_arrow(builder.new_scan().plan_all_files().splits()).to_pylist(), + builder.new_read().to_arrow(builder.new_scan().plan_all().splits()).to_pylist(), [{'id': 1, 'a': 'first', 'b': None, 'c': None}], ) diff --git a/paimon-python/pypaimon/tests/test_read_builder_nested_projection.py b/paimon-python/pypaimon/tests/test_read_builder_nested_projection.py index 8fc2cd7dea84..2c4d12804497 100644 --- a/paimon-python/pypaimon/tests/test_read_builder_nested_projection.py +++ b/paimon-python/pypaimon/tests/test_read_builder_nested_projection.py @@ -24,6 +24,7 @@ import pyarrow as pa from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.read.table_read import PythonTableRead from pypaimon.read.datasource.split_provider import ( CatalogSplitProvider, PreResolvedSplitProvider) from pypaimon.read.read_builder import ReadBuilder @@ -320,7 +321,7 @@ def test_named_duplicate_columns_after_primary_key_merge(self): self.assertEqual( expected, read.to_arrow_batch_reader( splits, parallelism=1).read_all().to_pydict()) - batches = read._read_one_split_to_batches( + batches = PythonTableRead(read)._read_one_split_to_batches( splits[0], read._output_arrow_schema(), _RemainingRows(None)) self.assertEqual(expected, pa.Table.from_batches(batches).to_pydict()) diff --git a/paimon-python/pypaimon/tests/write/distributed_input_execution_test.py b/paimon-python/pypaimon/tests/write/distributed_input_execution_test.py index 6acedebdd3bc..d1ed56bdceef 100644 --- a/paimon-python/pypaimon/tests/write/distributed_input_execution_test.py +++ b/paimon-python/pypaimon/tests/write/distributed_input_execution_test.py @@ -107,7 +107,7 @@ def _seed_snapshot(table): writer.close() commit.close() snapshot = table.snapshot_manager().get_latest_snapshot() - splits = pure.new_read_builder().new_scan().plan_all_files().splits() + splits = pure.new_read_builder().new_scan().plan_all().splits() plan = Plan([deserialize_split_v1( serialize_split_v1(split), table.partition_keys_fields, table.trimmed_primary_keys_fields) for split in splits]) diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py b/paimon-python/pypaimon/tests/write/table_write_test.py index 8059997abc4e..6b7743fab09e 100644 --- a/paimon-python/pypaimon/tests/write/table_write_test.py +++ b/paimon-python/pypaimon/tests/write/table_write_test.py @@ -24,6 +24,7 @@ from contextlib import contextmanager from unittest.mock import Mock, patch +from pypaimon.execution.python.read.table_scan import PythonTableScan from pypaimon import CatalogFactory, Schema from pypaimon.schema.arrow_schema import read_arrow_schema import pyarrow as pa @@ -1515,7 +1516,7 @@ def test_postpone_legacy_partition_migrates_to_fixed_bucket(self): self.assertEqual({2}, {m.total_buckets for m in messages}) fixed_commit.commit(messages) - scanner = fixed.new_read_builder().new_scan().file_scanner + scanner = PythonTableScan(fixed.new_read_builder().new_scan())._file_scanner() manifests, _ = scanner.manifest_scanner() entries = scanner.manifest_file_manager.read_entries_parallel( manifests, drop_stats=False diff --git a/paimon-python/pypaimon/tests/write/trusted_scan_metadata_test.py b/paimon-python/pypaimon/tests/write/trusted_scan_metadata_test.py new file mode 100644 index 000000000000..68566cd204e3 --- /dev/null +++ b/paimon-python/pypaimon/tests/write/trusted_scan_metadata_test.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Python writer metadata lookups stay outside public query authorization.""" + +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import pytest + +from pypaimon.execution.python.read.table_scan import PythonTableScan +from pypaimon.execution.python.write.bucket_metadata import dynamic_bucket_data_buckets +from pypaimon.execution.python.write.file_store_write import FileStoreWrite +from pypaimon.execution.python.write.postpone_bucket import PythonPostponeBucketPlanner + + +@pytest.mark.parametrize('operation', ['sequence', 'dynamic', 'postpone']) +def test_trusted_metadata_lookup_uses_private_python_scanner(operation): + class Scan: + @property + def file_scanner(self): + raise AssertionError('Writer metadata entered public scanner dispatch') + + scan = Scan() + builder = Mock() + builder.new_scan.return_value = scan + builder.with_filter.return_value = builder + builder.new_predicate_builder.return_value = SimpleNamespace(and_predicates=lambda values: None) + table = SimpleNamespace( + partition_keys=[], new_read_builder=lambda: builder, + options=SimpleNamespace(scan_manifest_parallelism=lambda: 4)) + entries = [SimpleNamespace( + bucket=bucket, total_buckets=8, partition=SimpleNamespace(values=['a']), + file=SimpleNamespace(max_sequence_number=sequence)) + for bucket, sequence in [(3, 9), (4, 7)]] + scanner = Mock(skip_level0=True) + + def plan_files(): + assert scanner.skip_level0 is False + return entries + + scanner.plan_files.side_effect = plan_files + scanner.manifest_scanner.return_value = (['manifest'], None) + scanner.manifest_file_manager.read_entries_parallel.return_value = entries + with patch.object(PythonTableScan, '_file_scanner', return_value=scanner) as private, \ + patch.object(PythonTableScan, '_scanner', side_effect=AssertionError('SELECT authorization')): + if operation == 'sequence': + writer = object.__new__(FileStoreWrite) + writer.table = table + writer._sequence_read_table = lambda: table + assert writer._load_seq_number_stats(()) == {3: 9, 4: 7} + elif operation == 'dynamic': + with patch('pypaimon.index.dynamic_bucket.bucket_read_builder', return_value=builder): + assert dynamic_bucket_data_buckets(table, (), object()) == {3, 4} + else: + assert PythonPostponeBucketPlanner._load_bucket_metadata(table) == {('a',): 8} + scanner.manifest_file_manager.read_entries_parallel.assert_called_once_with( + ['manifest'], max_workers=4) + private.assert_called_once_with() diff --git a/paimon-python/pypaimon/write/update_validation.py b/paimon-python/pypaimon/write/update_validation.py index b07c1fc27cea..897dc412fd3b 100644 --- a/paimon-python/pypaimon/write/update_validation.py +++ b/paimon-python/pypaimon/write/update_validation.py @@ -25,7 +25,7 @@ def _authorize_update(table): - """Require the same whole-table grant as plan_all_files for fast paths.""" + """Require the same whole-table grant as plan_all for fast paths.""" from pypaimon.catalog.catalog_exception import TableNoPermissionException from pypaimon.read.auth_support import authorize