From 06463862a3fc4a5e3b9002896c7a90d6b6282bae Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 22 Oct 2026 19:54:32 +0800 Subject: [PATCH 1/6] [Python] Dispatch row-id mutations and isolate write executors --- paimon-python/pypaimon/execution/README.md | 13 ++ .../execution/native/write/native_update.py | 15 ++ .../execution/native/write/postpone_bucket.py | 28 ++- .../execution/python/write/merge_into.py | 6 +- .../write/postpone_batch_table_write.py | 33 ++-- .../execution/python/write/table_delete.py | 70 ++++---- .../execution/python/write/table_update.py | 80 +++++---- .../python/write/table_update_by_row_id.py | 163 ++++++++---------- .../python/write/table_upsert_by_key.py | 41 +++-- .../execution/python/write/table_write.py | 53 +++--- .../pypaimon/ray/data_evolution_merge_join.py | 6 +- .../pypaimon/tests/arrow_schema_test.py | 3 +- .../tests/deletion_vector_path_test.py | 4 +- .../tests/native_authorized_update_test.py | 7 +- .../pypaimon/tests/native_blob_update_test.py | 4 +- .../tests/native_branch_write_test.py | 8 +- .../tests/native_data_directory_test.py | 4 +- .../tests/native_decimal_predicate_test.py | 4 +- .../tests/native_historical_update_test.py | 7 +- .../tests/native_map_shredding_write_test.py | 7 +- .../tests/native_nested_update_test.py | 6 +- .../native_parquet_vector_update_test.py | 4 +- .../native_partition_delete_changelog_test.py | 4 +- .../tests/native_partition_types_test.py | 2 +- .../tests/native_plan_capabilities_test.py | 4 +- .../tests/native_plan_chunk_shuffle_test.py | 4 +- .../tests/native_plan_distribution_test.py | 6 +- .../tests/native_plan_materialized_pk_test.py | 8 +- .../tests/native_predicate_delete_test.py | 6 +- .../tests/native_shard_update_test.py | 40 +++++ .../tests/native_sliced_split_read_test.py | 4 +- .../native_update_container_casts_test.py | 4 +- ...tive_update_inactive_vector_option_test.py | 8 +- .../native_update_source_nullability_test.py | 11 +- .../pypaimon/tests/native_update_test.py | 42 ++--- .../tests/native_upsert_capabilities_test.py | 8 +- .../tests/native_upsert_encoded_keys_test.py | 6 +- .../native_upsert_key_normalization_test.py | 4 +- .../tests/native_upsert_layout_test.py | 4 +- .../native_upsert_row_key_identity_test.py | 15 +- .../tests/native_upsert_row_shapes_test.py | 4 +- .../native_upsert_selected_values_test.py | 4 +- .../native_upsert_temporal_row_keys_test.py | 12 +- .../tests/native_variant_update_test.py | 4 +- .../pypaimon/tests/native_video_read_test.py | 2 +- .../tests/native_write_routing_test.py | 2 +- .../pypaimon/tests/reader_predicate_test.py | 4 +- .../tests/shard_table_updator_test.py | 3 +- .../table_update_arrow_compatibility_test.py | 8 +- .../table_update_by_row_id_chunked_test.py | 4 +- .../pypaimon/tests/table_update_test.py | 15 +- .../tests/table_upsert_by_key_test.py | 4 +- .../tests/upsert_btree_lookup_test.py | 3 +- .../tests/video_column_reassign_test.py | 8 +- .../tests/write/backend_boundary_test.py | 3 +- .../write/merge_execution_boundary_test.py | 63 +++++++ .../write/postpone_execution_boundary_test.py | 83 ++++++--- .../write/row_id_execution_boundary_test.py | 161 +++++++++++++++++ .../tests/write/row_kind_nullability_test.py | 13 +- .../pypaimon/tests/write/table_write_test.py | 12 +- .../write/update_backend_selection_test.py | 152 ++++++++++++++++ .../write/update_execution_protocol_test.py | 55 +++--- .../write/writer_execution_protocol_test.py | 70 +++++++- .../pypaimon/write/backend_dispatch.py | 54 +++++- .../write/postpone_batch_table_write.py | 18 -- .../pypaimon/write/postpone_bucket.py | 4 +- paimon-python/pypaimon/write/table_commit.py | 7 +- paimon-python/pypaimon/write/table_delete.py | 57 ++---- paimon-python/pypaimon/write/table_update.py | 129 ++++---------- .../pypaimon/write/table_update_by_row_id.py | 139 ++++----------- .../pypaimon/write/table_upsert_by_key.py | 58 ------- paimon-python/pypaimon/write/table_write.py | 41 ----- .../pypaimon/write/update_dispatch.py | 7 +- 73 files changed, 1108 insertions(+), 841 deletions(-) create mode 100644 paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py create mode 100644 paimon-python/pypaimon/tests/write/update_backend_selection_test.py diff --git a/paimon-python/pypaimon/execution/README.md b/paimon-python/pypaimon/execution/README.md index 2743b89010d4..822fe5e157ac 100644 --- a/paimon-python/pypaimon/execution/README.md +++ b/paimon-python/pypaimon/execution/README.md @@ -105,6 +105,19 @@ require unrestricted whole-table authorization because they expose raw file metadata. Internal writer metadata restoration uses the selected executor's private scanner and retains its write authorization boundary. +Direct `TableUpdateByRowId`, `TableDeleteByRowId` and `ShardTableUpdator` +construction also selects one execution backend before loading target files. +An operation that selected Python keeps its nested update/delete writers in +Python, including MERGE actions. Private schema, routing and rewrite algorithms +call their executor's helpers; public method overrides and user callbacks remain +public hooks. Broadcast file indexes retain their captured snapshot and are +consumed by the explicitly selected Python workers without per-worker rescans. + +Postpone bucket metadata is writer state. Native obtains it from the Core write +builder, preserving per-file layout validation and write authorization without +adding SELECT requirements. Shared input statistics and bucket arithmetic stay +outside either backend. + Manifest-source and follow-up-scanner callbacks belong to Python execution internals. Public scanner configuration selects the existing snapshot, delta or changelog strategies without accepting custom scanner callbacks. Explicit diff --git a/paimon-python/pypaimon/execution/native/write/native_update.py b/paimon-python/pypaimon/execution/native/write/native_update.py index 13a41dcb5de1..60b600230d64 100644 --- a/paimon-python/pypaimon/execution/native/write/native_update.py +++ b/paimon-python/pypaimon/execution/native/write/native_update.py @@ -451,6 +451,21 @@ def create_native_delete(table, commit_user): return NativeBatchTableUpdate(table, writer) +def create_native_row_id_delete(table): + """Adapt the direct deletion API to the existing core table updater.""" + operation = create_native_delete(table, table.new_batch_write_builder().commit_user) + return None if operation is None else NativeTableDeleteByRowId(operation) + + +class NativeTableDeleteByRowId: + + def __init__(self, operation): + self._operation = operation + + def delete(self, row_ids): + return self._operation.delete_by_row_id(row_ids) + + def _raise_native_row_id_error(error): detail = str(error) if 'duplicate UPDATE operations' in detail: diff --git a/paimon-python/pypaimon/execution/native/write/postpone_bucket.py b/paimon-python/pypaimon/execution/native/write/postpone_bucket.py index ed07f07c9eea..5ed29178bf10 100644 --- a/paimon-python/pypaimon/execution/native/write/postpone_bucket.py +++ b/paimon-python/pypaimon/execution/native/write/postpone_bucket.py @@ -17,8 +17,8 @@ """Read existing bucket layouts through Rust before distributed input work.""" -from pypaimon.execution.native.read.plan import native_plan from pypaimon.execution.native.runtime import native_runtime_available +from pypaimon.execution.native.write.native_commit import create_native_write_table from pypaimon.write.bucket_planning import PostponeBucketPlanning @@ -33,18 +33,14 @@ class NativePostponeBucketPlanner(PostponeBucketPlanning): @staticmethod def _load_bucket_metadata(table): - # PartitionStat does not carry total_buckets. Complete file planning - # preserves that layout in the shared Split v1 metadata without ever - # creating a Python scanner, reader or data writer. - known = {} - for split in native_plan(table, all_files=True).splits(): - if split.bucket < 0 or split.total_buckets is None or split.total_buckets <= 0: - continue - partition = tuple(split.partition.values) - previous = known.get(partition) - if previous is not None and previous != split.total_buckets: - raise RuntimeError( - "Partition {} has different total buckets {} and {}" - .format(partition, previous, split.total_buckets)) - known[partition] = split.total_buckets - return known + # Writer restoration uses the catalog's write identity. Query plans + # would add SELECT rules and can combine different physical layouts. + native_table = create_native_write_table(table) + if native_table is None: + raise RuntimeError('Native bucket metadata planning is unavailable') + plan = native_table.new_postpone_fixed_bucket_write_builder().bucket_plan() + # A partition key may itself be called total_buckets. Keep the + # positional schema used by the Core plan and with_bucket_plan input. + columns = [column.to_pylist() for column in plan.columns] + return {tuple(column[row] for column in columns[:-1]): columns[-1][row] + for row in range(plan.num_rows)} diff --git a/paimon-python/pypaimon/execution/python/write/merge_into.py b/paimon-python/pypaimon/execution/python/write/merge_into.py index e60defadc98c..04ce05a52966 100644 --- a/paimon-python/pypaimon/execution/python/write/merge_into.py +++ b/paimon-python/pypaimon/execution/python/write/merge_into.py @@ -477,9 +477,9 @@ def _prepare_commit_messages( update_snapshot_table = _copy_at_snapshot( table, base_snapshot.id if base_snapshot is not None else None ) - from pypaimon.write.write_builder import _new_update_by_row_id + from pypaimon.write.table_update_by_row_id import TableUpdateByRowId - updater = _new_update_by_row_id( + updater = TableUpdateByRowId._from_python( update_snapshot_table, commit_user, commit_identifier) update_msgs = updater.update_columns( update_table, list(update_cols_union) @@ -490,7 +490,7 @@ def _prepare_commit_messages( delete_snapshot_table = _copy_at_snapshot( table, base_snapshot.id if base_snapshot is not None else None ) - deleter = TableDeleteByRowId(delete_snapshot_table) + deleter = TableDeleteByRowId._from_python(delete_snapshot_table) row_id_name = SpecialFields.ROW_ID.name delete_msgs = deleter.delete( delete_table.column(row_id_name).to_pylist() diff --git a/paimon-python/pypaimon/execution/python/write/postpone_batch_table_write.py b/paimon-python/pypaimon/execution/python/write/postpone_batch_table_write.py index bf0531a97f91..255a8dc0a251 100644 --- a/paimon-python/pypaimon/execution/python/write/postpone_batch_table_write.py +++ b/paimon-python/pypaimon/execution/python/write/postpone_batch_table_write.py @@ -20,7 +20,7 @@ from pypaimon.write.commit_message import CommitMessage from pypaimon.execution.python.write.file_store_write import PostponeFixedBucketFileStoreWrite -from pypaimon.write.postpone_bucket import PostponeBucketPlanner +from pypaimon.execution.python.write.postpone_bucket import PythonPostponeBucketPlanner as PostponeBucketPlanner from pypaimon.execution.python.write.table_write import PythonTableWrite from pypaimon.execution.python.write.row_key_extractor import PostponeFixedBucketRowKeyExtractor from pypaimon.write.row_utils import ( @@ -55,7 +55,7 @@ def initialize( ) writer._plan_provided = bucket_plan is not None writer._pending_inputs = [] - PythonTableWrite(writer).initialize(table, commit_user, static_partition) + super().initialize(table, commit_user, static_partition) def _create_file_store_write(self, commit_user): writer = self.writer @@ -100,9 +100,9 @@ def write_arrow(self, table: pa.Table, bucket=None): writer = self.writer if bucket is not None: raise ValueError('Precomputed bucket writes require HASH_FIXED or HASH_DYNAMIC tables') - table = writer._prepare_arrow_data(table) - if not writer._buffer_input(table): - return PythonTableWrite(writer).write_arrow(table) + table = self._prepare_arrow_data(table) + if not self._buffer_input(table): + return super().write_arrow(table) writer._pending_inputs.extend( ("batch", batch) for batch in table.to_batches()) @@ -110,15 +110,15 @@ def write_arrow_batch(self, data: pa.RecordBatch, bucket=None): writer = self.writer if bucket is not None: raise ValueError('Precomputed bucket writes require HASH_FIXED or HASH_DYNAMIC tables') - data = writer._prepare_arrow_data(data) - if not writer._buffer_input(data): - return PythonTableWrite(writer).write_arrow_batch(data) + data = self._prepare_arrow_data(data) + if not self._buffer_input(data): + return super().write_arrow_batch(data) writer._pending_inputs.append(("batch", data)) def write_row(self, row): writer = self.writer if writer._plan_provided: - return PythonTableWrite(writer).write_row(row) + return super().write_row(row) values_by_name = row_to_named_values( row, writer.table.table_schema.fields) @@ -136,7 +136,7 @@ def write_row(self, row): partition = tuple( values_by_name[key] for key in writer.table.partition_keys) if writer._bucket_plan.contains(partition): - return PythonTableWrite(writer).write_row(row) + return super().write_row(row) arrow_row = row_values_to_arrow_table( values_by_name, writer.table.table_schema.fields, column_names) @@ -170,14 +170,13 @@ def _flush_pending_inputs(self): writer._pending_inputs = [] for input_type, value in inputs: if input_type == "batch": - PythonTableWrite(writer).write_arrow_batch(value) + super().write_arrow_batch(value) else: - PythonTableWrite(writer).write_row(value[0]) + super().write_row(value[0]) def prepare_commit(self, commit_identifier) -> List[CommitMessage]: - writer = self.writer - writer._flush_pending_inputs() - return PythonTableWrite(writer).prepare_commit(commit_identifier) + self._flush_pending_inputs() + return super().prepare_commit(commit_identifier) def _distributed_write_options(self, *, overwrite=False) -> Dict[str, Any]: writer = self.writer @@ -188,9 +187,9 @@ def _distributed_write_options(self, *, overwrite=False) -> Dict[str, Any]: def close(self): writer = self.writer writer._pending_inputs = [] - PythonTableWrite(writer).close() + super().close() def abort(self): writer = self.writer writer._pending_inputs = [] - PythonTableWrite(writer).abort() + super().abort() diff --git a/paimon-python/pypaimon/execution/python/write/table_delete.py b/paimon-python/pypaimon/execution/python/write/table_delete.py index 8cfd12595189..1db2ad427648 100644 --- a/paimon-python/pypaimon/execution/python/write/table_delete.py +++ b/paimon-python/pypaimon/execution/python/write/table_delete.py @@ -58,40 +58,35 @@ class _AnchorRangesInfo: class PythonTableDeleteByRowId: """Write data-evolution deletes as deletion-vector index updates.""" - def __init__(self, context): - self._context = context - def initialize( self, table, _precomputed_anchor_ranges: Optional[_AnchorRangesInfo] = None, ): - context = self._context from pypaimon.table.file_store_table import FileStoreTable - context.table: FileStoreTable = table - context.file_io = table.file_io - context._precomputed_anchor_ranges = _precomputed_anchor_ranges + self.table: FileStoreTable = table + self.file_io = table.file_io + self._precomputed_anchor_ranges = _precomputed_anchor_ranges + return self def _snapshot_anchor_ranges(self) -> _AnchorRangesInfo: - context = self._context - snapshot_id, anchors = context._scan_anchor_ranges() + snapshot_id, anchors = self._scan_anchor_ranges() return _AnchorRangesInfo(snapshot_id=snapshot_id, anchors=anchors) def delete(self, row_ids: Sequence[int]) -> List[CommitMessage]: - context = self._context - context._validate_delete() + self._validate_delete() - deduplicated = context._deduplicate_row_ids(row_ids) + deduplicated = self._deduplicate_row_ids(row_ids) if not deduplicated: return [] - snapshot_id, anchors = context._load_anchor_ranges() - targets = context._group_delete_targets(deduplicated, anchors) + snapshot_id, anchors = self._load_anchor_ranges() + targets = self._group_delete_targets(deduplicated, anchors) commit_messages = [] for (partition, bucket), file_positions in targets.items(): - message = context._build_commit_message( + message = self._build_commit_message( partition, bucket, file_positions, @@ -104,16 +99,15 @@ def delete(self, row_ids: Sequence[int]) -> List[CommitMessage]: def _validate_delete(self): # Java DataEvolutionDeleteSink.validateTable reports unsupported table # capabilities; Native maps the same failure to NotImplementedError. - context = self._context - if not context.table.options.data_evolution_enabled(): + if not self.table.options.data_evolution_enabled(): raise NotImplementedError( "delete requires 'data-evolution.enabled' = 'true'." ) - if not context.table.options.row_tracking_enabled(): + if not self.table.options.row_tracking_enabled(): raise NotImplementedError( "delete requires 'row-tracking.enabled' = 'true'." ) - if not context.table.options.deletion_vectors_enabled(False): + if not self.table.options.deletion_vectors_enabled(False): raise NotImplementedError( "delete requires 'deletion-vectors.enabled' = 'true'." ) @@ -132,17 +126,15 @@ def _deduplicate_row_ids(row_ids: Sequence[int]) -> List[int]: return result def _load_anchor_ranges(self) -> Tuple[int, List[_AnchorRange]]: - context = self._context - if context._precomputed_anchor_ranges is not None: + if self._precomputed_anchor_ranges is not None: return ( - context._precomputed_anchor_ranges.snapshot_id, - context._precomputed_anchor_ranges.anchors, + self._precomputed_anchor_ranges.snapshot_id, + self._precomputed_anchor_ranges.anchors, ) - return context._scan_anchor_ranges() + return self._scan_anchor_ranges() def _scan_anchor_ranges(self) -> Tuple[int, List[_AnchorRange]]: - context = self._context - scan = context.table.new_read_builder().new_scan() + scan = self.table.new_read_builder().new_scan() plan = scan.plan_all() snapshot_id = plan.snapshot_id if plan.snapshot_id is not None else -1 anchors = [] @@ -199,9 +191,8 @@ def _build_commit_message( file_positions: Dict[str, Set[int]], snapshot_id: int, ) -> Optional[CommitMessage]: - context = self._context - partition_row = GenericRow(list(partition), context.table.partition_keys_fields) - old_entries, deletion_vectors = context._read_existing_deletion_vectors( + partition_row = GenericRow(list(partition), self.table.partition_keys_fields) + old_entries, deletion_vectors = self._read_existing_deletion_vectors( partition_row, bucket, snapshot_id, @@ -219,7 +210,7 @@ def _build_commit_message( if not changed: return None - new_entry = context._write_deletion_vector_index( + new_entry = self._write_deletion_vector_index( partition_row, bucket, deletion_vectors, @@ -248,8 +239,7 @@ def _read_existing_deletion_vectors( bucket: int, snapshot_id: int, ) -> Tuple[List[IndexManifestEntry], Dict[str, DeletionVector]]: - context = self._context - snapshot_manager = context.table.snapshot_manager() + snapshot_manager = self.table.snapshot_manager() snapshot = ( snapshot_manager.get_snapshot_by_id(snapshot_id) if snapshot_id >= 0 else snapshot_manager.get_latest_snapshot() @@ -257,7 +247,7 @@ def _read_existing_deletion_vectors( if snapshot is None or not snapshot.index_manifest: return [], {} - index_manifest_file = IndexManifestFile(context.table) + index_manifest_file = IndexManifestFile(self.table) entries = [] deletion_vectors: Dict[str, DeletionVector] = {} for entry in index_manifest_file.read(snapshot.index_manifest): @@ -272,7 +262,7 @@ def _read_existing_deletion_vectors( if not entry.index_file.dv_ranges: continue - dv_path = context._index_file_path(entry.index_file, partition, bucket) + dv_path = self._index_file_path(entry.index_file, partition, bucket) for data_file_name, meta in entry.index_file.dv_ranges.items(): deletion_file = DeletionFile( dv_index_path=dv_path, @@ -281,7 +271,7 @@ def _read_existing_deletion_vectors( cardinality=meta.cardinality, ) deletion_vectors[data_file_name] = DeletionVector.read( - context.file_io, + self.file_io, deletion_file, ) @@ -293,9 +283,8 @@ def _write_deletion_vector_index( bucket: int, deletion_vectors: Dict[str, DeletionVector], ) -> IndexManifestEntry: - context = self._context file_name = f"{FileStorePathFactory.INDEX_PREFIX}{uuid.uuid4()}-1" - path, external = context.table.path_factory().new_bucket_index_path( + path, external = self.table.path_factory().new_bucket_index_path( tuple(partition.values), bucket, file_name) position = 1 @@ -316,10 +305,10 @@ def _write_deletion_vector_index( data = b"".join(chunks) try: - with context.file_io.new_output_stream(path) as output_stream: + with self.file_io.new_output_stream(path) as output_stream: output_stream.write(data) except Exception: - context.file_io.delete_quietly(path) + self.file_io.delete_quietly(path) raise index_file = IndexFileMeta( @@ -338,6 +327,5 @@ def _write_deletion_vector_index( ) def _index_file_path(self, index_file: IndexFileMeta, partition: GenericRow, bucket: int) -> str: - context = self._context - return context.table.path_factory().bucket_index_path( + return self.table.path_factory().bucket_index_path( tuple(partition.values), bucket, index_file) diff --git a/paimon-python/pypaimon/execution/python/write/table_update.py b/paimon-python/pypaimon/execution/python/write/table_update.py index 21be8bc245df..3074842f5c0b 100644 --- a/paimon-python/pypaimon/execution/python/write/table_update.py +++ b/paimon-python/pypaimon/execution/python/write/table_update.py @@ -47,6 +47,7 @@ from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER from pypaimon.table.special_fields import SpecialFields from pypaimon.write.commit_message import CommitMessage +from pypaimon.write.update_validation import predicate_fields from pypaimon.execution.python.write.row_id_file_index import RowIdFileIndex from pypaimon.write.table_delete import TableDeleteByRowId from pypaimon.execution.python.write.writer.data_writer import DataWriter @@ -113,7 +114,7 @@ def _divide_ranges( class PythonTableUpdate: - """Execute an update using the facade's configuration and legacy hooks.""" + """Execute an update using the public configuration.""" def __init__(self, context, commit_identifier=BATCH_COMMIT_IDENTIFIER, predicate=None, *, has_callable=False, has_array=False): @@ -127,26 +128,33 @@ def update_by_arrow_with_row_id(self, data): context = self._context columns = context.update_cols if context.update_cols is not None else [ name for name in data.column_names if name != SpecialFields.ROW_ID.name] - return context._new_row_id_updater(self._commit_identifier).update_columns(data, columns) + return self._new_row_id_updater(self._commit_identifier).update_columns(data, columns) def update_by_arrow_batches_with_row_id(self, tables): - return self._context._update_by_arrow_batches_with_row_id(tables, self._commit_identifier) + return self._update_by_arrow_batches_with_row_id(tables, self._commit_identifier) def delete_by_row_id(self, row_ids): - return self._context._delete_rows_with_python(row_ids) + return self._delete_rows_with_python(row_ids) def update(self, assignments, read_columns): - return self._context._execute_predicate_update( + return self._execute_predicate_update( self._predicate, assignments, self._commit_identifier, read_columns, self._has_callable, self._has_array) def delete(self): - return self._context._execute_predicate_delete(self._predicate, self._commit_identifier) + return self._execute_predicate_delete(self._predicate, self._commit_identifier) + + def _new_row_id_updater(self, commit_identifier, _precomputed_files_info=None): + from pypaimon.write.table_update_by_row_id import TableUpdateByRowId + context = self._context + _authorize_update(context.table) + return TableUpdateByRowId._from_python( + context.table, context.commit_user, commit_identifier, + _precomputed_files_info=_precomputed_files_info) def _matched_delete_row_ids( self, predicate: Optional[Predicate]) -> List[int]: - context = self._context - scan_table = context._matched_update_scan_table() + scan_table = self._matched_update_scan_table() read_builder = scan_table.new_read_builder() if predicate is not None: read_builder.with_filter(predicate) @@ -175,7 +183,7 @@ def _update_by_arrow_batches_with_row_id( if c != SpecialFields.ROW_ID.name ] if updater is None: - updater = context._new_row_id_updater(commit_identifier) + updater = self._new_row_id_updater(commit_identifier) updater.update_columns(table, cols) return [] if updater is None else updater.commit_messages except Exception: @@ -256,7 +264,7 @@ def _build_predicate_update_table( ) target_field = table_schema.field(col) arrays.append( - context._assignment_to_array( + self._assignment_to_array( value, target_field.type, matched.num_rows ) ) @@ -336,7 +344,7 @@ def _delete_by_partition_filter( ).with_all_buckets().read_manifest_entries(manifest_list_manager.read_all(snapshot)) for entry in data_entries: - message = context._partition_delete_message( + message = self._partition_delete_message( messages, tuple(entry.partition.values), entry.bucket, @@ -344,8 +352,8 @@ def _delete_by_partition_filter( ) message.deleted_files.append(entry.file) - for entry in context._partition_index_entries(snapshot, partition_filter): - message = context._partition_delete_message( + for entry in self._partition_index_entries(snapshot, partition_filter): + message = self._partition_delete_message( messages, tuple(entry.partition.values), entry.bucket, @@ -387,16 +395,16 @@ def _partition_only_delete_filter( context = self._context if predicate is None or not context.table.partition_keys: return None - predicate_fields = context._predicate_fields(predicate) - if not predicate_fields: + fields = predicate_fields(predicate) + if not fields: return None partition_keys = set(context.table.partition_keys) - if not predicate_fields.issubset(partition_keys): + if not fields.issubset(partition_keys): return None partition_index = { name: index for index, name in enumerate(context.table.partition_keys) } - return context._rewrite_predicate_to_partition_indices( + return self._rewrite_predicate_to_partition_indices( predicate, partition_index ) @@ -405,10 +413,9 @@ def _rewrite_predicate_to_partition_indices( predicate: Predicate, partition_index: Mapping[str, int], ) -> Predicate: - context = self._context if predicate.method in ('and', 'or'): return predicate.new_literals([ - context._rewrite_predicate_to_partition_indices( + self._rewrite_predicate_to_partition_indices( child, partition_index, ) for child in (predicate.literals or []) @@ -422,8 +429,7 @@ def _rewrite_predicate_to_partition_indices( def _execute_predicate_update( self, predicate, assignments, commit_identifier, read_columns, has_callable, has_array): - context = self._context - scan_table = context._matched_update_scan_table() + scan_table = self._matched_update_scan_table() read_builder = scan_table.new_read_builder() if predicate is not None: read_builder.with_filter(predicate) @@ -441,12 +447,12 @@ def _execute_predicate_update( snapshot_id, splits ) table_read = read_builder.new_read() - updater = context._new_row_id_updater(commit_identifier, files_info) + updater = self._new_row_id_updater(commit_identifier, files_info) try: if has_array: matched = table_read.to_arrow(splits) if matched.num_rows > 0: - update_table = context._build_predicate_update_table( + update_table = self._build_predicate_update_table( assignments, matched, ) @@ -454,11 +460,11 @@ def _execute_predicate_update( update_table, list(assignments.keys()) ) else: - for split in context._predicate_update_file_groups(splits): + for split in self._predicate_update_file_groups(splits): matched = table_read.to_arrow([split], parallelism=1) if matched.num_rows == 0: continue - update_table = context._build_predicate_update_table( + update_table = self._build_predicate_update_table( assignments, matched, ) @@ -472,23 +478,23 @@ def _execute_predicate_update( return updater.commit_messages def _execute_predicate_delete(self, predicate, commit_identifier): - context = self._context - partition_filter = context._partition_only_delete_filter(predicate) + partition_filter = self._partition_only_delete_filter(predicate) if partition_filter is not None: - return context._delete_by_partition_filter(partition_filter) + return self._delete_by_partition_filter(partition_filter) - row_ids = context._matched_delete_row_ids(predicate) - return context._delete_by_row_id(row_ids, commit_identifier) + row_ids = self._matched_delete_row_ids(predicate) + return self._delete_rows_with_python(row_ids) def _new_python_shard_updator(self, shard_num, total_shard_count): context = self._context from pypaimon.write.table_update import ShardTableUpdator - return ShardTableUpdator(context.table, context.projection, context.update_cols, - context.commit_user, shard_num, total_shard_count) + return ShardTableUpdator._from_python( + context.table, context.projection, context.update_cols, + context.commit_user, shard_num, total_shard_count) def _delete_rows_with_python(self, row_ids): context = self._context - return TableDeleteByRowId(context.table).delete(list(row_ids)) + return TableDeleteByRowId._from_python(context.table).delete(list(row_ids)) class _ShardBatchIterator: @@ -682,14 +688,14 @@ def _init_writer(self): context = self._context if context.writer is None: from pypaimon.write.table_update_by_row_id import TableUpdateByRowId - context.writer = TableUpdateByRowId( + context.writer = TableUpdateByRowId._from_python( context.table, context.commit_user, BATCH_COMMIT_IDENTIFIER, _precomputed_files_info=context._files_info) def _flush_group(self): context = self._context if context._group_batches: - context._init_writer() + self._init_writer() # Preserve staged messages if writing fails; they may already have # been published. A failed operation cannot later prepare success. context._failed = True @@ -731,7 +737,7 @@ def capacity(self) -> int: def write(self, data: pa.RecordBatch): context = self._context - if data.num_rows > context.capacity(): + if data.num_rows > self.capacity(): raise Exception("Data num size exceeds capacity.") context.written_records_count += data.num_rows context.writer.write(data) @@ -743,7 +749,7 @@ def partition(self) -> Tuple: def end(self) -> DataFileMeta: context = self._context - if context.capacity() != 0: + if self.capacity() != 0: raise Exception("There still capacity left in the writer.") files = context.writer.prepare_commit() if len(files) != 1: diff --git a/paimon-python/pypaimon/execution/python/write/table_update_by_row_id.py b/paimon-python/pypaimon/execution/python/write/table_update_by_row_id.py index fecf0699eb2e..106d785dc22b 100644 --- a/paimon-python/pypaimon/execution/python/write/table_update_by_row_id.py +++ b/paimon-python/pypaimon/execution/python/write/table_update_by_row_id.py @@ -40,6 +40,7 @@ from pypaimon.table.row.generic_row import GenericRow from pypaimon.table.special_fields import SpecialFields from pypaimon.write.commit_message import CommitMessage +from pypaimon.write.row_input import RowIdUpdateInputMixin from pypaimon.execution.python.write.file_store_write import FileStoreWrite from pypaimon.execution.python.write.row_id_file_index import RowIdFileIndex from pypaimon.execution.python.write.writer.blob_writer import BlobWriter @@ -185,40 +186,36 @@ def close(self): self._data_writer.close() -class PythonTableUpdateByRowId: +class PythonTableUpdateByRowId(RowIdUpdateInputMixin): FIRST_ROW_ID_COLUMN = '_FIRST_ROW_ID' - def __init__(self, context): - self._context = context - def initialize( self, table, commit_user: str, commit_identifier: int, _precomputed_files_info: Optional[RowIdFileIndex] = None, ): - context = self._context from pypaimon.table.file_store_table import FileStoreTable - context.table: FileStoreTable = table - context.commit_user = commit_user - context.commit_identifier = commit_identifier + self.table: FileStoreTable = table + self.commit_user = commit_user + self.commit_identifier = commit_identifier - info = _precomputed_files_info or context._load_existing_files_info() - context.snapshot_id = info.snapshot_id - context.first_row_ids = info.first_row_ids - context._first_row_id_index = info.first_row_id_index - context.valid_row_id_ranges = info.valid_row_id_ranges + info = _precomputed_files_info or self._load_existing_files_info() + self.snapshot_id = info.snapshot_id + self.first_row_ids = info.first_row_ids + self._first_row_id_index = info.first_row_id_index + self.valid_row_id_ranges = info.valid_row_id_ranges - context.commit_messages: List[CommitMessage] = [] - context._updated_first_row_ids_by_column: Dict[str, Set[int]] = {} + self.commit_messages: List[CommitMessage] = [] + self._updated_first_row_ids_by_column: Dict[str, Set[int]] = {} + return self def _snapshot_files_info(self) -> RowIdFileIndex: """Return the already loaded snapshot file index for broadcast.""" - context = self._context return RowIdFileIndex( - snapshot_id=context.snapshot_id, - first_row_ids=context.first_row_ids, - first_row_id_index=context._first_row_id_index, - valid_row_id_ranges=context.valid_row_id_ranges, + snapshot_id=self.snapshot_id, + first_row_ids=self.first_row_ids, + first_row_id_index=self._first_row_id_index, + valid_row_id_ranges=self.valid_row_id_ranges, ) def _load_existing_files_info(self) -> RowIdFileIndex: @@ -229,8 +226,7 @@ def _load_existing_files_info(self) -> RowIdFileIndex: id (a single id may belong to multiple files when data evolution has split a logical row range). """ - context = self._context - return RowIdFileIndex.from_table(context.table) + return RowIdFileIndex.from_table(self.table) def update_columns(self, data: pa.Table, column_names: List[str]) -> List[CommitMessage]: """ @@ -243,7 +239,6 @@ def update_columns(self, data: pa.Table, column_names: List[str]) -> List[Commit Returns: List of commit messages """ - context = self._context if not column_names: raise ValueError("column_names cannot be empty") @@ -253,17 +248,17 @@ def update_columns(self, data: pa.Table, column_names: List[str]) -> List[Commit raise ValueError(f"Input data must contain {SpecialFields.ROW_ID.name} column") for col_name in column_names: - if col_name not in context.table.field_names: + if col_name not in self.table.field_names: raise ValueError(f"Column {col_name} not found in table schema") - data_with_first_row_id = context._calculate_first_row_id(data) + data_with_first_row_id = self._calculate_first_row_id(data) first_row_ids = set( - data_with_first_row_id[context.FIRST_ROW_ID_COLUMN].to_pylist() + data_with_first_row_id[self.FIRST_ROW_ID_COLUMN].to_pylist() ) overlapping = {} for col_name in column_names: ids = first_row_ids.intersection( - context._updated_first_row_ids_by_column.get(col_name, set())) + self._updated_first_row_ids_by_column.get(col_name, set())) if ids: overlapping[col_name] = sorted(ids) if overlapping: @@ -271,20 +266,19 @@ def update_columns(self, data: pa.Table, column_names: List[str]) -> List[Commit "Input batches contain overlapping first_row_ids by column: " f"{overlapping}" ) - context._write_by_first_row_id(data_with_first_row_id, column_names) + self._write_by_first_row_id(data_with_first_row_id, column_names) for col_name in column_names: - context._updated_first_row_ids_by_column.setdefault( + self._updated_first_row_ids_by_column.setdefault( col_name, set()).update(first_row_ids) - return context.commit_messages + return self.commit_messages def _write_row_columns(self, update_data, column_names, blob_object_columns): - context = self._context - data_with_first_row_id = context._calculate_first_row_id(update_data) - context._write_by_first_row_id( + data_with_first_row_id = self._calculate_first_row_id(update_data) + self._write_by_first_row_id( data_with_first_row_id, column_names, blob_object_columns) - return context.commit_messages + return self.commit_messages def _calculate_first_row_id(self, data: pa.Table) -> pa.Table: """Append ``_FIRST_ROW_ID`` to *data* by looking up each ``_ROW_ID``. @@ -292,7 +286,6 @@ def _calculate_first_row_id(self, data: pa.Table) -> pa.Table: Validates that every input ``_ROW_ID`` is unique and belongs to a valid row_id range. Supports partial / non-consecutive updates. """ - context = self._context row_id_arr = data[SpecialFields.ROW_ID.name] row_ids = row_id_arr.to_pylist() @@ -301,27 +294,27 @@ def _calculate_first_row_id(self, data: pa.Table) -> pa.Table: if not row_ids: return data.append_column( - context.FIRST_ROW_ID_COLUMN, pa.array([], type=pa.int64()), + self.FIRST_ROW_ID_COLUMN, pa.array([], type=pa.int64()), ) for row_id in row_ids: - if not any(r.contains(row_id) for r in context.valid_row_id_ranges): + if not any(r.contains(row_id) for r in self.valid_row_id_ranges): raise ValueError( f"Row ID {row_id} does not belong to any valid range " - f"{[f'[{r.from_}, {r.to}]' for r in context.valid_row_id_ranges]}" + f"{[f'[{r.from_}, {r.to}]' for r in self.valid_row_id_ranges]}" ) - if not context.first_row_ids: + if not self.first_row_ids: raise ValueError("The input sorted sequence is empty.") - sorted_seq = context.first_row_ids + sorted_seq = self.first_row_ids bisect_right = bisect.bisect_right first_row_id_values = [ sorted_seq[bisect_right(sorted_seq, row_id) - 1] for row_id in row_ids ] return data.append_column( - context.FIRST_ROW_ID_COLUMN, + self.FIRST_ROW_ID_COLUMN, pa.array(first_row_id_values, type=pa.int64()), ) @@ -331,13 +324,12 @@ def _write_by_first_row_id( column_names: List[str], blob_object_columns: Optional[Dict[str, List[Any]]] = None): """Write data grouped by first_row_id.""" - context = self._context - first_row_id_array = data[context.FIRST_ROW_ID_COLUMN] + first_row_id_array = data[self.FIRST_ROW_ID_COLUMN] unique_first_row_ids = pc.unique(first_row_id_array).to_pylist() first_row_id_values = first_row_id_array.to_pylist() for first_row_id in unique_first_row_ids: - entry = context._first_row_id_index.get(first_row_id) + entry = self._first_row_id_index.get(first_row_id) if entry is None: raise ValueError(f"No existing file found for first_row_id {first_row_id}") split, _files = entry @@ -353,7 +345,7 @@ def _write_by_first_row_id( col_name: [values[i] for i in group_indices] for col_name, values in blob_object_columns.items() } - context._write_group( + self._write_group( split.partition, first_row_id, group_data, @@ -375,18 +367,16 @@ def _read_original_file_data(self, first_row_id: int, column_names: List[str]) - Returns: PyArrow Table containing the original values for the requested columns. """ - context = self._context - table_read, origin_split = context._original_file_read(first_row_id, column_names) + table_read, origin_split = self._original_file_read(first_row_id, column_names) original = table_read.to_arrow([origin_split]) return original.select(column_names) def _original_file_read(self, first_row_id, column_names): - context = self._context wanted = set(column_names) read_fields: List[DataField] = [ - table_field for table_field in context.table.fields if table_field.name in wanted + table_field for table_field in self.table.fields if table_field.name in wanted ] - entry = context._first_row_id_index.get(first_row_id) + entry = self._first_row_id_index.get(first_row_id) if entry is None: raise ValueError(f"No file found for first_row_id {first_row_id}") owning_split, target_files = entry @@ -401,10 +391,10 @@ def _original_file_read(self, first_row_id, column_names): # added after the original file was written, reading only those # missing columns can otherwise produce a zero-row table instead of # one null value per original row. - read_table = context.table + read_table = self.table if wanted.intersection( - context.table.options.blob_descriptor_fields() | context.table.options.blob_view_fields()): - read_table = context.table.copy({'blob-as-descriptor': 'true', 'blob-view.resolve.enabled': 'false'}) + self.table.options.blob_descriptor_fields() | self.table.options.blob_view_fields()): + read_table = self.table.copy({'blob-as-descriptor': 'true', 'blob-view.resolve.enabled': 'false'}) table_read = TableRead( read_table, predicate=None, @@ -414,8 +404,7 @@ def _original_file_read(self, first_row_id, column_names): def _merged_batches(self, first_row_id, data, column_names): """Merge ordinary columns a batch at a time in physical row order.""" - context = self._context - table_read, split = context._original_file_read(first_row_id, column_names) + table_read, split = self._original_file_read(first_row_id, column_names) updates = sorted(enumerate(data[SpecialFields.ROW_ID.name].to_pylist()), key=lambda item: item[1]) update_index = 0 @@ -444,7 +433,7 @@ def _merged_batches(self, first_row_id, data, column_names): update_index += 1 original = pa.Table.from_batches([batch]).select(column_names) if selected: - merged, _ = context._merge_update_with_original( + merged, _ = self._merge_update_with_original( original, data.take(selected), column_names, offset) else: merged = original @@ -455,18 +444,17 @@ def _merged_batches(self, first_row_id, data, column_names): raise ValueError('Update row IDs extend past the original file group') def _write_group_streaming(self, partition, first_row_id, data, column_names): - context = self._context writer = _RowIdUpdateFileWriter( - context.table, tuple(partition.values), column_names) - merged_batches = context._merged_batches(first_row_id, data, column_names) - batches = (context._normalize_write_data(batch) for batch in merged_batches) + self.table, tuple(partition.values), column_names) + merged_batches = self._merged_batches(first_row_id, data, column_names) + batches = (self._normalize_write_data(batch) for batch in merged_batches) try: files = writer.write_batches(batches) - context._assign_update_file_metadata(files, first_row_id, column_names, {}) + self._assign_update_file_metadata(files, first_row_id, column_names, {}) if files: - context.commit_messages.append(CommitMessage( + self.commit_messages.append(CommitMessage( partition=tuple(partition.values), bucket=0, new_files=files, - check_from_snapshot=context.snapshot_id)) + check_from_snapshot=self.snapshot_id)) except Exception: writer.abort() raise @@ -511,7 +499,6 @@ def _merge_update_with_original( Blob value list lengths may differ when the updated columns do not all have physical baselines. """ - context = self._context # Get the _ROW_ID values from update_data to determine which rows are being updated relative_indices = pc.subtract( @@ -536,14 +523,14 @@ def _merge_update_with_original( # non-empty group, so update_positions is non-empty here. blob_columns_with_baseline = blob_columns_with_baseline or set() for col_name in column_names: - if context._is_blob_column(col_name): + if self._is_blob_column(col_name): has_baseline = col_name in blob_columns_with_baseline blob_row_count = ( max(update_positions) + 1 if has_baseline else original_data.num_rows ) missing_value = ( - context._blob_placeholder(col_name) + self._blob_placeholder(col_name) if has_baseline else None ) if blob_object_columns and col_name in blob_object_columns: @@ -573,7 +560,7 @@ def _merge_update_with_original( raise IndexError( f"Update position {position} is outside column " f"range [0, {row_count})") - merged_columns[col_name] = context._merge_chunked_column( + merged_columns[col_name] = self._merge_chunked_column( original_col, update_col, sorted_updates) merged_table = None @@ -589,15 +576,14 @@ def _merge_update_with_original( def _normalize_write_data(self, data): # DE reads relax nested nullability. Restore and validate the declared # write types at the writer boundary, including untouched input batches. - context = self._context write_schema = PyarrowFieldParser.from_paimon_schema( - [context.table.field_dict[name] for name in data.schema.names]) + [self.table.field_dict[name] for name in data.schema.names]) normalized = normalize_arrow_write_nullability(data, write_schema) # Reads relax field nullability for schema evolution. A projected # update must still enforce the declared write type, as Java's # TableWrite.withWriteType does for every selected output column. for field, column in zip(write_schema, normalized.columns): - context._check_write_not_null(field.name, column.null_count > 0) + self._check_write_not_null(field.name, column.null_count > 0) if any(actual.nullable != expected.nullable for actual, expected in zip(normalized.schema, write_schema)): fields = [actual.with_nullable(expected.nullable) @@ -609,8 +595,7 @@ def _normalize_write_data(self, data): return normalized def _check_write_not_null(self, column_name, has_null): - context = self._context - if not context.table.field_dict[column_name].type.nullable and has_null: + if not self.table.field_dict[column_name].type.nullable and has_null: raise ValueError('Cannot write null to non-null column(%s)' % column_name) @classmethod @@ -817,8 +802,7 @@ def _coerce_column(col: pa.Array, target_type: pa.DataType) -> pa.Array: return pa.array(pylist, type=target_type) def _blob_placeholder(self, column_name: str): - context = self._context - for table_field in context.table.fields: + for table_field in self.table.fields: if table_field.name == column_name: if is_map_blob_type(table_field.type): return Blob.MAP_PLACE_HOLDER @@ -841,20 +825,19 @@ def _write_group( Reads the original file data, merges in the update values, and writes a single output file (rolling disabled) for the group. """ - context = self._context # Specialized writers still own their sidecars and physical encoding. - if _RowIdUpdateFileWriter.supports_table(context.table): - context._write_group_streaming(partition, first_row_id, data, column_names) + if _RowIdUpdateFileWriter.supports_table(self.table): + self._write_group_streaming(partition, first_row_id, data, column_names) return - original_data = context._read_original_file_data(first_row_id, column_names) - _, target_files = context._first_row_id_index[first_row_id] + original_data = self._read_original_file_data(first_row_id, column_names) + _, target_files = self._first_row_id_index[first_row_id] blob_columns_with_baseline = { column_name for file in target_files if DataFileMeta.is_blob_file(file.file_name) for column_name in (file.write_cols or []) } - merged_data, blob_columns = context._merge_update_with_original( + merged_data, blob_columns = self._merge_update_with_original( original_data, data, column_names, @@ -867,7 +850,7 @@ def _write_group( # the merged values that will be written, including row-based inputs, # while placeholders and NULL children of non-NULL containers remain valid. for column_name, values in blob_columns.items(): - context._check_write_not_null(column_name, any(value is None for value in values)) + self._check_write_not_null(column_name, any(value is None for value in values)) partition_tuple = tuple(partition.values) new_files = [] @@ -876,25 +859,25 @@ def _write_group( success = False try: if merged_data is not None: - merged_data = context._normalize_write_data(merged_data) - file_store_write = FileStoreWrite(context.table, context.commit_user) + merged_data = self._normalize_write_data(merged_data) + file_store_write = FileStoreWrite(self.table, self.commit_user) file_store_write.disable_rolling() file_store_write.write_cols = list(merged_data.column_names) for batch in merged_data.to_batches(): file_store_write.write(partition_tuple, 0, batch) - new_messages = file_store_write.prepare_commit(context.commit_identifier) + new_messages = file_store_write.prepare_commit(self.commit_identifier) for msg in new_messages: new_files.extend(msg.new_files) for column_name, values in blob_columns.items(): blob_writer = BlobWriter( - context.table, + self.table, partition_tuple, 0, 0, column_name, - context.table.options, - video=column_name in context.table.options.video_frame_fields(), + self.table.options, + video=column_name in self.table.options.video_frame_fields(), ) blob_writers.append(blob_writer) arrow_type = original_data.schema.field(column_name).type @@ -904,14 +887,14 @@ def _write_group( blob_writer._release_prepared_files() if new_files: - context._assign_update_file_metadata( + self._assign_update_file_metadata( new_files, first_row_id, column_names, blob_columns) - context.commit_messages.append( + self.commit_messages.append( CommitMessage( partition=partition_tuple, bucket=0, new_files=new_files, - check_from_snapshot=context.snapshot_id, + check_from_snapshot=self.snapshot_id, ) ) success = True diff --git a/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py b/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py index 73171ed9da56..41ae859ca338 100644 --- a/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py +++ b/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py @@ -33,6 +33,7 @@ from pypaimon.table.special_fields import SpecialFields from pypaimon.write.commit_message import CommitMessage from pypaimon.write.row_utils import require_columns +from pypaimon.write.update_validation import _authorize_update from pypaimon.write.table_write import StreamTableWrite # Composite key is represented as a tuple of values @@ -80,16 +81,16 @@ def __init__(self, context, data=None, upsert_keys=None, update_cols=None, *, ro self._rows = rows def upsert(self): - context = self._context if self._rows: - return context._upsert_rows(self._data, self._upsert_keys, self._update_cols) - return context._upsert_arrow(self._data, self._upsert_keys, self._update_cols) + return self._upsert_rows(self._data, self._upsert_keys, self._update_cols) + return self._upsert_arrow(self._data, self._upsert_keys, self._update_cols) def _new_row_id_updater(self): context = self._context - from pypaimon.write.write_builder import _new_update_by_row_id + from pypaimon.write.table_update_by_row_id import TableUpdateByRowId - return _new_update_by_row_id( + _authorize_update(context.table) + return TableUpdateByRowId._from_python( context.table, context.commit_user, context.commit_identifier) def _group_rows_by_partition( @@ -132,10 +133,10 @@ def _upsert_row_partition( if isinstance(value, Blob): raise ValueError("Blob values are not supported as upsert keys.") - row_items, input_key_tuples = context._dedup_row_items_last_write_wins( + row_items, input_key_tuples = self._dedup_row_items_last_write_wins( row_items, input_key_tuples, partition_spec) - key_to_row_ids = context._build_key_to_row_ids_map( + key_to_row_ids = self._build_key_to_row_ids_map( match_keys, partition_spec, set(input_key_tuples) ) @@ -158,14 +159,14 @@ def _upsert_row_partition( ) for _, values_by_name in matched_items: require_columns(values_by_name, cols_to_update, "upsert_by_key") - commit_messages.extend(context._new_row_id_updater().update_rows_columns( + commit_messages.extend(self._new_row_id_updater().update_rows_columns( [row for row, _ in matched_items], matched_row_ids, cols_to_update, )) if new_items: - commit_messages.extend(context._append_rows(new_items)) + commit_messages.extend(self._append_rows(new_items)) return commit_messages @@ -255,13 +256,13 @@ def _upsert_partition( ] # 2. Deduplicate – keep the LAST occurrence of every repeated key. - partition_data, input_key_tuples = context._dedup_last_write_wins( + partition_data, input_key_tuples = self._dedup_last_write_wins( partition_data, input_key_tuples, partition_spec, ) # 3. Look up candidate rows using key predicates and available indexes, # keeping only exact input-key matches. - key_to_row_ids = context._build_key_to_row_ids_map( + key_to_row_ids = self._build_key_to_row_ids_map( match_keys, partition_spec, set(input_key_tuples), ) @@ -286,12 +287,12 @@ def _upsert_partition( commit_messages: List[CommitMessage] = [] if matched_indices: - commit_messages.extend(context._do_updates( + commit_messages.extend(self._do_updates( partition_data, matched_indices, input_key_tuples, key_to_row_ids, update_cols, )) if new_indices: - commit_messages.extend(context._do_appends(partition_data, new_indices)) + commit_messages.extend(self._do_appends(partition_data, new_indices)) return commit_messages @staticmethod @@ -427,7 +428,7 @@ def _do_updates( ) cols_to_update = list(update_cols) if update_cols else list(context.table.field_names) - return context._new_row_id_updater().update_columns( + return self._new_row_id_updater().update_columns( update_data, cols_to_update) def _do_appends( @@ -466,7 +467,7 @@ def _append_rows( row_items: List[Tuple[Any, Dict[str, Any]]], ) -> List[CommitMessage]: context = self._context - all_ordered_cols = context._append_row_column_names(row_items) + all_ordered_cols = self._append_row_column_names(row_items) table_write = StreamTableWrite(context.table, context.commit_user) try: @@ -507,12 +508,11 @@ def _append_row_column_names( ] def _upsert_arrow(self, data, upsert_keys, effective_update_cols): - context = self._context all_commit_messages: List[CommitMessage] = [] # Process each partition independently - for partition_spec, partition_data in context._group_by_partition(data): - msgs = context._upsert_partition( + for partition_spec, partition_data in self._group_by_partition(data): + msgs = self._upsert_partition( partition_data, upsert_keys, partition_spec, effective_update_cols ) all_commit_messages.extend(msgs) @@ -520,11 +520,10 @@ def _upsert_arrow(self, data, upsert_keys, effective_update_cols): return all_commit_messages def _upsert_rows(self, row_items, upsert_keys, effective_update_cols): - context = self._context commit_messages: List[CommitMessage] = [] - for partition_spec, partition_items in context._group_rows_by_partition(row_items): + for partition_spec, partition_items in self._group_rows_by_partition(row_items): commit_messages.extend( - context._upsert_row_partition( + self._upsert_row_partition( partition_items, upsert_keys, partition_spec, diff --git a/paimon-python/pypaimon/execution/python/write/table_write.py b/paimon-python/pypaimon/execution/python/write/table_write.py index 917a050e0ddd..c893807b0aaa 100644 --- a/paimon-python/pypaimon/execution/python/write/table_write.py +++ b/paimon-python/pypaimon/execution/python/write/table_write.py @@ -52,7 +52,7 @@ def initialize(self, table, commit_user, static_partition: Optional[dict] = None writer.commit_user = commit_user writer.static_partition = static_partition writer.restore_snapshot_id = restore_snapshot_id - writer.file_store_write = writer._create_file_store_write(commit_user) + writer.file_store_write = self._create_file_store_write(commit_user) # Java overwrite writers ignore previous files even without an explicit # restore snapshot. HASH restoration is separately disabled by the row # key extractor; sequence restoration must also start from empty state. @@ -63,7 +63,7 @@ def initialize(self, table, commit_user, static_partition: Optional[dict] = None # overwrite commit does not publish changelog manifests; avoid # writing unreferenced changelog files in the first place. writer.file_store_write.changelog_producer = ChangelogProducer.NONE - writer.row_key_extractor = writer._create_row_key_extractor(static_partition) + writer.row_key_extractor = self._create_row_key_extractor(static_partition) def _create_file_store_write(self, commit_user): writer = self.writer @@ -83,18 +83,19 @@ def _create_row_key_extractor(self, static_partition): def write_arrow(self, data, bucket: Optional[int] = None): """Write Arrow data, optionally to a bucket assigned upstream.""" - writer = self.writer - data = writer._prepare_arrow_schema(data) + data = self._prepare_arrow_schema(data) + # Public batch overrides remain observable for Arrow and pandas input. + # Schema, routing and preparation helpers stay within this executor. if isinstance(data, pa.RecordBatch): - return writer.write_arrow_batch(data, bucket) + return self.writer.write_arrow_batch(data, bucket) for batch in data.to_batches(): - writer.write_arrow_batch(batch, bucket) + self.writer.write_arrow_batch(batch, bucket) def write_arrow_batch(self, data: pa.RecordBatch, bucket: Optional[int] = None): writer = self.writer if bucket is not None: - return writer._write_precomputed_bucket(data, bucket) - data = writer._prepare_arrow_data(data) + return self._write_precomputed_bucket(data, bucket) + data = self._prepare_arrow_data(data) if data.num_rows == 0: return @@ -120,7 +121,7 @@ def write_arrow_batch(self, data: pa.RecordBatch, bucket: Optional[int] = None): sub_table = data.slice(lo, count) else: sub_table = pa.compute.take(data, row_indices) - writer._write_partition_bucket_batch(partition, bucket, sub_table) + self._write_partition_bucket_batch(partition, bucket, sub_table) def roll_before_group_if_needed(self, row_count: int): """Roll current files before the next logical write group if needed.""" @@ -153,7 +154,7 @@ def _write_precomputed_bucket(self, data, bucket): upper_bound = MAX_DYNAMIC_BUCKETS if bucket_mode == BucketMode.HASH_DYNAMIC else writer.table.options.bucket() if not 0 <= bucket < upper_bound: raise ValueError('Bucket id must be between 0 and {}'.format(upper_bound - 1)) - data = writer._prepare_arrow_data(data) + data = self._prepare_arrow_data(data) if bucket_mode == BucketMode.HASH_DYNAMIC: partition = writer.row_key_extractor.notify_precomputed_bucket_batch(data, bucket) else: @@ -165,7 +166,7 @@ def _write_precomputed_bucket(self, data, bucket): raise ValueError('A precomputed bucket group contained multiple partitions') if partition is None: return - writer._write_partition_bucket_batch(partition, bucket, data) + self._write_partition_bucket_batch(partition, bucket, data) def write_row(self, row): writer = self.writer @@ -179,7 +180,7 @@ def write_row(self, row): partition, bucket = ( writer.row_key_extractor.extract_partition_bucket_row(values_by_name) ) - writer._write_partition_bucket_row( + self._write_partition_bucket_row( partition, bucket, row, values_by_name ) @@ -198,7 +199,7 @@ def write_pandas(self, dataframe): # Column-subset write (append-only ``with_write_type``): build the # RecordBatch against the subset schema so the input only needs the # written columns, mirroring the ``write_arrow`` path. - pa_schema = writer._write_cols_pyarrow_schema(write_cols) + pa_schema = self._write_cols_pyarrow_schema(write_cols) else: pa_schema = writer.table_pyarrow_schema record_batch = _pandas_write_batch( @@ -249,7 +250,7 @@ def close(self): try: writer.file_store_write.close() finally: - writer._release_prepared_indexes() + self._release_prepared_indexes() def abort(self): writer = self.writer @@ -285,7 +286,7 @@ def prepare_commit(self, commit_identifier) -> List[CommitMessage]: # Hand off data, changelog and HASH indexes together. Until every # preparation succeeds, retain the metadata for retry and abort. writer.file_store_write._release_prepared_files() - writer._release_prepared_indexes() + self._release_prepared_indexes() return commit_messages def _release_prepared_indexes(self) -> None: @@ -296,12 +297,12 @@ def _release_prepared_indexes(self) -> None: def _prepare_arrow_data(self, data): writer = self.writer - data = writer._prepare_arrow_schema(data) + data = self._prepare_arrow_schema(data) # Schema-only writers can normalize input without table options. table = getattr(writer, 'table', None) if table is None: return data - data = writer._normalize_arrow_write_schema(data) + data = self._normalize_arrow_write_schema(data) from pypaimon.write.row_kind import filter_write_batch selected = (writer.file_store_write.write_cols if not table.is_primary_key_table else None) write_cols = selected @@ -326,11 +327,11 @@ def _prepare_arrow_data(self, data): physical = self._project_write_input(data, selected) if selected is not None else data target = writer.table_pyarrow_schema if physical.schema.names != target.names: - target = writer._write_cols_pyarrow_schema(physical.schema.names) + target = self._write_cols_pyarrow_schema(physical.schema.names) # Validate only surviving values. Keep the input transport metadata: # PyArrow otherwise rejects hidden NULL children of NULL ROW parents. validated = normalize_arrow_write_nullability(physical, target) - writer._validate_pyarrow_schema(validated.schema) + self._validate_pyarrow_schema(validated.schema) return data @staticmethod @@ -354,7 +355,7 @@ def _prepare_arrow_schema(self, data): writer = self.writer if getattr(writer, 'table', None) is None: data = normalize_arrow_strings(data) - writer._validate_pyarrow_schema( + self._validate_pyarrow_schema( data.schema, check_nested_nullability=getattr(writer, 'table', None) is None) return data @@ -379,14 +380,14 @@ def _normalize_arrow_write_schema(self, data): def _validate_pyarrow_schema(self, data_schema: pa.Schema, *, check_nested_nullability=True): writer = self.writer - if writer._is_compatible_pyarrow_schema( + if self._is_compatible_pyarrow_schema( data_schema, writer.table_pyarrow_schema, check_nested_nullability=check_nested_nullability): return write_cols = writer.file_store_write.write_cols if write_cols is not None: - write_cols_schema = writer._write_cols_pyarrow_schema(write_cols) - if writer._is_compatible_pyarrow_schema( + write_cols_schema = self._write_cols_pyarrow_schema(write_cols) + if self._is_compatible_pyarrow_schema( data_schema, write_cols_schema, check_nested_nullability=check_nested_nullability): return table = getattr(writer, 'table', None) @@ -396,12 +397,12 @@ def _validate_pyarrow_schema(self, data_schema: pa.Schema, *, check_nested_nulla allowed = selected | self._write_routing_columns() if (len(set(names)) == len(names) and set(names).issubset(allowed) and [name for name in names if name in selected] == write_cols): - transport_schema = writer._write_cols_pyarrow_schema(names) - if writer._is_compatible_pyarrow_schema( + transport_schema = self._write_cols_pyarrow_schema(names) + if self._is_compatible_pyarrow_schema( data_schema, transport_schema, check_nested_nullability=check_nested_nullability): return - writer._raise_inconsistent_schema(data_schema) + self._raise_inconsistent_schema(data_schema) def _is_compatible_pyarrow_schema( self, data_schema: pa.Schema, expected_schema: pa.Schema, *, check_nested_nullability=True) -> bool: diff --git a/paimon-python/pypaimon/ray/data_evolution_merge_join.py b/paimon-python/pypaimon/ray/data_evolution_merge_join.py index db10eb2e92af..433f35a19f78 100644 --- a/paimon-python/pypaimon/ray/data_evolution_merge_join.py +++ b/paimon-python/pypaimon/ray/data_evolution_merge_join.py @@ -162,7 +162,7 @@ def build_self_merge_update_plan( CoreOptions, GlobalIndexSearchMode, ) from pypaimon.table.special_fields import SpecialFields - from pypaimon.write.table_update import TableUpdate + from pypaimon.execution.python.write.table_update import PythonTableUpdate row_id_name = SpecialFields.ROW_ID.name needed_cols = set(resolve_target_projection( @@ -207,7 +207,7 @@ def build_self_merge_update_plan( ) # A packed scan split may contain multiple logical row-id groups. Keep # each group intact, but do not materialize unrelated groups together. - file_groups = list(TableUpdate._predicate_update_file_groups( + file_groups = list(PythonTableUpdate._predicate_update_file_groups( scan_plan.splits() )) update_schema = build_update_schema(target_pa_schema, update_cols, row_id_name) @@ -924,7 +924,7 @@ def distributed_delete_apply( if base_snapshot_id is not None else table ) - planner = TableDeleteByRowId(scan_table) + planner = TableDeleteByRowId._from_python(scan_table) anchor_info = planner._snapshot_anchor_ranges() if not anchor_info.anchors: return [], 0, [] diff --git a/paimon-python/pypaimon/tests/arrow_schema_test.py b/paimon-python/pypaimon/tests/arrow_schema_test.py index db65dbb675d8..a0f26438aa85 100644 --- a/paimon-python/pypaimon/tests/arrow_schema_test.py +++ b/paimon-python/pypaimon/tests/arrow_schema_test.py @@ -28,6 +28,7 @@ from pypaimon import CatalogFactory, Schema from pypaimon.schema.arrow_schema import arrow_schemas_compatible, normalize_arrow_strings from pypaimon.write.table_write import TableWrite +from pypaimon.execution.python.write.table_write import PythonTableWrite class ArrowSchemaTest(unittest.TestCase): @@ -130,7 +131,7 @@ def test_schema_only_writer_normalizes_input(self, entry): if entry == 'batch': source = source.to_batches()[0] - result = writer._prepare_arrow_data(source) + result = PythonTableWrite(writer)._prepare_arrow_data(source) self.assertIsInstance(result, type(source)) self.assertEqual(result.schema, writer.table_pyarrow_schema) diff --git a/paimon-python/pypaimon/tests/deletion_vector_path_test.py b/paimon-python/pypaimon/tests/deletion_vector_path_test.py index 981fe8dd8259..56f603d72613 100644 --- a/paimon-python/pypaimon/tests/deletion_vector_path_test.py +++ b/paimon-python/pypaimon/tests/deletion_vector_path_test.py @@ -17,6 +17,7 @@ """Deletion-vector locations must survive writes, further deletes and time travel.""" +from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId import os import subprocess import sys @@ -36,7 +37,6 @@ from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.schema.data_types import AtomicType from pypaimon.write.commit_message import CommitMessage -from pypaimon.write.table_delete import TableDeleteByRowId _PLANNERS = ['python', pytest.param('native', marks=[pytest.mark.native_plan, pytest.mark.skipif( @@ -474,7 +474,7 @@ def test_multiple_dv_index_files_in_one_bucket_keep_all_deletions(tmp_path, plan commit.close() _delete(table, [0, 4]) old = _entries(table, 3)[0] - deleter = TableDeleteByRowId(table) + deleter = PythonTableDeleteByRowId().initialize(table) _, vectors = deleter._read_existing_deletion_vectors(old.partition, old.bucket, 3) assert len(vectors) == 2 # Java rolls DV index files by target size, so one bucket can legitimately diff --git a/paimon-python/pypaimon/tests/native_authorized_update_test.py b/paimon-python/pypaimon/tests/native_authorized_update_test.py index 46321821b245..366990c3793b 100644 --- a/paimon-python/pypaimon/tests/native_authorized_update_test.py +++ b/paimon-python/pypaimon/tests/native_authorized_update_test.py @@ -87,9 +87,10 @@ def _rows(table): def _native_only(stack, use_native): if use_native: for method in ( - 'pypaimon.write.table_update_by_row_id.TableUpdateByRowId._load_existing_files_info', - 'pypaimon.write.table_upsert_by_key.TableUpsertByKey._upsert_partition', - 'pypaimon.write.table_delete.TableDeleteByRowId._scan_anchor_ranges', + 'pypaimon.execution.python.write.table_update_by_row_id.' + 'PythonTableUpdateByRowId._load_existing_files_info', + 'pypaimon.execution.python.write.table_upsert_by_key.PythonTableUpsert._upsert_partition', + 'pypaimon.execution.python.write.table_delete.PythonTableDeleteByRowId._scan_anchor_ranges', 'pypaimon.write.table_update.ShardTableUpdator', 'pypaimon.execution.python.write.merge_into._build_tables', 'pypaimon.read.table_read.TableRead.to_arrow'): diff --git a/paimon-python/pypaimon/tests/native_blob_update_test.py b/paimon-python/pypaimon/tests/native_blob_update_test.py index 4598ab355ac2..19e2968f2b6f 100644 --- a/paimon-python/pypaimon/tests/native_blob_update_test.py +++ b/paimon-python/pypaimon/tests/native_blob_update_test.py @@ -17,6 +17,7 @@ """Native updates share the Java Blob placeholder and row-id layout.""" +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId from contextlib import ExitStack from unittest.mock import patch from urllib.parse import urlparse @@ -34,7 +35,6 @@ from pypaimon.execution.native.write.native_update import NativeTableUpdateByRowId from pypaimon.write.write_dispatch import NativeTableWrite from pypaimon.write.table_update import TableUpdate -from pypaimon.write.table_update_by_row_id import TableUpdateByRowId from pypaimon.write.table_upsert_by_key import TableUpsertByKey pytestmark = pytest.mark.native_plan @@ -87,7 +87,7 @@ def _read(table, descriptors=False, projection=None, resolve_views=True): def _native_only(): stack = ExitStack() - for cls, method in [(TableUpdateByRowId, '_load_existing_files_info'), + for cls, method in [(PythonTableUpdateByRowId, '_load_existing_files_info'), (TableUpdate, '_build_predicate_update_table'), (TableUpsertByKey, '_upsert_partition'), (TableUpsertByKey, '_upsert_row_partition'), diff --git a/paimon-python/pypaimon/tests/native_branch_write_test.py b/paimon-python/pypaimon/tests/native_branch_write_test.py index 03ca2d0e1562..eb71cc56e318 100644 --- a/paimon-python/pypaimon/tests/native_branch_write_test.py +++ b/paimon-python/pypaimon/tests/native_branch_write_test.py @@ -22,6 +22,8 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert +from pypaimon.execution.python.write.table_update import PythonTableUpdate from pypaimon.common.identifier import Identifier from pypaimon.common.options.core_options import MergeEngine from pypaimon.write.write_dispatch import NativeTableWrite @@ -299,8 +301,6 @@ def test_native_branch_schema_evolution_is_independent(tmp_path, native_rest_cat @pytest.mark.parametrize('rest', [False, True], ids=['filesystem', 'rest']) def test_native_branch_predicate_upsert_and_batch_updates(tmp_path, native_rest_catalog, rest): from pypaimon.read.table_read import TableRead - from pypaimon.write.table_update import BatchTableUpdate - from pypaimon.write.table_upsert_by_key import TableUpsertByKey from pypaimon.write.table_delete import TableDeleteByRowId catalog, main = _table(tmp_path, native_rest_catalog, rest, { @@ -316,7 +316,7 @@ def test_native_branch_predicate_upsert_and_batch_updates(tmp_path, native_rest_ predicate, {'value': lambda batch: pa.compute.add(batch['value'], 1)}, ['value']) _commit(builder, messages) builder = branch.new_batch_write_builder() - with patch.object(TableUpsertByKey, '_upsert_partition', + with patch.object(PythonTableUpsert, '_upsert_partition', side_effect=AssertionError('Python upsert fallback')): messages = builder.new_update().upsert_by_arrow_with_key(pa.Table.from_pylist([ {'id': 1, 'value': 12, 'p': 'a'}, {'id': 3, 'value': 30, 'p': 'c'}], schema=_SCHEMA), ['id']) @@ -328,7 +328,7 @@ def test_native_branch_predicate_upsert_and_batch_updates(tmp_path, native_rest_ ids = {row['id']: row['_ROW_ID'] for row in read.new_read().to_arrow( read.new_scan().plan().splits()).to_pylist()} builder = branch.new_batch_write_builder() - with patch.object(BatchTableUpdate, '_update_by_arrow_batches_with_row_id', + with patch.object(PythonTableUpdate, '_update_by_arrow_batches_with_row_id', side_effect=AssertionError('Python grouped update fallback')): messages = builder.new_update().with_update_type(['value']).update_by_arrow_batches_with_row_id([ pa.table({'_ROW_ID': [ids[1]], 'value': [13]}), diff --git a/paimon-python/pypaimon/tests/native_data_directory_test.py b/paimon-python/pypaimon/tests/native_data_directory_test.py index 579cde26ed79..0c49b8adb101 100644 --- a/paimon-python/pypaimon/tests/native_data_directory_test.py +++ b/paimon-python/pypaimon/tests/native_data_directory_test.py @@ -26,10 +26,10 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert from pypaimon.execution.native.read.plan import (native_plan) from pypaimon.schema.data_types import AtomicType, DataField from pypaimon.write.write_dispatch import NativeTableWrite -from pypaimon.write.table_upsert_by_key import TableUpsertByKey pytestmark = pytest.mark.native_plan @@ -171,7 +171,7 @@ def test_data_directory_updates_deletes_history_and_abort(tmp_path, directory, n update = builder.new_update().with_update_type(['value']) with ExitStack() as stack: if native: - stack.enter_context(patch.object(TableUpsertByKey, '_upsert_partition', + stack.enter_context(patch.object(PythonTableUpsert, '_upsert_partition', side_effect=AssertionError('Python upsert fallback'))) messages = update.upsert_by_arrow_with_key( pa.table({'id': [1, 4], 'pt': ['a', 'b'], 'value': [11, 40]}, schema=_SCHEMA), ['id']) diff --git a/paimon-python/pypaimon/tests/native_decimal_predicate_test.py b/paimon-python/pypaimon/tests/native_decimal_predicate_test.py index 9ee484c58501..b527e6cec452 100644 --- a/paimon-python/pypaimon/tests/native_decimal_predicate_test.py +++ b/paimon-python/pypaimon/tests/native_decimal_predicate_test.py @@ -25,9 +25,9 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.write.table_update import PythonTableUpdate from pypaimon.common.predicate import Predicate from pypaimon.globalindex.create_global_index import GlobalIndexBuilder -from pypaimon.write.table_update import TableUpdate pytestmark = pytest.mark.native_plan @@ -148,7 +148,7 @@ def test_decimal_predicates_select_the_same_rows_for_updates_and_deletes(tmp_pat with ExitStack() as stack: if native: for method in ['_build_predicate_update_table', '_matched_delete_row_ids']: - stack.enter_context(patch.object(TableUpdate, method, + stack.enter_context(patch.object(PythonTableUpdate, method, side_effect=AssertionError('Python matching fallback'))) if operation == 'delete': messages = update.delete_by_predicate(predicate, 19) if stream else update.delete_by_predicate(predicate) diff --git a/paimon-python/pypaimon/tests/native_historical_update_test.py b/paimon-python/pypaimon/tests/native_historical_update_test.py index f9ee0d05335f..7bf94265916d 100644 --- a/paimon-python/pypaimon/tests/native_historical_update_test.py +++ b/paimon-python/pypaimon/tests/native_historical_update_test.py @@ -182,9 +182,10 @@ def _append_mutation(table, native, operation, only_matched=False, stream=False) schema=pa.schema([('id', pa.int32()), ('value', pa.int32()), ('keep', pa.int32())])) with ExitStack() as stack: if native: - for method in ('pypaimon.write.table_upsert_by_key.TableUpsertByKey._upsert_partition', - 'pypaimon.write.table_upsert_by_key.TableUpsertByKey._upsert_row_partition', - 'pypaimon.execution.python.write.merge_into._build_tables'): + for method in ( + 'pypaimon.execution.python.write.table_upsert_by_key.PythonTableUpsert._upsert_partition', + 'pypaimon.execution.python.write.table_upsert_by_key.PythonTableUpsert._upsert_row_partition', + 'pypaimon.execution.python.write.merge_into._build_tables'): stack.enter_context(patch(method, side_effect=AssertionError('Python historical mutation fallback'))) if operation == 'upsert': return update.upsert_by_arrow_with_key(data, ['id'], **kwargs) diff --git a/paimon-python/pypaimon/tests/native_map_shredding_write_test.py b/paimon-python/pypaimon/tests/native_map_shredding_write_test.py index e067e67d96b0..df647e636176 100644 --- a/paimon-python/pypaimon/tests/native_map_shredding_write_test.py +++ b/paimon-python/pypaimon/tests/native_map_shredding_write_test.py @@ -17,6 +17,7 @@ """Native MAP shared-shredding writes must preserve Java's physical contract.""" +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId from contextlib import nullcontext from unittest.mock import patch @@ -239,7 +240,6 @@ def test_native_map_abort_removes_rolled_files(tmp_path): @pytest.mark.parametrize('native_update', [False, True]) def test_map_row_id_update_keeps_physical_layout(tmp_path, native_update): - from pypaimon.write.table_update_by_row_id import TableUpdateByRowId table, schema = _table(tmp_path, 'evolution') _write(table, schema, [ @@ -253,7 +253,7 @@ def test_map_row_id_update_keeps_physical_layout(tmp_path, native_update): ], schema=update_schema) table = table.copy({'write.native.enabled': str(native_update).lower()}) builder = table.new_batch_write_builder() - guard = patch.object(TableUpdateByRowId, '_load_existing_files_info', + guard = patch.object(PythonTableUpdateByRowId, '_load_existing_files_info', side_effect=AssertionError('Python update fallback')) with guard if native_update else nullcontext(): messages = builder.new_update().with_update_type(['metrics']).update_by_arrow_with_row_id(update) @@ -279,7 +279,6 @@ def test_native_map_predicate_update_and_upsert(tmp_path, stream, operation): from contextlib import ExitStack from pypaimon.common.predicate_builder import PredicateBuilder from pypaimon.write.table_update import TableUpdate - from pypaimon.write.table_update_by_row_id import TableUpdateByRowId from pypaimon.write.table_upsert_by_key import TableUpsertByKey table, schema = _table(tmp_path, 'evolution') @@ -291,7 +290,7 @@ def test_native_map_predicate_update_and_upsert(tmp_path, stream, operation): update = builder.new_update().with_update_type(['metrics']) replacement = [('new', 7), ('a', None)] with ExitStack() as stack: - for cls, method in [(TableUpdateByRowId, '_load_existing_files_info'), + for cls, method in [(PythonTableUpdateByRowId, '_load_existing_files_info'), (TableUpdate, '_build_predicate_update_table'), (TableUpsertByKey, '_upsert_partition')]: stack.enter_context(patch.object(cls, method, side_effect=AssertionError('Python update fallback'))) diff --git a/paimon-python/pypaimon/tests/native_nested_update_test.py b/paimon-python/pypaimon/tests/native_nested_update_test.py index 9ae2aaf385ba..ed5df8a02df0 100644 --- a/paimon-python/pypaimon/tests/native_nested_update_test.py +++ b/paimon-python/pypaimon/tests/native_nested_update_test.py @@ -17,6 +17,7 @@ """Nested Native update dispatch and parity with the Python implementation.""" +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId from contextlib import ExitStack from unittest.mock import patch @@ -27,7 +28,6 @@ from pypaimon.common.predicate_builder import PredicateBuilder from pypaimon.table.row.generic_row import GenericRow from pypaimon.write.table_update import TableUpdate -from pypaimon.write.table_update_by_row_id import TableUpdateByRowId from pypaimon.write.table_upsert_by_key import TableUpsertByKey @@ -69,7 +69,7 @@ def _table(tmp_path, data_type, native): def _native_only(native): stack = ExitStack() if native: - for cls, method in [(TableUpdateByRowId, '_load_existing_files_info'), + for cls, method in [(PythonTableUpdateByRowId, '_load_existing_files_info'), (TableUpdate, '_build_predicate_update_table'), (TableUpsertByKey, '_upsert_partition'), (TableUpsertByKey, '_upsert_row_partition')]: @@ -201,7 +201,7 @@ def test_nested_row_id_coercion_matches_python(tmp_path, native, input_values, t data = pa.table({'_ROW_ID': list(range(len(input_values))), 'value': input_values}) before = set(tmp_path.rglob('*.parquet')) try: - expected = TableUpdateByRowId._coerce_column(input_values, target).to_pylist() + expected = PythonTableUpdateByRowId._coerce_column(input_values, target).to_pylist() except (ValueError, TypeError, pa.ArrowException): with _native_only(native), pytest.raises((ValueError, TypeError, pa.ArrowException)): update.update_by_arrow_with_row_id(data) diff --git a/paimon-python/pypaimon/tests/native_parquet_vector_update_test.py b/paimon-python/pypaimon/tests/native_parquet_vector_update_test.py index 4fdbf44e17ad..06f3c8dc7d1e 100644 --- a/paimon-python/pypaimon/tests/native_parquet_vector_update_test.py +++ b/paimon-python/pypaimon/tests/native_parquet_vector_update_test.py @@ -17,6 +17,7 @@ """Parquet vector sidecars must not force row-level updates back to Python.""" +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId from contextlib import ExitStack from unittest.mock import patch @@ -29,7 +30,6 @@ from pypaimon.schema.data_types import PyarrowFieldParser from pypaimon.table.data_evolution_merge_into import WhenMatched, WhenNotMatched, source_col from pypaimon.write.table_update import TableUpdate -from pypaimon.write.table_update_by_row_id import TableUpdateByRowId from pypaimon.write.table_upsert_by_key import TableUpsertByKey @@ -76,7 +76,7 @@ def _rows(table): def _native_only(native): stack = ExitStack() if native: - for owner, method in [(TableUpdateByRowId, '_load_existing_files_info'), + for owner, method in [(PythonTableUpdateByRowId, '_load_existing_files_info'), (TableUpsertByKey, '_upsert_partition'), (TableUpdate, '_build_predicate_update_table')]: stack.enter_context(patch.object(owner, method, side_effect=AssertionError('Python update fallback'))) diff --git a/paimon-python/pypaimon/tests/native_partition_delete_changelog_test.py b/paimon-python/pypaimon/tests/native_partition_delete_changelog_test.py index 73d4f073c29d..7bc668456228 100644 --- a/paimon-python/pypaimon/tests/native_partition_delete_changelog_test.py +++ b/paimon-python/pypaimon/tests/native_partition_delete_changelog_test.py @@ -24,11 +24,11 @@ import pytest from pypaimon import Schema +from pypaimon.execution.python.write.table_update import PythonTableUpdate from pypaimon.common.predicate import Predicate from pypaimon.common.predicate_builder import PredicateBuilder from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.execution.native.write.native_update import create_native_predicate_delete, create_native_update -from pypaimon.write.table_update import TableUpdate pytestmark = [pytest.mark.python_write, pytest.mark.python_commit] @@ -91,7 +91,7 @@ def test_input_changelog_partition_delete_keeps_other_partitions(native_rest_cat with ExitStack() as stack: if native: stack.enter_context(patch.object( - TableUpdate, '_delete_by_partition_filter', side_effect=AssertionError('Python delete fallback'))) + PythonTableUpdate, '_delete_by_partition_filter', side_effect=AssertionError('Python delete fallback'))) messages = builder.new_update().delete_by_predicate(predicate) assert [tuple(message.partition) for message in messages] == [('a',)] assert sum(len(message.deleted_files) for message in messages) == 1 diff --git a/paimon-python/pypaimon/tests/native_partition_types_test.py b/paimon-python/pypaimon/tests/native_partition_types_test.py index 836120d77322..708747573b72 100644 --- a/paimon-python/pypaimon/tests/native_partition_types_test.py +++ b/paimon-python/pypaimon/tests/native_partition_types_test.py @@ -117,7 +117,7 @@ def test_native_evolution_updates_use_typed_partition_values(tmp_path, partition rows = [dict(id=i, p=value, value=i) for i, value in enumerate(values)] _write(table, schema, rows, stream) # Fail if the Python update planner or file writer takes over the operation. - with patch('pypaimon.write.table_update.BatchTableUpdate._build_predicate_update_table', + with patch('pypaimon.execution.python.write.table_update.PythonTableUpdate._build_predicate_update_table', side_effect=AssertionError('Python row-ID update')): builder = table.new_batch_write_builder() messages = builder.new_update().update_by_predicate(None, {'value': 99}) diff --git a/paimon-python/pypaimon/tests/native_plan_capabilities_test.py b/paimon-python/pypaimon/tests/native_plan_capabilities_test.py index 54616c644366..d9dc6fd7540d 100644 --- a/paimon-python/pypaimon/tests/native_plan_capabilities_test.py +++ b/paimon-python/pypaimon/tests/native_plan_capabilities_test.py @@ -17,6 +17,7 @@ """Native/Python planner parity on persisted snapshots and data files.""" +from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId import json import tempfile import unittest @@ -37,7 +38,6 @@ from pypaimon.table.row.generic_row import GenericRow from pypaimon.utils.range import Range from pypaimon.write.commit_message import CommitMessage -from pypaimon.write.table_delete import TableDeleteByRowId @pytest.mark.native_plan @@ -310,7 +310,7 @@ def test_primary_key_deletion_vectors_preserve_compacted_rows(self): vector.delete(3) # Python's delete API is DE-only. Use its production bitmap writer to # create the same index payload for this compacted primary-key fixture. - index_entry = TableDeleteByRowId(table)._write_deletion_vector_index( + index_entry = PythonTableDeleteByRowId().initialize(table)._write_deletion_vector_index( GenericRow([], []), 0, {data_file.file_name: vector}) commit = table.new_batch_write_builder().new_commit() try: diff --git a/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py b/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py index 967bd0ebda94..c005c9e9f0da 100644 --- a/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py +++ b/paimon-python/pypaimon/tests/native_plan_chunk_shuffle_test.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId import json from collections import Counter from unittest.mock import patch @@ -26,7 +27,6 @@ from pypaimon.execution.python.deletionvectors.bitmap_deletion_vector import BitmapDeletionVector from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.write.commit_message import CommitMessage -from pypaimon.write.table_delete import TableDeleteByRowId pytestmark = [pytest.mark.native_plan, pytest.mark.skipif( @@ -95,7 +95,7 @@ def _create_chunk_table(kind, tmp_path): vector = BitmapDeletionVector() vector.delete(0) index_adds.append((tuple(split.partition.values), split.bucket, - TableDeleteByRowId(table)._write_deletion_vector_index( + PythonTableDeleteByRowId().initialize(table)._write_deletion_vector_index( split.partition, split.bucket, {file.file_name: vector}))) commit = table.new_batch_write_builder().new_commit() try: diff --git a/paimon-python/pypaimon/tests/native_plan_distribution_test.py b/paimon-python/pypaimon/tests/native_plan_distribution_test.py index 28582efb4270..5e4dd0b8e120 100644 --- a/paimon-python/pypaimon/tests/native_plan_distribution_test.py +++ b/paimon-python/pypaimon/tests/native_plan_distribution_test.py @@ -17,6 +17,7 @@ """Distributed native planning must preserve assignment before reader limits.""" +from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId import tempfile import unittest from contextlib import ExitStack @@ -33,7 +34,6 @@ from pypaimon.table.row.generic_row import GenericRow from pypaimon.utils.range import Range from pypaimon.write.commit_message import CommitMessage -from pypaimon.write.table_delete import TableDeleteByRowId class _DistributionFixture: @@ -160,7 +160,7 @@ def _append_dv_table(self, name, deleted_positions): vector = BitmapDeletionVector() for position in deleted_positions: vector.delete(position) - entry = TableDeleteByRowId(table)._write_deletion_vector_index( + entry = PythonTableDeleteByRowId().initialize(table)._write_deletion_vector_index( GenericRow([], []), 0, {first_file.file_name: vector}) commit = table.new_batch_write_builder().new_commit() try: @@ -309,7 +309,7 @@ def test_raw_primary_key_shards_distribute_files_within_one_bucket(self): vector = BitmapDeletionVector() vector.delete(0) vectors[files[key].file_name] = vector - entry = TableDeleteByRowId(table)._write_deletion_vector_index( + entry = PythonTableDeleteByRowId().initialize(table)._write_deletion_vector_index( GenericRow([], []), 0, vectors) commit = table.new_batch_write_builder().new_commit() try: diff --git a/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py b/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py index 93227d3a82da..58e036255416 100644 --- a/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py +++ b/paimon-python/pypaimon/tests/native_plan_materialized_pk_test.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId from contextlib import ExitStack from dataclasses import replace from pathlib import Path @@ -29,7 +30,6 @@ from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.table.row.generic_row import GenericRow from pypaimon.write.commit_message import CommitMessage -from pypaimon.write.table_delete import TableDeleteByRowId pytestmark = [pytest.mark.native_plan, pytest.mark.skipif( @@ -78,7 +78,7 @@ def test_clustered_materialized_dv_files_use_native_raw_splits(tmp_path, engine, vector = BitmapDeletionVector() vector.delete(0) # clustered first file starts with id=3 - entry = TableDeleteByRowId(table)._write_deletion_vector_index( + entry = PythonTableDeleteByRowId().initialize(table)._write_deletion_vector_index( GenericRow([], []), 0, {files[0].file_name: vector}) commit = table.new_batch_write_builder().new_commit() try: @@ -174,7 +174,7 @@ def test_first_row_level_zero_merges_before_filtering(tmp_path, target_size, com commit.close() vector = BitmapDeletionVector() vector.delete(1) # Remove id=2 from the first sorted L0 run. - entry = TableDeleteByRowId(table)._write_deletion_vector_index( + entry = PythonTableDeleteByRowId().initialize(table)._write_deletion_vector_index( GenericRow(['pending'], table.partition_keys_fields), 0, {first_file.file_name: vector}) commit = table.new_batch_write_builder().new_commit() try: @@ -255,7 +255,7 @@ def test_pk_merge_engines_read_deletion_vectors_natively(tmp_path, engine, merge deleted_position = pq.read_table(first_path).column('id').to_pylist().index(2) vector = BitmapDeletionVector() vector.delete(deleted_position) - entry = TableDeleteByRowId(table)._write_deletion_vector_index( + entry = PythonTableDeleteByRowId().initialize(table)._write_deletion_vector_index( GenericRow([], []), 0, {first_file.file_name: vector}) commit = table.new_batch_write_builder().new_commit() try: diff --git a/paimon-python/pypaimon/tests/native_predicate_delete_test.py b/paimon-python/pypaimon/tests/native_predicate_delete_test.py index 9d2580cc424f..2d0c68d345c4 100644 --- a/paimon-python/pypaimon/tests/native_predicate_delete_test.py +++ b/paimon-python/pypaimon/tests/native_predicate_delete_test.py @@ -22,9 +22,9 @@ import pytest from pypaimon import Schema +from pypaimon.execution.python.write.table_update import PythonTableUpdate from pypaimon.manifest.index_manifest_file import IndexManifestFile from pypaimon.tests import native_plan_rest_test -from pypaimon.write.table_update import TableUpdate pytestmark = pytest.mark.native_plan @@ -74,8 +74,8 @@ def _ids(table): def _core_only(): stack = ExitStack() - for method in ('_delete_by_partition_filter', '_matched_delete_row_ids', '_delete_by_row_id'): - stack.enter_context(patch.object(TableUpdate, method, side_effect=AssertionError( + for method in ('_delete_by_partition_filter', '_matched_delete_row_ids', 'delete_by_row_id'): + stack.enter_context(patch.object(PythonTableUpdate, method, side_effect=AssertionError( 'Python predicate deletion selected: ' + method))) return stack diff --git a/paimon-python/pypaimon/tests/native_shard_update_test.py b/paimon-python/pypaimon/tests/native_shard_update_test.py index 6daff43262ba..403c1b397ed6 100644 --- a/paimon-python/pypaimon/tests/native_shard_update_test.py +++ b/paimon-python/pypaimon/tests/native_shard_update_test.py @@ -68,6 +68,46 @@ def _rows(table): return builder.new_read().to_arrow(builder.new_scan().plan().splits()).sort_by('id').to_pydict() +@pytest.mark.parametrize('operation', ['update', 'delete', 'shard']) +def test_direct_public_mutation_constructors_select_native(native_rest_catalog, operation): + from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER + from pypaimon.write.table_delete import TableDeleteByRowId + from pypaimon.write.table_update import ShardTableUpdator + from pypaimon.write.table_update_by_row_id import TableUpdateByRowId + + table, _ = _table(native_rest_catalog, [ + dict(p='a', id=1, value=10), dict(p='a', id=2, value=20)]) + builder = table.new_batch_write_builder() + if operation == 'update': + with patch('pypaimon.execution.python.write.table_update_by_row_id.PythonTableUpdateByRowId.initialize', + side_effect=AssertionError('Direct update initialized Python')): + update = TableUpdateByRowId(table, builder.commit_user, BATCH_COMMIT_IDENTIFIER) + messages = update.update_columns(pa.table({'_ROW_ID': [0], 'value': [11]}), ['value']) + assert len(update.commit_messages) == len(messages) + expected = dict(p=['a', 'a'], id=[1, 2], value=[11, 20]) + elif operation == 'delete': + with patch('pypaimon.execution.python.write.table_delete.PythonTableDeleteByRowId.initialize', + side_effect=AssertionError('Direct delete initialized Python')): + messages = TableDeleteByRowId(table).delete([0, 0]) + expected = dict(p=['a'], id=[2], value=[20]) + else: + with patch('pypaimon.execution.python.write.table_update.ShardTableUpdator.initialize', + side_effect=AssertionError('Direct shard initialized Python')): + update = ShardTableUpdator(table, ['id'], ['value'], builder.commit_user, 0, 1) + try: + with update.arrow_reader() as reader: + for batch in reader: + update.update_by_arrow_batch(pa.record_batch([ + pc.multiply(batch['id'], 100)], names=['value'])) + messages = update.prepare_commit() + finally: + update.close() + expected = dict(p=['a', 'a'], id=[1, 2], value=[100, 200]) + assert messages and all(message.check_from_snapshot == 1 for message in messages) + builder.new_commit().commit(messages) + assert _rows(table) == expected + + @pytest.mark.parametrize('stream', [False, True]) def test_native_shard_updates_use_the_captured_reader_order(native_rest_catalog, stream): table, schema = _table(native_rest_catalog, [ diff --git a/paimon-python/pypaimon/tests/native_sliced_split_read_test.py b/paimon-python/pypaimon/tests/native_sliced_split_read_test.py index 3de22c889750..6b4d755a19a3 100644 --- a/paimon-python/pypaimon/tests/native_sliced_split_read_test.py +++ b/paimon-python/pypaimon/tests/native_sliced_split_read_test.py @@ -17,6 +17,7 @@ """Python-planned file slices remain selectors when read with Native.""" +from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId import json from contextlib import ExitStack from types import SimpleNamespace @@ -45,7 +46,6 @@ from pypaimon.tests import native_plan_rest_test from pypaimon.utils.range import Range from pypaimon.write.commit_message import CommitMessage -from pypaimon.write.table_delete import TableDeleteByRowId pytestmark = [pytest.mark.python_plan, pytest.mark.python_write, pytest.mark.python_commit] @@ -85,7 +85,7 @@ def _table(catalog, tracked, deleted=False): files = plan.splits()[0].files vector = BitmapDeletionVector() vector.delete(0) - entry = TableDeleteByRowId(table)._write_deletion_vector_index( + entry = PythonTableDeleteByRowId().initialize(table)._write_deletion_vector_index( plan.splits()[0].partition, 0, {files[1].file_name: vector}) commit = table.new_batch_write_builder().new_commit() try: diff --git a/paimon-python/pypaimon/tests/native_update_container_casts_test.py b/paimon-python/pypaimon/tests/native_update_container_casts_test.py index 12e86897c981..3b2be5e9842d 100644 --- a/paimon-python/pypaimon/tests/native_update_container_casts_test.py +++ b/paimon-python/pypaimon/tests/native_update_container_casts_test.py @@ -17,6 +17,7 @@ """Equivalent Arrow container inputs must work on the existing update APIs.""" +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId from contextlib import ExitStack from decimal import Decimal from unittest.mock import patch @@ -28,7 +29,6 @@ from pypaimon.common.predicate_builder import PredicateBuilder from pypaimon.table.data_evolution_merge_into import WhenMatched, source_col from pypaimon.write.table_update import TableUpdate -from pypaimon.write.table_update_by_row_id import TableUpdateByRowId from pypaimon.write.table_upsert_by_key import TableUpsertByKey @@ -95,7 +95,7 @@ def _table(tmp_path, value_type, native): def _native_only(native): stack = ExitStack() if native: - for cls, method in [(TableUpdateByRowId, '_load_existing_files_info'), + for cls, method in [(PythonTableUpdateByRowId, '_load_existing_files_info'), (TableUpdate, '_build_predicate_update_table'), (TableUpsertByKey, '_upsert_partition')]: stack.enter_context(patch.object(cls, method, side_effect=AssertionError('Python update fallback'))) diff --git a/paimon-python/pypaimon/tests/native_update_inactive_vector_option_test.py b/paimon-python/pypaimon/tests/native_update_inactive_vector_option_test.py index cd92414d9db3..083845654f70 100644 --- a/paimon-python/pypaimon/tests/native_update_inactive_vector_option_test.py +++ b/paimon-python/pypaimon/tests/native_update_inactive_vector_option_test.py @@ -17,6 +17,8 @@ """An unused vector format must not disable Parquet/Blob Native mutations.""" +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId +from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId from contextlib import ExitStack from unittest.mock import patch @@ -30,9 +32,7 @@ from pypaimon.table.row.generic_row import GenericRow from pypaimon.tests import native_plan_rest_test from pypaimon.execution.native.write.native_update import _native_update_formats_supported -from pypaimon.write.table_delete import TableDeleteByRowId from pypaimon.write.table_update import TableUpdate -from pypaimon.write.table_update_by_row_id import TableUpdateByRowId from pypaimon.write.table_upsert_by_key import TableUpsertByKey @@ -83,11 +83,11 @@ def _read(table): def _native_only(native): stack = ExitStack() if native: - for owner, method in [(TableUpdateByRowId, '_load_existing_files_info'), + for owner, method in [(PythonTableUpdateByRowId, '_load_existing_files_info'), (TableUpsertByKey, '_upsert_partition'), (TableUpsertByKey, '_upsert_row_partition'), (TableUpdate, '_build_predicate_update_table'), - (TableDeleteByRowId, '_scan_anchor_ranges')]: + (PythonTableDeleteByRowId, '_scan_anchor_ranges')]: stack.enter_context(patch.object(owner, method, side_effect=AssertionError('Python mutation fallback'))) stack.enter_context(patch('pypaimon.execution.python.write.merge_into._build_tables', side_effect=AssertionError('Python MERGE fallback'))) diff --git a/paimon-python/pypaimon/tests/native_update_source_nullability_test.py b/paimon-python/pypaimon/tests/native_update_source_nullability_test.py index dee3e7ca99b8..aa81f0bdd26f 100644 --- a/paimon-python/pypaimon/tests/native_update_source_nullability_test.py +++ b/paimon-python/pypaimon/tests/native_update_source_nullability_test.py @@ -81,8 +81,9 @@ def _messages(table, source, native, stream, operation): with ExitStack() as stack: if native: for method in ('_upsert_partition', '_upsert_row_partition'): - stack.enter_context(patch('pypaimon.write.table_upsert_by_key.TableUpsertByKey.' + method, - side_effect=AssertionError('Python upsert fallback'))) + stack.enter_context(patch( + 'pypaimon.execution.python.write.table_upsert_by_key.PythonTableUpsert.' + method, + side_effect=AssertionError('Python upsert fallback'))) stack.enter_context(patch('pypaimon.execution.python.write.merge_into._build_tables', side_effect=AssertionError('Python MERGE fallback'))) if operation == 'merge': @@ -208,7 +209,8 @@ def test_selected_blob_nullability_applies_to_dedicated_files( with ExitStack() as stack: if native: stack.enter_context(patch( - 'pypaimon.write.table_update_by_row_id.TableUpdateByRowId._load_existing_files_info', + 'pypaimon.execution.python.write.table_update_by_row_id.' + 'PythonTableUpdateByRowId._load_existing_files_info', side_effect=AssertionError('Python row-ID update fallback'))) updater = builder.new_update().new_update_by_row_id() @@ -252,7 +254,8 @@ def test_not_null_blob_containers_keep_nullable_children( with ExitStack() as stack: if native: stack.enter_context(patch( - 'pypaimon.write.table_update_by_row_id.TableUpdateByRowId._load_existing_files_info', + 'pypaimon.execution.python.write.table_update_by_row_id.' + 'PythonTableUpdateByRowId._load_existing_files_info', side_effect=AssertionError('Python row-ID update fallback'))) messages = builder.new_update().new_update_by_row_id().update_columns(source, ['payload']) commit = builder.new_commit() diff --git a/paimon-python/pypaimon/tests/native_update_test.py b/paimon-python/pypaimon/tests/native_update_test.py index 1cfa95b6ab6c..894b54426aba 100644 --- a/paimon-python/pypaimon/tests/native_update_test.py +++ b/paimon-python/pypaimon/tests/native_update_test.py @@ -17,18 +17,19 @@ """End-to-end coverage of the optional native batch row-ID update bridge.""" +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId from unittest.mock import patch import pyarrow as pa import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert +from pypaimon.execution.python.write.table_update import PythonTableUpdate from pypaimon.execution.native.write.native_update import create_native_delete, create_native_update from pypaimon.write.table_delete import TableDeleteByRowId from pypaimon.read.table_read import TableRead -from pypaimon.write.table_update import BatchTableUpdate, StreamTableUpdate from pypaimon.write.table_update_by_row_id import TableUpdateByRowId -from pypaimon.write.table_upsert_by_key import TableUpsertByKey pytestmark = pytest.mark.native_plan @@ -58,7 +59,7 @@ def test_incremental_row_id_updater_uses_core_and_accumulates_columns(tmp_path, commit_id = 27 if stream else BATCH_COMMIT_IDENTIFIER # Factory configuration must not fix columns for this incremental API. update = builder.new_update().with_update_type(['id']) - with patch.object(TableUpdateByRowId, '_load_existing_files_info', + with patch.object(PythonTableUpdateByRowId, '_load_existing_files_info', side_effect=AssertionError('Python loaded the file index')): updater = update.new_update_by_row_id(commit_id) if stream else update.new_update_by_row_id() assert isinstance(updater, NativeTableUpdateByRowId) @@ -109,7 +110,7 @@ def test_partitioned_upsert_and_incremental_update_use_core(tmp_path, stream): builder = table.new_stream_write_builder() if stream else table.new_batch_write_builder() update = builder.new_update() input_rows = pa.table({'p': ['a', 'c', None], 'id': [1, 1, 1], 'v': [11, 40, 31]}, schema=schema) - with patch.object(TableUpsertByKey, '_upsert_partition', + with patch.object(PythonTableUpsert, '_upsert_partition', side_effect=AssertionError('Python partition upsert selected')): messages = (update.upsert_by_arrow_with_key(input_rows, ['id'], 1) if stream else update.upsert_by_arrow_with_key(input_rows, ['id'])) @@ -118,7 +119,7 @@ def test_partitioned_upsert_and_incremental_update_use_core(tmp_path, stream): commit.close() builder = table.new_stream_write_builder() if stream else table.new_batch_write_builder() update = builder.new_update() - with patch.object(TableUpsertByKey, '_upsert_row_partition', + with patch.object(PythonTableUpsert, '_upsert_row_partition', side_effect=AssertionError('Python partition row upsert selected')): rows = [GenericRow(['a', 1, 12], table.fields)] messages = (update.upsert_by_key(rows, ['id'], 2) if stream @@ -272,7 +273,7 @@ def test_stream_update_and_delete_use_native_writers(tmp_path): committer.commit(messages, 10) predicate = table.new_read_builder().new_predicate_builder() - with patch.object(StreamTableUpdate, '_build_predicate_update_table', + with patch.object(PythonTableUpdate, '_build_predicate_update_table', side_effect=AssertionError('Python assignments selected')), \ patch.object(TableRead, 'to_arrow', side_effect=AssertionError('Python predicate read selected')): @@ -324,7 +325,7 @@ def test_native_batch_update_preserves_input_table_boundaries(tmp_path): pa.record_batch([pa.array([1], type=pa.int64()), pa.array([22])], names=['_ROW_ID', 'age']), ]) - with patch.object(BatchTableUpdate, '_update_by_arrow_batches_with_row_id', + with patch.object(PythonTableUpdate, '_update_by_arrow_batches_with_row_id', side_effect=AssertionError('Python batch update selected')): with pytest.raises(ValueError, match='overlapping first_row_ids.*0'): table.new_batch_write_builder().new_update().with_update_type( @@ -371,7 +372,7 @@ def interleaved_tables(): concurrent.new_commit().commit(changed.update_by_arrow_with_row_id( pa.Table.from_pydict({'_ROW_ID': [1], 'age': [23]}))) - with patch.object(BatchTableUpdate, '_update_by_arrow_batches_with_row_id', + with patch.object(PythonTableUpdate, '_update_by_arrow_batches_with_row_id', side_effect=AssertionError('Python batch update selected')): staged = (table.new_batch_write_builder().new_update() .with_update_type(['age']) @@ -405,9 +406,9 @@ def test_native_predicate_update_invokes_callable_by_file_group(tmp_path): seen = [] predicate = table.new_read_builder().new_predicate_builder().greater_or_equal('id', 2) builder = table.new_batch_write_builder() - with patch.object(BatchTableUpdate, '_build_predicate_update_table', + with patch.object(PythonTableUpdate, '_build_predicate_update_table', side_effect=AssertionError('Python assignments selected')), \ - patch.object(BatchTableUpdate, '_matched_update_scan_table', + patch.object(PythonTableUpdate, '_matched_update_scan_table', side_effect=AssertionError('Python scan planning selected')): messages = builder.new_update().update_by_predicate( predicate, @@ -450,12 +451,12 @@ def test_native_upsert_matches_duplicate_source_and_target_keys(tmp_path): 'id': [1, 1, 3], 'age': [100, 101, 30], }, schema=schema) builder = table.new_batch_write_builder() - with patch.object(TableUpsertByKey, '_upsert_partition', + with patch.object(PythonTableUpsert, '_upsert_partition', side_effect=AssertionError('Python upsert selected')): messages = builder.new_update().upsert_by_arrow_with_key(updates, ['id']) builder.new_commit().commit(messages) builder = table.new_batch_write_builder() - with patch.object(TableUpsertByKey, '_upsert_row_partition', + with patch.object(PythonTableUpsert, '_upsert_row_partition', side_effect=AssertionError('Python row upsert selected')): messages = builder.new_update().upsert_by_key([ GenericRow([1, 201], table.fields), @@ -474,7 +475,7 @@ def test_native_upsert_matches_duplicate_source_and_target_keys(tmp_path): stream_updates = pa.Table.from_pydict({ 'id': [2, 5], 'age': [25, 50], }, schema=schema) - with patch.object(TableUpsertByKey, '_upsert_partition', + with patch.object(PythonTableUpsert, '_upsert_partition', side_effect=AssertionError('Python stream selected')): messages = stream.new_update().upsert_by_arrow_with_key( stream_updates, ['id'], 77) @@ -513,9 +514,9 @@ def test_native_upsert_matches_duplicate_source_and_target_keys(tmp_path): before = set(tmp_path.rglob('*.parquet')) snapshot_id = table.snapshot_manager().get_latest_snapshot().id with patch.object(NativeTableUpsert, 'upsert', side_effect=RuntimeError('core upsert failed')), \ - patch.object(TableUpsertByKey, '_upsert_partition', + patch.object(PythonTableUpsert, '_upsert_partition', side_effect=AssertionError('retried as Python Arrow upsert')), \ - patch.object(TableUpsertByKey, '_upsert_row_partition', + patch.object(PythonTableUpsert, '_upsert_row_partition', side_effect=AssertionError('retried as Python row upsert')): with pytest.raises(RuntimeError, match='core upsert failed'): builder.new_update().upsert_by_arrow_with_key(fallback_updates, ['id']) @@ -680,7 +681,7 @@ def assign(matched): assignment = assign if callable_assignment else pa.chunked_array( [[101], [102, 103, 104]], type=pa.int32()) - guard = (patch.object(BatchTableUpdate, '_matched_update_scan_table', + guard = (patch.object(PythonTableUpdate, '_matched_update_scan_table', side_effect=AssertionError('Python planning selected')) if native else nullcontext()) with guard: @@ -726,7 +727,7 @@ def test_native_row_upsert_uses_public_operation_with_composite_null_keys(tmp_pa ]) builder = table.new_stream_write_builder() if stream else table.new_batch_write_builder() update = builder.new_update().with_update_type(['age']) - with patch.object(TableUpsertByKey, '_upsert_row_partition', + with patch.object(PythonTableUpsert, '_upsert_row_partition', side_effect=AssertionError('Python upsert selected')): if stream: messages = update.upsert_by_key(rows, ['id', 'part'], 88) @@ -784,8 +785,7 @@ def test_row_upsert_input_capabilities(tmp_path, case): 'id': [1, 2], 'age': [10, 20], 'region': ['east', 'west'], } return - from pypaimon.write.table_upsert_by_key import TableUpsertByKey - with patch.object(TableUpsertByKey, '_upsert_row_partition', + with patch.object(PythonTableUpsert, '_upsert_row_partition', side_effect=AssertionError('Python upsert fallback')): messages = builder.new_update().with_update_type(['age']).upsert_by_key(rows, ['id']) builder.new_commit().commit(messages) @@ -866,7 +866,7 @@ def prepare(): return updater.update_by_arrow_with_row_id(data) try: - expected = TableUpdateByRowId._coerce_column(values, target).to_pylist() + expected = PythonTableUpdateByRowId._coerce_column(values, target).to_pylist() except (ValueError, pa.ArrowException): with pytest.raises((ValueError, pa.ArrowException)): prepare() @@ -954,7 +954,7 @@ def test_core_nested_row_update_uses_whole_column_constructor_fallback(tmp_path, data = pa.table({'_ROW_ID': list(range(len(values))), 'value': values}) before = set(tmp_path.rglob('*.parquet')) try: - expected = TableUpdateByRowId._coerce_column(values, target).to_pylist() + expected = PythonTableUpdateByRowId._coerce_column(values, target).to_pylist() except (ValueError, pa.ArrowException): with pytest.raises((ValueError, pa.ArrowException)): builder.new_update().update_by_arrow_with_row_id(data) diff --git a/paimon-python/pypaimon/tests/native_upsert_capabilities_test.py b/paimon-python/pypaimon/tests/native_upsert_capabilities_test.py index b1ae5e3f1151..ed29b4b78b0a 100644 --- a/paimon-python/pypaimon/tests/native_upsert_capabilities_test.py +++ b/paimon-python/pypaimon/tests/native_upsert_capabilities_test.py @@ -24,9 +24,9 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert from pypaimon.schema.data_types import PyarrowFieldParser from pypaimon.table.row.generic_row import GenericRow -from pypaimon.write.table_upsert_by_key import TableUpsertByKey pytestmark = pytest.mark.native_plan @@ -68,7 +68,7 @@ def test_partial_upsert_keys_and_values(tmp_path, native, streaming, row_input, with ExitStack() as stack: if native: for method in ('_upsert_partition', '_upsert_row_partition'): - stack.enter_context(patch.object(TableUpsertByKey, method, + stack.enter_context(patch.object(PythonTableUpsert, method, side_effect=AssertionError('Python upsert fallback'))) if row_input: fields = [field for field in table.fields if field.name in source.column_names] @@ -140,7 +140,7 @@ def test_temporal_upsert_keys(tmp_path, native, streaming, row_input, key_type): with ExitStack() as stack: if native: for method in ('_upsert_partition', '_upsert_row_partition'): - stack.enter_context(patch.object(TableUpsertByKey, method, + stack.enter_context(patch.object(PythonTableUpsert, method, side_effect=AssertionError('Python upsert fallback'))) if row_input: rows = [GenericRow([row[field.name] for field in table.fields], table.fields) @@ -204,7 +204,7 @@ def relevant(path): 'sub': [1, 2, 1, 1], 'value': [15, 30, 40, 20]}, schema=schema) with ExitStack() as stack: if native: - stack.enter_context(patch.object(TableUpsertByKey, '_upsert_partition', + stack.enter_context(patch.object(PythonTableUpsert, '_upsert_partition', side_effect=AssertionError('Python upsert fallback'))) messages = builder.new_update().with_update_type(['value']).upsert_by_arrow_with_key( source, ['id']) diff --git a/paimon-python/pypaimon/tests/native_upsert_encoded_keys_test.py b/paimon-python/pypaimon/tests/native_upsert_encoded_keys_test.py index d6338e5f1867..087577c72423 100644 --- a/paimon-python/pypaimon/tests/native_upsert_encoded_keys_test.py +++ b/paimon-python/pypaimon/tests/native_upsert_encoded_keys_test.py @@ -26,11 +26,11 @@ import pytest from pypaimon import Schema +from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.execution.native.write.native_update import ( _native_upsert_key_compatible, _native_upsert_key_type_compatible, create_native_upsert, ) -from pypaimon.write.table_upsert_by_key import TableUpsertByKey pytestmark = [pytest.mark.python_write, pytest.mark.python_commit] @@ -118,9 +118,9 @@ def _table(catalog, key_type, keys, native, guarded): def _native_only(stack, native): if native: - stack.enter_context(patch.object(TableUpsertByKey, '_upsert_partition', + stack.enter_context(patch.object(PythonTableUpsert, '_upsert_partition', side_effect=AssertionError('Python Arrow upsert fallback'))) - stack.enter_context(patch.object(TableUpsertByKey, '_upsert_row_partition', + stack.enter_context(patch.object(PythonTableUpsert, '_upsert_row_partition', side_effect=AssertionError('Python Row upsert fallback'))) diff --git a/paimon-python/pypaimon/tests/native_upsert_key_normalization_test.py b/paimon-python/pypaimon/tests/native_upsert_key_normalization_test.py index d1ad0a63d058..7165cebedc50 100644 --- a/paimon-python/pypaimon/tests/native_upsert_key_normalization_test.py +++ b/paimon-python/pypaimon/tests/native_upsert_key_normalization_test.py @@ -30,8 +30,8 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert from pypaimon.schema.data_types import PyarrowFieldParser -from pypaimon.write.table_upsert_by_key import TableUpsertByKey pytestmark = pytest.mark.native_plan @@ -123,7 +123,7 @@ def _upsert(table, source, native, stream, columns=('value',)): with ExitStack() as stack: if native: stack.enter_context(patch.object( - TableUpsertByKey, '_upsert_partition', side_effect=AssertionError('Python key matching'))) + PythonTableUpsert, '_upsert_partition', side_effect=AssertionError('Python key matching'))) update = builder.new_update() if columns is not None: update.with_update_type(list(columns)) diff --git a/paimon-python/pypaimon/tests/native_upsert_layout_test.py b/paimon-python/pypaimon/tests/native_upsert_layout_test.py index daed72532fb9..223fda60793c 100644 --- a/paimon-python/pypaimon/tests/native_upsert_layout_test.py +++ b/paimon-python/pypaimon/tests/native_upsert_layout_test.py @@ -17,6 +17,7 @@ """Equivalent Arrow layouts must not send upsert matching back to Python.""" +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId from contextlib import ExitStack from unittest.mock import patch @@ -26,7 +27,6 @@ from pypaimon import CatalogFactory, Schema from pypaimon.schema.data_types import PyarrowFieldParser from pypaimon.write.table_update import TableUpdate -from pypaimon.write.table_update_by_row_id import TableUpdateByRowId from pypaimon.write.table_upsert_by_key import TableUpsertByKey @@ -80,7 +80,7 @@ def _native_only(native): stack = ExitStack() if native: for owner, method in [(TableUpsertByKey, '_upsert_partition'), - (TableUpdateByRowId, '_load_existing_files_info'), + (PythonTableUpdateByRowId, '_load_existing_files_info'), (TableUpdate, '_build_predicate_update_table')]: stack.enter_context(patch.object(owner, method, side_effect=AssertionError('Python update fallback'))) return stack diff --git a/paimon-python/pypaimon/tests/native_upsert_row_key_identity_test.py b/paimon-python/pypaimon/tests/native_upsert_row_key_identity_test.py index e2380ed548cb..f5d316508901 100644 --- a/paimon-python/pypaimon/tests/native_upsert_row_key_identity_test.py +++ b/paimon-python/pypaimon/tests/native_upsert_row_key_identity_test.py @@ -24,6 +24,7 @@ import pytest from pypaimon import Schema +from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.schema.data_types import AtomicType from pypaimon.schema.schema_change import SchemaChange @@ -87,10 +88,10 @@ def test_lossy_row_key_encoding_cannot_update_an_existing_key( fields = Schema.from_pyarrow_schema(pa.schema([('key', source_type), ('value', pa.int32())])).fields row = GenericRow([key, 99], fields) builder = target.new_stream_write_builder() if stream else target.new_batch_write_builder() - original = TableUpsertByKey._upsert_row_partition + original = PythonTableUpsert._upsert_row_partition # The unsafe encoding is rejected before starting a core mutation. Python # still matches with the original key and constructs a distinct new row. - with patch.object(TableUpsertByKey, '_upsert_row_partition', autospec=True, side_effect=original) as fallback: + with patch.object(PythonTableUpsert, '_upsert_row_partition', autospec=True, side_effect=original) as fallback: update = builder.new_update().with_update_type(['value']) messages = update.upsert_by_key([row], ['key'], 17) if stream else update.upsert_by_key([row], ['key']) fallback.assert_called_once() @@ -122,7 +123,7 @@ def test_exact_row_key_encodings_still_use_native_upsert( row = GenericRow([key, 99], fields) with ExitStack() as stack: if native: - stack.enter_context(patch.object(TableUpsertByKey, '_upsert_row_partition', + stack.enter_context(patch.object(PythonTableUpsert, '_upsert_row_partition', side_effect=AssertionError('Python exact-key fallback'))) update = target.new_batch_write_builder().new_update().with_update_type(['value']) messages = update.upsert_by_key([row], ['key']) @@ -136,8 +137,8 @@ def test_large_integer_row_keys_retain_the_existing_constructor_fallback(native_ key = 2 ** 53 + 2 latest, target = _tables(native_rest_catalog, pa.float64(), float(key), native, False) fields = Schema.from_pyarrow_schema(pa.schema([('key', pa.int64()), ('value', pa.int32())])).fields - original = TableUpsertByKey._upsert_row_partition - with patch.object(TableUpsertByKey, '_upsert_row_partition', autospec=True, side_effect=original) as fallback: + original = PythonTableUpsert._upsert_row_partition + with patch.object(PythonTableUpsert, '_upsert_row_partition', autospec=True, side_effect=original) as fallback: messages = target.new_batch_write_builder().new_update().with_update_type(['value']).upsert_by_key( [GenericRow([key, 99], fields)], ['key']) # PyArrow's constructor rejects Python integers outside its consecutive @@ -196,8 +197,8 @@ def test_text_binary_row_key_constructors_cannot_change_match_identity( native_rest_catalog, native, key_type, initial, source_type, key): latest, target = _tables(native_rest_catalog, key_type, initial, native, False) fields = Schema.from_pyarrow_schema(pa.schema([('key', source_type), ('value', pa.int32())])).fields - original = TableUpsertByKey._upsert_row_partition - with patch.object(TableUpsertByKey, '_upsert_row_partition', autospec=True, side_effect=original) as fallback: + original = PythonTableUpsert._upsert_row_partition + with patch.object(PythonTableUpsert, '_upsert_row_partition', autospec=True, side_effect=original) as fallback: messages = target.new_batch_write_builder().new_update().with_update_type(['value']).upsert_by_key( [GenericRow([key, 99], fields)], ['key']) fallback.assert_called_once() diff --git a/paimon-python/pypaimon/tests/native_upsert_row_shapes_test.py b/paimon-python/pypaimon/tests/native_upsert_row_shapes_test.py index 5b4eea832c6b..53b046f0d95d 100644 --- a/paimon-python/pypaimon/tests/native_upsert_row_shapes_test.py +++ b/paimon-python/pypaimon/tests/native_upsert_row_shapes_test.py @@ -83,10 +83,10 @@ def _upsert(table, native, stream, rows, columns=('value',)): with ExitStack() as stack: if native: stack.enter_context(patch( - 'pypaimon.write.table_upsert_by_key.TableUpsertByKey._upsert_row_partition', + 'pypaimon.execution.python.write.table_upsert_by_key.PythonTableUpsert._upsert_row_partition', side_effect=AssertionError('named-row upsert fell back to Python'))) stack.enter_context(patch( - 'pypaimon.write.table_upsert_by_key.TableUpsertByKey._build_key_to_row_ids_map', + 'pypaimon.execution.python.write.table_upsert_by_key.PythonTableUpsert._build_key_to_row_ids_map', side_effect=AssertionError('upsert keys were matched in Python'))) messages = (update.upsert_by_key(rows, ['id'], 101) if stream else update.upsert_by_key(rows, ['id'])) diff --git a/paimon-python/pypaimon/tests/native_upsert_selected_values_test.py b/paimon-python/pypaimon/tests/native_upsert_selected_values_test.py index 9a6f6df9a686..b329c548071a 100644 --- a/paimon-python/pypaimon/tests/native_upsert_selected_values_test.py +++ b/paimon-python/pypaimon/tests/native_upsert_selected_values_test.py @@ -26,8 +26,8 @@ import pytest from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert from pypaimon.schema.data_types import PyarrowFieldParser -from pypaimon.write.table_upsert_by_key import TableUpsertByKey pytestmark = pytest.mark.native_plan @@ -67,7 +67,7 @@ def _upsert(table, source, native, stream, columns=('value',)): with ExitStack() as stack: if native: stack.enter_context(patch.object( - TableUpsertByKey, '_upsert_partition', + PythonTableUpsert, '_upsert_partition', side_effect=AssertionError('upsert fell back to Python'))) update = builder.new_update().with_update_type(list(columns)) messages = (update.upsert_by_arrow_with_key(source, ['key'], 17) if stream diff --git a/paimon-python/pypaimon/tests/native_upsert_temporal_row_keys_test.py b/paimon-python/pypaimon/tests/native_upsert_temporal_row_keys_test.py index 443eaa0812f3..0986d57d0379 100644 --- a/paimon-python/pypaimon/tests/native_upsert_temporal_row_keys_test.py +++ b/paimon-python/pypaimon/tests/native_upsert_temporal_row_keys_test.py @@ -26,12 +26,12 @@ import pytest from pypaimon import Schema +from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.schema.data_types import AtomicType from pypaimon.schema.schema_change import SchemaChange from pypaimon.table.row.generic_row import GenericRow from pypaimon.execution.native.write.native_update import NativeTableUpsert, _upsert_row_batches, create_native_upsert -from pypaimon.write.table_upsert_by_key import TableUpsertByKey pytestmark = [pytest.mark.python_plan, pytest.mark.python_read, @@ -130,8 +130,8 @@ def test_lossy_temporal_row_key_does_not_overwrite_existing_rows( fields = Schema.from_pyarrow_schema(pa.schema([('key', source_type), ('value', pa.int32())])).fields row = GenericRow([value, 99], fields) builder = target.new_stream_write_builder() if stream else target.new_batch_write_builder() - original = TableUpsertByKey._upsert_row_partition - with patch.object(TableUpsertByKey, '_upsert_row_partition', autospec=True, side_effect=original) as fallback: + original = PythonTableUpsert._upsert_row_partition + with patch.object(PythonTableUpsert, '_upsert_row_partition', autospec=True, side_effect=original) as fallback: update = builder.new_update().with_update_type(['value']) messages = update.upsert_by_key([row], ['key'], 17) if stream else update.upsert_by_key([row], ['key']) fallback.assert_called_once() @@ -158,7 +158,7 @@ def test_exact_temporal_row_keys_keep_native_matching(native_rest_catalog, nativ fields = Schema.from_pyarrow_schema(pa.schema([('key', source_type), ('value', pa.int32())])).fields with ExitStack() as stack: if native: - stack.enter_context(patch.object(TableUpsertByKey, '_upsert_row_partition', + stack.enter_context(patch.object(PythonTableUpsert, '_upsert_row_partition', side_effect=AssertionError('Exact temporal key fell back'))) messages = target.new_batch_write_builder().new_update().with_update_type(['value']).upsert_by_key( [GenericRow([value, 99], fields)], ['key']) @@ -185,8 +185,8 @@ def test_integer_row_keys_are_not_reinterpreted_as_existing_temporal_keys(native key_type, initial, source_type, value = _case(kind) latest, target = _tables(native_rest_catalog, key_type, initial, native, False) fields = Schema.from_pyarrow_schema(pa.schema([('key', source_type), ('value', pa.int32())])).fields - original = TableUpsertByKey._upsert_row_partition - with patch.object(TableUpsertByKey, '_upsert_row_partition', autospec=True, side_effect=original) as fallback: + original = PythonTableUpsert._upsert_row_partition + with patch.object(PythonTableUpsert, '_upsert_row_partition', autospec=True, side_effect=original) as fallback: messages = target.new_batch_write_builder().new_update().with_update_type(['value']).upsert_by_key( [GenericRow([value, 99], fields)], ['key']) fallback.assert_called_once() diff --git a/paimon-python/pypaimon/tests/native_variant_update_test.py b/paimon-python/pypaimon/tests/native_variant_update_test.py index 93c546581918..0335cf9956db 100644 --- a/paimon-python/pypaimon/tests/native_variant_update_test.py +++ b/paimon-python/pypaimon/tests/native_variant_update_test.py @@ -17,6 +17,7 @@ """Native partial updates retain Java's shredded VARIANT physical layout.""" +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId import json from contextlib import ExitStack from unittest.mock import patch @@ -32,7 +33,6 @@ from pypaimon.execution.native.write.native_update import NativeTableUpdateByRowId from pypaimon.write.write_dispatch import NativeTableWrite from pypaimon.write.table_update import TableUpdate -from pypaimon.write.table_update_by_row_id import TableUpdateByRowId from pypaimon.write.table_upsert_by_key import TableUpsertByKey pytestmark = pytest.mark.native_plan @@ -106,7 +106,7 @@ def _read(table, native, projection=None): def _native_only(): stack = ExitStack() - for cls, method in [(TableUpdateByRowId, '_load_existing_files_info'), + for cls, method in [(PythonTableUpdateByRowId, '_load_existing_files_info'), (TableUpdate, '_build_predicate_update_table'), (TableUpsertByKey, '_upsert_partition')]: stack.enter_context(patch.object(cls, method, side_effect=AssertionError('Python update fallback'))) diff --git a/paimon-python/pypaimon/tests/native_video_read_test.py b/paimon-python/pypaimon/tests/native_video_read_test.py index dc6d4fd7a341..83441fd99de2 100644 --- a/paimon-python/pypaimon/tests/native_video_read_test.py +++ b/paimon-python/pypaimon/tests/native_video_read_test.py @@ -130,7 +130,7 @@ def test_native_update_of_normal_columns_keeps_packed_video_frames(tmp_path, nat before = _read(table, True)[0].to_pylist() builder = table.new_batch_write_builder() update = builder.new_update().with_update_type(['id']) - with patch('pypaimon.write.table_update.BatchTableUpdate._update_by_arrow_batches_with_row_id', + with patch('pypaimon.execution.python.write.table_update.PythonTableUpdate._update_by_arrow_batches_with_row_id', side_effect=AssertionError('Python update fallback')): messages = update.update_by_arrow_batches_with_row_id(iter([ pa.table({'_ROW_ID': [0], 'id': [20]}, schema=pa.schema([('_ROW_ID', pa.int64()), ('id', pa.int32())]))])) diff --git a/paimon-python/pypaimon/tests/native_write_routing_test.py b/paimon-python/pypaimon/tests/native_write_routing_test.py index 38e7ecf1c561..bd6b6f83fb18 100644 --- a/paimon-python/pypaimon/tests/native_write_routing_test.py +++ b/paimon-python/pypaimon/tests/native_write_routing_test.py @@ -110,7 +110,7 @@ def test_native_postpone_ray_preserves_fixed_bucket_planner(tmp_path, plan, over @pytest.mark.parametrize('explicit_plan', [False, True]) def test_ray_postpone_plans_use_effective_overwrite(tmp_path, native_writer, mode, explicit_plan): from pypaimon.write.postpone_batch_table_write import PostponeFixedBucketBatchTableWrite - from pypaimon.write.postpone_bucket import PostponeBucketPlanner + from pypaimon.execution.python.write.postpone_bucket import PythonPostponeBucketPlanner as PostponeBucketPlanner table = _table(tmp_path, 'postpone') plan = PostponeBucketPlan({('a',): 8}) if explicit_plan else None diff --git a/paimon-python/pypaimon/tests/reader_predicate_test.py b/paimon-python/pypaimon/tests/reader_predicate_test.py index 054f324bf947..30b7b4ba6fce 100644 --- a/paimon-python/pypaimon/tests/reader_predicate_test.py +++ b/paimon-python/pypaimon/tests/reader_predicate_test.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId import os import shutil import tempfile @@ -38,7 +39,6 @@ def test_predicate_preserves_deletion_vector_positions(tmp_path, mode, batch_siz from pypaimon.execution.python.deletionvectors.bitmap_deletion_vector import BitmapDeletionVector from pypaimon.table.row.generic_row import GenericRow from pypaimon.write.commit_message import CommitMessage - from pypaimon.write.table_delete import TableDeleteByRowId catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) catalog.create_database('default', False) @@ -68,7 +68,7 @@ def test_predicate_preserves_deletion_vector_positions(tmp_path, mode, batch_siz commit.close() vector = BitmapDeletionVector() vector.delete(1) - entry = TableDeleteByRowId(table)._write_deletion_vector_index( + entry = PythonTableDeleteByRowId().initialize(table)._write_deletion_vector_index( GenericRow([], []), messages[0].bucket, {messages[0].new_files[0].file_name: vector}) commit = builder.new_commit() try: diff --git a/paimon-python/pypaimon/tests/shard_table_updator_test.py b/paimon-python/pypaimon/tests/shard_table_updator_test.py index e799f5c6a531..8e39fed553ef 100644 --- a/paimon-python/pypaimon/tests/shard_table_updator_test.py +++ b/paimon-python/pypaimon/tests/shard_table_updator_test.py @@ -25,6 +25,7 @@ import pyarrow as pa from pypaimon import CatalogFactory, Schema +from pypaimon.execution.python.write.table_update import PythonTableUpdate class ShardUpdateValidationTest(unittest.TestCase): @@ -41,7 +42,7 @@ def test_empty_update_columns_are_rejected_before_selecting_a_backend(self): update = update_type(table, 'writer').with_update_type([]) with patch('pypaimon.write.update_dispatch.prepare_native_write_operation', side_effect=AssertionError('Prepared a backend for invalid columns')) as prepare, \ - patch.object(update, '_new_python_shard_updator', + patch.object(PythonTableUpdate, '_new_python_shard_updator', side_effect=AssertionError('Created a Python shard updater')) as python: with self.assertRaisesRegex(ValueError, 'Shard update columns must not be empty'): update.new_shard_updator(0, 1) diff --git a/paimon-python/pypaimon/tests/table_update_arrow_compatibility_test.py b/paimon-python/pypaimon/tests/table_update_arrow_compatibility_test.py index 28e72a054ada..4434b94a52af 100644 --- a/paimon-python/pypaimon/tests/table_update_arrow_compatibility_test.py +++ b/paimon-python/pypaimon/tests/table_update_arrow_compatibility_test.py @@ -18,7 +18,7 @@ import pyarrow as pa import pytest -from pypaimon.write.table_update import BatchTableUpdate +from pypaimon.execution.python.write.table_update import PythonTableUpdate WRAPPERS = [ @@ -59,16 +59,16 @@ def test_predicate_assignment_struct_cast(wrap_type, wrap_value, chunked, projec if projection == "reordered": with pytest.raises(ValueError, match="struct fields must match in order"): - BatchTableUpdate._assignment_to_array(values, target, 2) + PythonTableUpdate._assignment_to_array(values, target, 2) else: try: expected = values.cast(target).to_pylist() except (pa.ArrowNotImplementedError, pa.ArrowTypeError) as error: # Older Arrow versions reject unsupported STRUCT field projections. with pytest.raises(type(error)) as raised: - BatchTableUpdate._assignment_to_array(values, target, 2) + PythonTableUpdate._assignment_to_array(values, target, 2) assert str(raised.value) == str(error) return - result = BatchTableUpdate._assignment_to_array(values, target, 2) + result = PythonTableUpdate._assignment_to_array(values, target, 2) assert result.type == target assert result.to_pylist() == expected diff --git a/paimon-python/pypaimon/tests/table_update_by_row_id_chunked_test.py b/paimon-python/pypaimon/tests/table_update_by_row_id_chunked_test.py index 7994c2a27d1e..e8f8a096e30e 100644 --- a/paimon-python/pypaimon/tests/table_update_by_row_id_chunked_test.py +++ b/paimon-python/pypaimon/tests/table_update_by_row_id_chunked_test.py @@ -25,8 +25,8 @@ from pypaimon.common.options.core_options import ChangelogProducer from pypaimon.table.special_fields import SpecialFields -from pypaimon.write.table_update_by_row_id import ( - TableUpdateByRowId, +from pypaimon.execution.python.write.table_update_by_row_id import ( + PythonTableUpdateByRowId as TableUpdateByRowId, ) from pypaimon.execution.python.write.table_update_by_row_id import ( _RowIdUpdateFileWriter, diff --git a/paimon-python/pypaimon/tests/table_update_test.py b/paimon-python/pypaimon/tests/table_update_test.py index c225b6db54ff..6987c931664d 100644 --- a/paimon-python/pypaimon/tests/table_update_test.py +++ b/paimon-python/pypaimon/tests/table_update_test.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId import os import random import string @@ -25,6 +26,7 @@ import pyarrow as pa import pytest +from pypaimon.execution.python.write.table_update import PythonTableUpdate from pypaimon.read.read_builder import ReadBuilder from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER from pypaimon.tests.data_evolution_test_helpers import ( @@ -70,7 +72,7 @@ def test_write_builder_creates_row_id_updater_with_commit_identity( assert updater.table is table assert updater.commit_user == update.commit_user == builder.commit_user assert updater.commit_identifier == commit_identifier - assert updater.snapshot_id == 42 + assert updater._snapshot_files_info().snapshot_id == 42 table.new_read_builder.assert_not_called() @@ -113,7 +115,7 @@ def test_batch_row_id_update_batches_reuse_file_index(): ] with mock.patch.object( - BatchTableUpdate, "_new_row_id_updater") as factory: + PythonTableUpdate, "_new_row_id_updater") as factory: updater = factory.return_value updater.commit_messages = [] @@ -137,7 +139,7 @@ def update_columns(batch, columns): def test_batch_row_id_update_detects_overlap_before_write(): - updater = TableUpdateByRowId.__new__(TableUpdateByRowId) + updater = PythonTableUpdateByRowId.__new__(PythonTableUpdateByRowId) updater.table = mock.Mock(field_names=['age']) updater.commit_messages = [] updater._updated_first_row_ids_by_column = {} @@ -1702,7 +1704,7 @@ def fail_after_prepare_commit( wb = self._make_write_builder(table) tu = wb.new_update().with_update_type(columns) with mock.patch.object( - TableUpdateByRowId, + PythonTableUpdateByRowId, '_assign_update_file_metadata', new=staticmethod(fail_after_prepare_commit)): with self.assertRaisesRegex( @@ -1822,7 +1824,6 @@ def test_update_batches_allow_same_file_for_different_columns(self): ) def test_callable_output_preserves_large_offset_chunks(self): - from pypaimon.write.table_update import TableUpdate from pypaimon.write.table_update_by_row_id import TableUpdateByRowId chunks = [ @@ -1834,14 +1835,14 @@ def test_callable_output_preserves_large_offset_chunks(self): ] value = pa.chunked_array(chunks) - result = TableUpdate._assignment_to_array(value, value.type, 2) + result = PythonTableUpdate._assignment_to_array(value, value.type, 2) self.assertEqual(2, result.num_chunks) self.assertEqual(2_200_000_000, sum( len(chunk.values) for chunk in result.chunks )) - updater = TableUpdateByRowId.__new__(TableUpdateByRowId) + updater = PythonTableUpdateByRowId.__new__(PythonTableUpdateByRowId) updater.table = mock.Mock(field_names=['payload']) updater.commit_messages = [] updater._updated_first_row_ids_by_column = {} diff --git a/paimon-python/pypaimon/tests/table_upsert_by_key_test.py b/paimon-python/pypaimon/tests/table_upsert_by_key_test.py index 824b04b89c47..24d8cae32030 100644 --- a/paimon-python/pypaimon/tests/table_upsert_by_key_test.py +++ b/paimon-python/pypaimon/tests/table_upsert_by_key_test.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId import os from contextlib import contextmanager import unittest @@ -750,7 +751,6 @@ def test_commit_rewrite_uses_checked_base_entries(self): self._compact_all_data_files(table) from pypaimon.execution.python.write.row_id_file_index import RowIdFileIndex - from pypaimon.write.table_update_by_row_id import TableUpdateByRowId original_build = RowIdFileIndex.from_entries advanced = [False] @@ -769,7 +769,7 @@ def build_after_concurrent_compaction( return original_build(current_table, snapshot_id, entries) with mock.patch.object( - TableUpdateByRowId, + PythonTableUpdateByRowId, '_load_existing_files_info', side_effect=AssertionError("unexpected snapshot scan"), ), mock.patch.object( diff --git a/paimon-python/pypaimon/tests/upsert_btree_lookup_test.py b/paimon-python/pypaimon/tests/upsert_btree_lookup_test.py index f3b731acb8d4..0044190c97b0 100644 --- a/paimon-python/pypaimon/tests/upsert_btree_lookup_test.py +++ b/paimon-python/pypaimon/tests/upsert_btree_lookup_test.py @@ -23,6 +23,7 @@ import pyarrow as pa +from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert from pypaimon.index.data_evolution_index_source_meta import DataEvolutionIndexSourceMeta from pypaimon.index.index_file_handler import IndexFileHandler from pypaimon.manifest.schema.data_file_meta import DataFileMeta @@ -51,7 +52,7 @@ class UpsertBTreeLookupTest(BatchModeMixin, DataEvolutionTestBase, unittest.Test } def _lookup(self, table, keys, key_set, partition=None): - return TableUpsertByKey(table, 'lookup', 1)._build_key_to_row_ids_map( + return PythonTableUpsert(TableUpsertByKey(table, 'lookup', 1))._build_key_to_row_ids_map( keys, partition or {}, key_set) def _counted_lookup(self, table, keys, key_set): diff --git a/paimon-python/pypaimon/tests/video_column_reassign_test.py b/paimon-python/pypaimon/tests/video_column_reassign_test.py index bcd8cd3ba06e..0f8c018ed37f 100644 --- a/paimon-python/pypaimon/tests/video_column_reassign_test.py +++ b/paimon-python/pypaimon/tests/video_column_reassign_test.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId from dataclasses import replace from uuid import uuid4 @@ -27,7 +28,6 @@ from pypaimon.schema.data_types import AtomicType from pypaimon.schema.schema_change import SchemaChange from pypaimon.snapshot.renaming_snapshot_commit import RenamingSnapshotCommit -from pypaimon.write.table_update_by_row_id import TableUpdateByRowId def _publish_reassigned_partition(table, partition_value): @@ -91,13 +91,13 @@ def concurrent_commit(): _publish_reassigned_partition(raw, 'a' if concurrent_change == 'same_partition' else 'b') injected.append(raw.snapshot_manager().get_latest_snapshot().id) - real_init = TableUpdateByRowId.__init__ + real_init = PythonTableUpdateByRowId.initialize real_commit = RenamingSnapshotCommit.commit def init_after_reassign(updater, *args, **kwargs): if not injected: concurrent_commit() - real_init(updater, *args, **kwargs) + return real_init(updater, *args, **kwargs) def commit_after_reassign(committer, base_uuid, snapshot, statistics): if not injected and snapshot.commit_kind == 'APPEND': @@ -108,7 +108,7 @@ def commit_after_reassign(committer, base_uuid, snapshot, statistics): with monkeypatch.context() as patch: if phase == 'after_mapping': - patch.setattr(TableUpdateByRowId, '__init__', init_after_reassign) + patch.setattr(PythonTableUpdateByRowId, 'initialize', init_after_reassign) else: patch.setattr(RenamingSnapshotCommit, 'commit', commit_after_reassign) if concurrent_change == 'same_partition': diff --git a/paimon-python/pypaimon/tests/write/backend_boundary_test.py b/paimon-python/pypaimon/tests/write/backend_boundary_test.py index 7af31c2cf735..bac2672f1a2e 100644 --- a/paimon-python/pypaimon/tests/write/backend_boundary_test.py +++ b/paimon-python/pypaimon/tests/write/backend_boundary_test.py @@ -180,4 +180,5 @@ def no_native(name, *args, **kwargs): return original_import(name, *args, **kwargs) with patch('builtins.__import__', side_effect=no_native): - assert updater._matched_delete_row_ids(predicate) == [1] + from pypaimon.execution.python.write.table_update import PythonTableUpdate + assert PythonTableUpdate(updater)._matched_delete_row_ids(predicate) == [1] diff --git a/paimon-python/pypaimon/tests/write/merge_execution_boundary_test.py b/paimon-python/pypaimon/tests/write/merge_execution_boundary_test.py index e4b2517ccd6f..804755b51167 100644 --- a/paimon-python/pypaimon/tests/write/merge_execution_boundary_test.py +++ b/paimon-python/pypaimon/tests/write/merge_execution_boundary_test.py @@ -18,6 +18,7 @@ """Local and distributed MERGE use the same Python action execution.""" import importlib +from unittest.mock import patch import pyarrow as pa import pytest @@ -83,3 +84,65 @@ def test_insert_actions_choose_first_match_across_arrow_chunks(): assert transform(batch).to_pylist() == [ {'value': 20}, {'value': 10}, {'value': None}, {'value': 5}, ] + + +@pytest.mark.parametrize('action', ['update', 'delete']) +@pytest.mark.parametrize('stream', [False, True]) +def test_selected_python_merge_keeps_actions_in_python(tmp_path, action, stream): + from pypaimon import CatalogFactory, Schema + from pypaimon.write.merge_types import WhenMatched, WhenNotMatched + + schema = pa.schema([('id', pa.int64()), ('value', pa.int32())]) + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + catalog.create_table('default.actions', Schema.from_pyarrow_schema(schema, options={ + 'row-tracking.enabled': 'true', 'data-evolution.enabled': 'true', + 'deletion-vectors.enabled': 'true', + 'write.native.enabled': 'false', 'commit.native.enabled': 'false', + 'scan.native-plan.enabled': 'false', 'read.native.enabled': 'false', + }), False) + table = catalog.get_table('default.actions') + builder = table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.table({'id': [1, 2], 'value': [10, 20]}, schema=schema)) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + + # Native may be unavailable for the whole MERGE request. Its chosen Python + # executor must not attempt another Native selection for individual actions. + table = table.copy({'write.native.enabled': 'true'}) + builder = table.new_stream_write_builder() if stream else table.new_batch_write_builder() + commit = builder.new_commit() + source = pa.table({'id': [1, 3], 'value': [11, 30]}, schema=schema) + matched = WhenMatched.update({'value': SourceColumnRef('value')}) if action == 'update' else WhenMatched.delete() + with patch('pypaimon.write.backend_dispatch.prepare_native_merge_into', return_value=None) as select, \ + patch('pypaimon.write.backend_dispatch.prepare_native_write_operation', + side_effect=AssertionError('Python MERGE tried to select Native update')) as select_update, \ + patch('pypaimon.execution.native.write.native_update.create_native_row_id_delete', + side_effect=AssertionError('Python MERGE tried to select Native delete')) as select_delete, \ + patch('pypaimon.execution.native.write.native_write.create_native_write', + side_effect=AssertionError('Python MERGE tried to select Native insert')) as select_insert: + try: + update = builder.new_update() + arguments = dict(on=['id'], when_matched=[matched], when_not_matched=[WhenNotMatched('*')]) + if stream: + messages = update.merge_into(source, commit_identifier=7, **arguments) + commit.commit(messages, 7) + else: + messages = update.merge_into(source, **arguments) + commit.commit(messages) + finally: + commit.close() + select.assert_called_once() + select_update.assert_not_called() + select_delete.assert_not_called() + select_insert.assert_not_called() + + read = table.new_read_builder() + actual = read.new_read().to_arrow(read.new_scan().plan().splits()).sort_by('id').to_pylist() + expected = ([{'id': 1, 'value': 11}] if action == 'update' else []) + [ + {'id': 2, 'value': 20}, {'id': 3, 'value': 30}] + assert actual == expected diff --git a/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py b/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py index 06f964ae0bd2..ce9a50a21003 100644 --- a/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py +++ b/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py @@ -27,10 +27,6 @@ from pypaimon import CatalogFactory, Schema from pypaimon.write.write_dispatch import NativePostponeFixedBucketTableWrite -from pypaimon.read.plan import Plan -from pypaimon.read.split import DataSplit -from pypaimon.read.split_serializer import deserialize_split_v1, serialize_split_v1 -from pypaimon.table.row.generic_row import GenericRow from pypaimon.write.bucket_plan import PostponeBucketPlan @@ -85,12 +81,12 @@ def find_spec(self, fullname, path=None, target=None): assert result.returncode == 0, result.stdout + result.stderr -def _table(tmp_path): +def _table(tmp_path, partition_name='p'): catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) catalog.create_database('default', True) catalog.create_table('default.t', Schema.from_pyarrow_schema( - pa.schema([('id', pa.int32()), ('p', pa.string()), ('value', pa.int64())]), - primary_keys=['id', 'p'], partition_keys=['p'], options={ + pa.schema([('id', pa.int32()), (partition_name, pa.string()), ('value', pa.int64())]), + primary_keys=['id', partition_name], partition_keys=[partition_name], options={ 'bucket': '-2', 'write.native.enabled': 'true', 'postpone.batch-write-fixed-bucket': 'true', 'postpone.target-row-num-per-bucket': '2', @@ -99,16 +95,11 @@ def _table(tmp_path): return catalog.get_table('default.t') -def _plan(table): - # Native planning returns decoded Split v1 metadata, including GenericRow - # partitions. A tuple stand-in would hide invalid partition access here. - splits = [DataSplit( - files=[], partition=GenericRow([partition], table.partition_keys_fields), - bucket=bucket, total_buckets=total, bucket_path=table.table_path, - ) for partition, bucket, total in [('existing', 1, 3), ('pending', -2, 1)]] - return Plan([deserialize_split_v1( - serialize_split_v1(split), table.partition_keys_fields, - table.trimmed_primary_keys_fields) for split in splits]) +def _native_table(table): + native = Mock() + native.new_postpone_fixed_bucket_write_builder.return_value.bucket_plan.return_value = pa.record_batch([ + pa.array(['existing']), pa.array([3], type=pa.int32())], names=['p', 'total_buckets']) + return native @pytest.fixture @@ -133,7 +124,8 @@ def test_native_ray_planning_uses_native_metadata_and_shared_input_routing( writer = NativePostponeFixedBucketTableWrite(table, 'user', None, None, core) dataset = Mock() with patch('pypaimon.execution.native.write.postpone_bucket.native_runtime_available', return_value=True), \ - patch('pypaimon.execution.native.write.postpone_bucket.native_plan', return_value=_plan(table)) as plan, \ + patch('pypaimon.execution.native.write.postpone_bucket.create_native_write_table', + return_value=_native_table(table)) as plan, \ patch('pypaimon.write.ray_datasink._collect_partition_stats', return_value=( dataset, {('existing',): (2, 0), ('new',): (9, 0), ('pending',): (1, 0)})), \ patch('pypaimon.write.ray_datasink._write_primary_key_groups') as write: @@ -149,7 +141,7 @@ def test_native_ray_planning_uses_native_metadata_and_shared_input_routing( if overwrite: plan.assert_not_called() else: - plan.assert_called_once_with(table, all_files=True) + plan.assert_called_once_with(table) assert writer._python_writer is None core.write_arrow.assert_not_called() writer.close() @@ -162,7 +154,7 @@ def test_native_ray_explicit_plan_does_not_load_metadata(tmp_path, forbid_python writer = NativePostponeFixedBucketTableWrite(table, 'user', None, explicit, Mock()) dataset = Mock() with patch('pypaimon.execution.native.write.postpone_bucket.native_runtime_available', return_value=True), \ - patch('pypaimon.execution.native.write.postpone_bucket.native_plan', + patch('pypaimon.execution.native.write.postpone_bucket.create_native_write_table', side_effect=AssertionError('explicit bucket plan must not read metadata')), \ patch('pypaimon.write.ray_datasink._collect_partition_stats', return_value=(dataset, {})), \ patch('pypaimon.write.ray_datasink._write_primary_key_groups') as write: @@ -173,14 +165,16 @@ def test_native_ray_explicit_plan_does_not_load_metadata(tmp_path, forbid_python def test_native_metadata_preparation_failure_selects_python_before_input(tmp_path): from pypaimon.write.backend_dispatch import create_postpone_bucket_planner - from pypaimon.write.postpone_bucket import PostponeBucketPlanner + from pypaimon.execution.python.write.postpone_bucket import PythonPostponeBucketPlanner table = _table(tmp_path) with patch('pypaimon.execution.native.write.postpone_bucket.native_runtime_available', return_value=True), \ - patch('pypaimon.execution.native.write.postpone_bucket.native_plan', + patch('pypaimon.execution.native.write.postpone_bucket.create_native_write_table', side_effect=RuntimeError('native metadata unavailable')), \ - patch.object(PostponeBucketPlanner, '_load_bucket_metadata', return_value={('existing',): 3}) as load, \ - patch('pypaimon.write.postpone_bucket.PostponeBucketPlanner', wraps=PostponeBucketPlanner) as pure: + patch.object(PythonPostponeBucketPlanner, '_load_bucket_metadata', + return_value={('existing',): 3}) as load, \ + patch('pypaimon.execution.python.write.postpone_bucket.PythonPostponeBucketPlanner', + wraps=PythonPostponeBucketPlanner) as pure: planner = create_postpone_bucket_planner(table) assert planner.plan({('existing',): (100, 0), ('new',): (3, 0)}).as_dict() == { ('existing',): 3, ('new',): 2} @@ -195,7 +189,8 @@ def test_native_ray_execution_failure_does_not_replay_python_planning(tmp_path, dataset = Mock() failure = TimeoutError('Ray task may have written files') with patch('pypaimon.execution.native.write.postpone_bucket.native_runtime_available', return_value=True), \ - patch('pypaimon.execution.native.write.postpone_bucket.native_plan', return_value=_plan(table)), \ + patch('pypaimon.execution.native.write.postpone_bucket.create_native_write_table', + return_value=_native_table(table)), \ patch('pypaimon.write.ray_datasink._collect_partition_stats', return_value=(dataset, {})), \ patch('pypaimon.write.ray_datasink._write_primary_key_groups', side_effect=failure) as write: with pytest.raises(TimeoutError) as raised: @@ -204,3 +199,41 @@ def test_native_ray_execution_failure_does_not_replay_python_planning(tmp_path, write.assert_called_once() assert writer._python_writer is None writer.close() + + +def test_direct_public_bucket_planner_uses_native_file_metadata(tmp_path, forbid_python_execution): + from pypaimon.write.postpone_bucket import PostponeBucketPlanner + + table = _table(tmp_path) + with patch('pypaimon.execution.native.write.postpone_bucket.native_runtime_available', return_value=True), \ + patch('pypaimon.execution.native.write.postpone_bucket.create_native_write_table', + return_value=_native_table(table)) as files: + planner = PostponeBucketPlanner(table) + assert planner.current_plan().as_dict() == {('existing',): 3} + assert planner.plan({('new',): (3, 0)}).as_dict() == {('existing',): 3, ('new',): 2} + files.assert_called_once_with(table) + + +def test_native_bucket_metadata_reports_core_conflicts(tmp_path, forbid_python_execution): + from pypaimon.execution.native.write.postpone_bucket import NativePostponeBucketPlanner + + table = _table(tmp_path) + native = _native_table(table) + failure = ValueError('Existing postpone files contain conflicting total bucket counts') + native.new_postpone_fixed_bucket_write_builder.return_value.bucket_plan.side_effect = failure + with patch('pypaimon.execution.native.write.postpone_bucket.create_native_write_table', return_value=native): + with pytest.raises(ValueError, match='conflicting total bucket counts') as raised: + NativePostponeBucketPlanner(table) + assert raised.value is failure + + +def test_native_bucket_plan_keeps_a_partition_named_total_buckets(tmp_path): + from pypaimon.execution.native.write.postpone_bucket import NativePostponeBucketPlanner + + table = _table(tmp_path, 'total_buckets') + native = _native_table(table) + native.new_postpone_fixed_bucket_write_builder.return_value.bucket_plan.return_value = pa.record_batch([ + pa.array(['west', 'east']), pa.array([3, 7], type=pa.int32())], + names=['total_buckets', 'total_buckets']) + with patch('pypaimon.execution.native.write.postpone_bucket.create_native_write_table', return_value=native): + assert NativePostponeBucketPlanner._load_bucket_metadata(table) == {('west',): 3, ('east',): 7} diff --git a/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py b/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py new file mode 100644 index 000000000000..c8fc3052a44f --- /dev/null +++ b/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pathlib import Path +import subprocess +import sys +from unittest.mock import Mock, patch + +import pyarrow as pa +import pytest + +from pypaimon.execution.python.write.row_id_file_index import RowIdFileIndex +from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId +from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId +from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER +from pypaimon.write.table_delete import TableDeleteByRowId +from pypaimon.write.table_update_by_row_id import TableUpdateByRowId + + +def test_direct_row_id_update_caches_executor_and_delegates_messages(): + table, execution = Mock(), Mock() + data = pa.table({'_ROW_ID': [0], 'value': [10]}) + messages = execution.update_columns.return_value + with patch('pypaimon.write.backend_dispatch.create_row_id_update_execution', return_value=execution) as prepare, \ + patch.object(PythonTableUpdateByRowId, 'initialize', side_effect=AssertionError('Python initialized')): + update = TableUpdateByRowId(table, 'user', BATCH_COMMIT_IDENTIFIER) + assert update.update_columns(data, ['value']) is messages + assert update.commit_messages is execution.commit_messages + update._write_row_columns(data, ['value'], {'value': [10]}) + prepare.assert_called_once_with(update, table, 'user', BATCH_COMMIT_IDENTIFIER, None) + execution.update_columns.assert_called_once_with(data, ['value']) + execution._write_row_columns.assert_called_once_with(data, ['value'], {'value': [10]}) + assert not hasattr(update, '_merge_update_with_original') + assert not hasattr(update, '_write_by_first_row_id') + + +def test_direct_row_id_execution_error_never_initializes_python(): + execution = Mock() + execution.update_columns.side_effect = RuntimeError('already staged') + with patch('pypaimon.write.backend_dispatch.create_row_id_update_execution', return_value=execution), \ + patch.object(PythonTableUpdateByRowId, 'initialize', side_effect=AssertionError('Python fallback')): + update = TableUpdateByRowId(Mock(), 'user', BATCH_COMMIT_IDENTIFIER) + with pytest.raises(RuntimeError, match='already staged'): + update.update_columns(pa.table({'_ROW_ID': [0], 'value': [10]}), ['value']) + + +def test_direct_delete_caches_executor_and_preserves_execution_error(): + execution = Mock() + with patch('pypaimon.write.backend_dispatch.create_row_id_delete_execution', return_value=execution) as prepare, \ + patch.object(PythonTableDeleteByRowId, 'initialize', side_effect=AssertionError('Python initialized')): + table = Mock() + delete = TableDeleteByRowId(table) + assert delete.delete([1, 1]) is execution.delete.return_value + execution.delete.side_effect = RuntimeError('already staged') + with pytest.raises(RuntimeError, match='already staged'): + delete.delete([2]) + prepare.assert_called_once_with(delete, table, None) + assert not hasattr(delete, '_scan_anchor_ranges') + assert not hasattr(delete, '_write_deletion_vector_index') + + +def test_selected_python_row_id_coordinator_reuses_captured_index(): + info = RowIdFileIndex(snapshot_id=7, first_row_ids=[100]) + with patch('pypaimon.write.backend_dispatch.create_row_id_update_execution', + side_effect=AssertionError('backend selected again')), \ + patch.object(RowIdFileIndex, 'from_table', side_effect=AssertionError('snapshot replanned')): + update = TableUpdateByRowId._from_python(Mock(), 'user', BATCH_COMMIT_IDENTIFIER, info) + captured = update._snapshot_files_info() + assert captured.snapshot_id == 7 + assert captured.first_row_ids == [100] + assert captured.first_row_id_index is info.first_row_id_index + assert update.commit_messages == [] + assert isinstance(update._execution, PythonTableUpdateByRowId) + + +def test_selected_python_delete_coordinator_keeps_anchor_snapshot(): + table = Mock() + anchors = [Mock()] + with patch('pypaimon.write.backend_dispatch.create_row_id_delete_execution', + side_effect=AssertionError('backend selected again')), \ + patch.object(PythonTableDeleteByRowId, '_scan_anchor_ranges', return_value=(7, anchors)) as scan: + delete = TableDeleteByRowId._from_python(table) + captured = delete._snapshot_anchor_ranges() + assert captured.snapshot_id == 7 + assert captured.anchors is anchors + scan.assert_called_once_with() + assert isinstance(delete._execution, PythonTableDeleteByRowId) + + +def test_builder_row_id_fallback_prepares_native_only_once(): + from pypaimon.write.table_update import BatchTableUpdate + + table = Mock() + index = RowIdFileIndex(snapshot_id=7, first_row_ids=[]) + with patch('pypaimon.write.update_dispatch._authorize_update'), \ + patch('pypaimon.write.update_dispatch.prepare_native_write_operation', return_value=None) as prepare, \ + patch('pypaimon.write.backend_dispatch.prepare_native_write_operation', + side_effect=AssertionError('Native prepared twice')), \ + patch.object(RowIdFileIndex, 'from_table', return_value=index): + update = BatchTableUpdate(table, 'user').new_update_by_row_id() + prepare.assert_called_once_with('update_by_row_id', table, 'user', BATCH_COMMIT_IDENTIFIER) + assert update._snapshot_files_info().snapshot_id == 7 + + +@pytest.mark.parametrize('operation', ['update', 'delete']) +def test_direct_native_row_id_api_does_not_import_python_execution(operation): + source = r''' +import importlib.abc +import sys +from types import SimpleNamespace + +class BlockPython(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.startswith('pypaimon.execution.python'): + raise AssertionError('Native imported Python execution: ' + fullname) + +sys.meta_path.insert(0, BlockPython()) +import pyarrow as pa +from pypaimon.execution.native.write import native_update +from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER +from pypaimon.write.table_delete import TableDeleteByRowId +from pypaimon.write.table_update_by_row_id import TableUpdateByRowId + +class Execution: + commit_messages = ['message'] + def update_columns(self, data, columns): + assert data['_ROW_ID'].to_pylist() == [3] and columns == ['value'] + return self.commit_messages + def delete(self, row_ids): + assert row_ids == [3] + return self.commit_messages + +native_update.create_native_update_by_row_id = lambda *args: Execution() +native_update.create_native_row_id_delete = lambda *args: Execution() +table = SimpleNamespace(options=SimpleNamespace(native_write_enabled=lambda: True)) +if sys.argv[1] == 'update': + update = TableUpdateByRowId(table, 'user', BATCH_COMMIT_IDENTIFIER) + assert update.update_columns(pa.table({'_ROW_ID': [3], 'value': [10]}), ['value']) == ['message'] +else: + assert TableDeleteByRowId(table).delete([3]) == ['message'] +assert not any(name.startswith('pypaimon.execution.python') for name in sys.modules) +''' + result = subprocess.run([sys.executable, '-c', source, operation], + cwd=str(Path(__file__).resolve().parents[3]), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/paimon-python/pypaimon/tests/write/row_kind_nullability_test.py b/paimon-python/pypaimon/tests/write/row_kind_nullability_test.py index 6d1e2848de4d..9e4bf09078c8 100644 --- a/paimon-python/pypaimon/tests/write/row_kind_nullability_test.py +++ b/paimon-python/pypaimon/tests/write/row_kind_nullability_test.py @@ -250,6 +250,7 @@ def test_native_bridge_leaves_nested_value_validation_to_core(tmp_path): def test_schema_only_writer_keeps_strict_nested_validation(): from pypaimon.write.table_write import TableWrite + from pypaimon.execution.python.write.table_write import PythonTableWrite writer = TableWrite.__new__(TableWrite) writer.file_store_write = SimpleNamespace(write_cols=None) @@ -257,11 +258,11 @@ def test_schema_only_writer_keeps_strict_nested_validation(): ('items', pa.list_(pa.field('element', pa.int32(), nullable=False)))]) data = pa.table({'items': [[None]]}, schema=pa.schema([('items', pa.list_(pa.int32()))])) with pytest.raises(ValueError, match='consistent'): - writer._prepare_arrow_data(data) + PythonTableWrite(writer)._prepare_arrow_data(data) def test_python_filters_before_nested_normalization(tmp_path): - from pypaimon.write.table_write import TableWrite + from pypaimon.execution.python.write.table_write import PythonTableWrite table, _ = _table(tmp_path) writer = table.new_batch_write_builder().new_write() @@ -283,7 +284,7 @@ def normalize(selected, expected): with patch( 'pypaimon.execution.python.write.table_write.normalize_arrow_write_nullability', side_effect=normalize): - selected = TableWrite._prepare_arrow_data(writer, data) + selected = PythonTableWrite(writer)._prepare_arrow_data(data) assert selected.to_pydict() == {'id': [2], 'op': ['+I'], 'seq': [2], 'value': [[42]]} finally: writer.close() @@ -291,6 +292,8 @@ def normalize(selected, expected): @pytest.mark.parametrize('ignored', [False, True]) def test_python_retained_nested_schema_stays_strict_without_native_extension(tmp_path, ignored): + from pypaimon.execution.python.write.table_write import PythonTableWrite + table, _ = _table(tmp_path) writer = table.new_batch_write_builder().new_write() writer.table_pyarrow_schema = pa.schema([field if field.name != 'value' else field.with_type( @@ -301,10 +304,10 @@ def test_python_retained_nested_schema_stays_strict_without_native_extension(tmp try: with patch('pypaimon.execution.native.runtime.native_runtime_available', return_value=False): if ignored: - assert writer._prepare_arrow_data(data).num_rows == 0 + assert PythonTableWrite(writer)._prepare_arrow_data(data).num_rows == 0 else: with pytest.raises(ValueError, match='consistent'): - writer._prepare_arrow_data(data) + PythonTableWrite(writer)._prepare_arrow_data(data) assert not writer.file_store_write.data_writers finally: writer.close() diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py b/paimon-python/pypaimon/tests/write/table_write_test.py index 6b7743fab09e..fffae7605dd2 100644 --- a/paimon-python/pypaimon/tests/write/table_write_test.py +++ b/paimon-python/pypaimon/tests/write/table_write_test.py @@ -37,6 +37,7 @@ from pypaimon.common.options.options import Options from pypaimon.manifest.manifest_list_manager import ManifestListManager from pypaimon.write.table_write import TableWrite +from pypaimon.execution.python.write.table_write import PythonTableWrite from pypaimon.execution.python.write.writer.append_only_data_writer import AppendOnlyDataWriter @@ -52,7 +53,7 @@ def test_large_string_compatibility_rejects_other_type_changes(self): ('id', pa.int32()), ('nested', pa.struct([pa.field('text', pa.large_string(), nullable=False)])), ]) - writer._validate_pyarrow_schema(valid) + PythonTableWrite(writer)._validate_pyarrow_schema(valid) invalid = [ valid.set(0, pa.field('id', pa.int64())), pa.schema(list(reversed(list(valid)))), @@ -64,7 +65,7 @@ def test_large_string_compatibility_rejects_other_type_changes(self): for schema in invalid: with self.subTest(schema=schema): with self.assertRaisesRegex(ValueError, 'consistent'): - writer._validate_pyarrow_schema(schema) + PythonTableWrite(writer)._validate_pyarrow_schema(schema) @parameterized.expand([('append', False, '-1'), ('primary', True, '1'), ('postpone', True, '-2')]) def test_large_string_input_roundtrip(self, mode, primary_key, bucket): @@ -235,7 +236,6 @@ def _mock_table_write(partitions, buckets, schema): is_primary_key_table=False, options=CoreOptions(Options({})), fields=Schema.from_pyarrow_schema(schema).fields) table_write.table_pyarrow_schema = schema - table_write._validate_pyarrow_schema = Mock() table_write.row_key_extractor = Mock() table_write.file_store_write = Mock(write_cols=None) table_write.row_key_extractor.extract_partition_bucket_groups.side_effect = ( @@ -313,7 +313,6 @@ def test_write_arrow_batch_noncontiguous_group_uses_take(self): is_primary_key_table=False, options=CoreOptions(Options({})), fields=Schema.from_pyarrow_schema(data.schema).fields) table_write.table_pyarrow_schema = data.schema - table_write._validate_pyarrow_schema = Mock() table_write.file_store_write = Mock(write_cols=None) table_write.row_key_extractor = Mock() table_write.row_key_extractor.extract_partition_bucket_groups.return_value = [ @@ -341,7 +340,6 @@ def test_write_arrow_batch_contiguous_group_uses_zero_copy_slice(self): is_primary_key_table=False, options=CoreOptions(Options({})), fields=Schema.from_pyarrow_schema(data.schema).fields) table_write.table_pyarrow_schema = data.schema - table_write._validate_pyarrow_schema = Mock() table_write.file_store_write = Mock(write_cols=None) table_write.row_key_extractor = Mock() table_write.row_key_extractor.extract_partition_bucket_groups.return_value = [ @@ -2002,13 +2000,13 @@ def test_validate_schema_allows_binary_family_for_write_cols(self): write_builder = table.new_batch_write_builder() table_write = write_builder.new_write() - table_write._validate_pyarrow_schema(pa.schema([ + PythonTableWrite(table_write)._validate_pyarrow_schema(pa.schema([ ('id', pa.int32()), ('payload', pa.binary(4)), ])) table_write.with_write_type(['payload']) - table_write._validate_pyarrow_schema(pa.schema([ + PythonTableWrite(table_write)._validate_pyarrow_schema(pa.schema([ ('payload', pa.binary(4)), ])) diff --git a/paimon-python/pypaimon/tests/write/update_backend_selection_test.py b/paimon-python/pypaimon/tests/write/update_backend_selection_test.py new file mode 100644 index 000000000000..0c7e5e934907 --- /dev/null +++ b/paimon-python/pypaimon/tests/write/update_backend_selection_test.py @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Public update constructors select one backend before acquiring resources.""" + +import subprocess +import sys +from unittest.mock import Mock, patch + +import pytest + +from pypaimon.write.table_update import ShardTableUpdator + + +@pytest.mark.parametrize('fails', [False, True]) +def test_direct_shard_constructor_never_loads_or_replays_python(tmp_path, fails): + source = r''' +import importlib.abc +import sys +from unittest.mock import Mock, patch + +class BlockPythonExecution(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == 'pypaimon.execution.python' or fullname.startswith('pypaimon.execution.python.'): + raise AssertionError('Native shard loaded Python execution: ' + fullname) + +sys.meta_path.insert(0, BlockPythonExecution()) +from pypaimon.write.table_update import ShardTableUpdator + +failure = OSError('A staged mutation may already have succeeded') +core = Mock() +core.prepare_commit.return_value = ['prepared'] +core.update_by_arrow_batch.side_effect = failure if sys.argv[1] == 'True' else None +with patch('pypaimon.write.backend_dispatch.prepare_native_write_operation', return_value=core) as prepare: + table = object() + shard = ShardTableUpdator(table, ['value'], ['value'], 'writer', 0, 2) + prepare.assert_called_once_with('shard_update', table, 'writer', ['value'], ['value'], 0, 2) + assert shard.arrow_reader() is core.arrow_reader.return_value + data = object() + try: + shard.update_by_arrow_batch(data) + except OSError as caught: + assert sys.argv[1] == 'True' and caught is failure + else: + assert sys.argv[1] == 'False' + assert shard.prepare_commit() == ['prepared'] + shard.close() +core.update_by_arrow_batch.assert_called_once_with(data) +core.close.assert_called_once_with() +''' + result = subprocess.run( + [sys.executable, '-c', source, str(fails)], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True, timeout=30) + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize('shard,count,columns', [ + (-1, 1, ['value']), (1, 1, ['value']), (0, 0, ['value']), (0, 1, [])]) +def test_direct_shard_constructor_validates_before_backend_preparation(shard, count, columns): + with patch('pypaimon.write.backend_dispatch.prepare_native_write_operation') as prepare: + with pytest.raises(ValueError): + ShardTableUpdator(Mock(), None, columns, 'writer', shard, count) + prepare.assert_not_called() + + +@pytest.mark.parametrize('operation', ['direct', 'grouped', 'predicate', 'upsert-arrow', 'upsert-rows', + 'delete-predicate', 'delete-id', 'shard']) +def test_python_update_keeps_selected_backend_for_nested_work(tmp_path, operation): + import pyarrow as pa + + from pypaimon import CatalogFactory, Schema + from pypaimon.table.row.generic_row import GenericRow + + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + schema = pa.schema([('id', pa.int32()), ('value', pa.int32())]) + catalog.create_table('default.data', Schema.from_pyarrow_schema(schema, options={ + 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', + 'deletion-vectors.enabled': 'true', 'write.native.enabled': 'false', + 'commit.native.enabled': 'false', 'scan.native-plan.enabled': 'false', 'read.native.enabled': 'false', + }), False) + table = catalog.get_table('default.data') + builder = table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.table({'id': [1, 2], 'value': [10, 20]}, schema=schema)) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + + table = table.copy({'write.native.enabled': 'true'}) + update = table.new_batch_write_builder().new_update().with_update_type(['value']) + data = pa.table({'id': [1], 'value': [11]}, schema=schema) + row_ids = pa.table({'_ROW_ID': [0], 'value': pa.array([11], pa.int32())}) + # The top-level preparation is unsupported. Nested target-file writing + # must use the selected Python backend without reopening Native resources. + prepare = Mock(return_value=None) + with patch('pypaimon.write.backend_dispatch.prepare_native_write_operation', prepare), \ + patch('pypaimon.write.update_dispatch.prepare_native_write_operation', prepare): + if operation == 'direct': + messages = update.update_by_arrow_with_row_id(row_ids) + elif operation == 'grouped': + messages = update.update_by_arrow_batches_with_row_id(iter([row_ids])) + elif operation == 'predicate': + messages = update.update_by_predicate(None, {'value': 11}) + elif operation == 'upsert-arrow': + messages = update.upsert_by_arrow_with_key(data, ['id']) + elif operation == 'upsert-rows': + messages = update.upsert_by_key([GenericRow([1, 11], table.fields)], ['id']) + elif operation == 'delete-predicate': + messages = update.delete_by_predicate(update.new_predicate_builder().equal('id', 1)) + elif operation == 'delete-id': + messages = update.delete_by_row_id([0]) + else: + shard = update.new_shard_updator(0, 1) + try: + reader = shard.arrow_reader() + for batch in reader: + shard.update_by_arrow_batch(pa.record_batch( + [pa.array([11] * batch.num_rows, pa.int32())], names=['value'])) + messages = shard.prepare_commit() + finally: + shard.close() + assert messages + prepare.assert_called_once() + commit = table.new_batch_write_builder().new_commit() + try: + commit.commit(messages) + finally: + commit.close() + read = table.new_read_builder() + rows = read.new_read().to_arrow(read.new_scan().plan().splits()).sort_by('id').to_pylist() + if operation.startswith('delete'): + assert rows == [{'id': 2, 'value': 20}] + else: + assert rows == [{'id': 1, 'value': 11}, + {'id': 2, 'value': 11 if operation in ('predicate', 'shard') else 20}] diff --git a/paimon-python/pypaimon/tests/write/update_execution_protocol_test.py b/paimon-python/pypaimon/tests/write/update_execution_protocol_test.py index 149aad0d17ce..27110439d22d 100644 --- a/paimon-python/pypaimon/tests/write/update_execution_protocol_test.py +++ b/paimon-python/pypaimon/tests/write/update_execution_protocol_test.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Bound update execution retains public hooks and preparation boundaries.""" +"""Bound update execution retains public methods and preparation boundaries.""" import subprocess import sys @@ -70,71 +70,56 @@ def _values(table): return read.new_read().to_arrow(read.new_scan().plan().splits()).sort_by('id').to_pydict() -def test_bound_update_uses_live_configuration_and_public_predicate_hooks(table): +def test_bound_update_uses_live_configuration_and_callable_assignments(table): built = [] - class PublicUpdate(BatchTableUpdate): - def _build_predicate_update_table(self, assignments, matched): - built.append(matched.to_pydict()) - return super()._build_predicate_update_table(assignments, matched) + def assignment(matched): + built.append(matched.to_pydict()) + return pa.compute.add(matched['value'], 1) - update = PublicUpdate(table, 'public-update') + update = BatchTableUpdate(table, 'public-update') update.with_update_type(['value']) _commit(table, update.update_by_arrow_with_row_id(pa.table({'_ROW_ID': [0], 'value': [11]}))) update.with_update_type(['id']) _commit(table, update.update_by_arrow_with_row_id(pa.table({'_ROW_ID': [1], 'id': [3]}))) predicate = update.new_predicate_builder().equal('id', 3) _commit(table, update.update_by_predicate( - predicate, {'value': lambda matched: pa.compute.add(matched['value'], 1)}, read_columns=['value'])) + predicate, {'value': assignment}, read_columns=['value'])) assert built == [{'value': [20], '_ROW_ID': [1]}] assert _values(table) == {'id': [1, 3], 'value': [11, 21]} -def test_bound_row_id_execution_retains_public_state_and_write_hook(table): - written = [] - - class PublicRowIdUpdate(TableUpdateByRowId): - def _write_by_first_row_id(self, data, column_names, blob_object_columns=None): - written.append((list(column_names), data['_ROW_ID'].to_pylist())) - return super()._write_by_first_row_id(data, column_names, blob_object_columns) - - update = PublicRowIdUpdate(table, 'row-id-update', BATCH_COMMIT_IDENTIFIER) - assert update.first_row_ids == [0] +def test_bound_row_id_execution_owns_file_routing_and_committable_state(table): + update = TableUpdateByRowId(table, 'row-id-update', BATCH_COMMIT_IDENTIFIER) messages = update.update_columns(pa.table({'_ROW_ID': [0], 'value': [33]}), ['value']) assert messages is update.commit_messages - assert written == [(['value'], [0])] + assert not hasattr(update, 'first_row_ids') + assert not hasattr(update, '_write_by_first_row_id') _commit(table, messages) assert _values(table) == {'id': [1, 2], 'value': [33, 20]} -def test_bound_delete_keeps_public_message_hook_and_committable_result(table): - deleted = [] - - class PublicDelete(TableDeleteByRowId): - def _build_commit_message(self, partition, bucket, file_positions, snapshot_id): - deleted.append(sum(len(positions) for positions in file_positions.values())) - return super()._build_commit_message(partition, bucket, file_positions, snapshot_id) - - delete = PublicDelete(table) +def test_bound_delete_owns_anchor_routing_and_committable_result(table): + delete = TableDeleteByRowId(table) messages = delete.delete([0, 0]) - assert delete.file_io is table.file_io - assert deleted == [1] + assert not hasattr(delete, '_build_commit_message') + assert not hasattr(delete, '_write_deletion_vector_index') _commit(table, messages) assert _values(table) == {'id': [2], 'value': [20]} @pytest.mark.parametrize('rows', [False, True]) -def test_prepared_python_upsert_keeps_public_partition_hooks(table, rows): +def test_prepared_python_upsert_keeps_public_input_methods(table, rows): visited = [] class PublicUpsert(TableUpsertByKey): - def _upsert_partition(self, data, keys, partition_spec, update_cols): + def upsert(self, data, keys, update_cols=None): visited.append(('arrow', data.num_rows)) - return super()._upsert_partition(data, keys, partition_spec, update_cols) + return super().upsert(data, keys, update_cols) - def _upsert_row_partition(self, row_items, keys, partition_spec, update_cols): + def upsert_rows(self, row_items, keys, update_cols=None): visited.append(('rows', len(row_items))) - return super()._upsert_row_partition(row_items, keys, partition_spec, update_cols) + return super().upsert_rows(row_items, keys, update_cols) update = PublicUpsert(table, 'upsert', BATCH_COMMIT_IDENTIFIER) data = [{'id': 1, 'value': 12}, {'id': 3, 'value': 30}] diff --git a/paimon-python/pypaimon/tests/write/writer_execution_protocol_test.py b/paimon-python/pypaimon/tests/write/writer_execution_protocol_test.py index 3cb081571a9e..a2c1fd9a1a6d 100644 --- a/paimon-python/pypaimon/tests/write/writer_execution_protocol_test.py +++ b/paimon-python/pypaimon/tests/write/writer_execution_protocol_test.py @@ -132,15 +132,20 @@ def test_stream_prepare_requires_one_signed_identifier_before_io(tmp_path, nativ writer.close() -def test_bound_python_writer_keeps_live_context_and_dynamic_batch_hook(): +def test_bound_python_writer_uses_executor_helpers_and_live_context(): batches = [] original = pa.record_batch([pa.array([1], pa.int32())], names=['id']) replacement = pa.record_batch([pa.array([2], pa.int32())], names=['id']) - owner = SimpleNamespace(_prepare_arrow_schema=lambda data: original, - write_arrow_batch=lambda batch, bucket=None: batches.append((batch, bucket))) - execution = PythonTableWrite(owner) + owner = SimpleNamespace(data=original) + + class SelectingWriter(PythonTableWrite): + def _prepare_arrow_schema(self, data): + return self.writer.data + + owner.write_arrow_batch = lambda batch, bucket=None: batches.append((batch, bucket)) + execution = SelectingWriter(owner) execution.write_arrow('first', 1) - owner._prepare_arrow_schema = lambda data: replacement + owner.data = replacement execution.write_arrow('second', 2) assert batches == [(original, 1), (replacement, 2)] assert execution.writer is owner @@ -297,3 +302,58 @@ def test_native_unpartitioned_rows_only_call_the_write_interface(tmp_path): assert core.write_arrow.call_args[0][0].to_pydict() == {'id': [2]} finally: writer.close() + + +@pytest.mark.parametrize('postpone', [False, True]) +@pytest.mark.parametrize('input_', ['table', 'batch', 'row', 'pandas']) +def test_python_executor_writes_without_private_writer_hooks(tmp_path, postpone, input_): + """Routing and prepare stay in the executor, including postpone overrides.""" + from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWrite + from pypaimon.table.row.generic_row import GenericRow + + schema = pa.schema([ + pa.field('id', pa.int32(), nullable=False), + pa.field('pt', pa.string(), nullable=False), + ('value', pa.int32()), + ]) + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + catalog.create_table('default.input', Schema.from_pyarrow_schema( + schema, primary_keys=['id', 'pt'] if postpone else [], partition_keys=['pt'], options={ + 'bucket': '-2' if postpone else '1', + 'postpone.target-row-num-per-bucket': '1', + 'write.native.enabled': 'false', 'commit.native.enabled': 'false', + 'scan.native-plan.enabled': 'false', 'read.native.enabled': 'false', + }), False) + table = catalog.get_table('default.input') + builder = table.new_batch_write_builder() + # The context exposes only public batch delegation, with no private schema + # or routing hooks from the public writer implementation. + owner = SimpleNamespace() + execution = (PythonPostponeFixedBucketWrite(owner) if postpone else PythonTableWrite(owner)) + owner.write_arrow_batch = execution.write_arrow_batch + execution.initialize(table, builder.commit_user) + commit = builder.new_commit() + rows = [{'id': 1, 'pt': 'a', 'value': 10}, {'id': 2, 'pt': 'b', 'value': 20}] + data = pa.Table.from_pylist(rows, schema=schema) + try: + if input_ == 'table': + execution.write_arrow(data) + elif input_ == 'batch': + execution.write_arrow_batch(data.to_batches()[0]) + elif input_ == 'row': + for row in rows: + execution.write_row(GenericRow([row[name] for name in schema.names], table.fields)) + else: + execution.write_pandas(data.to_pandas()) + messages = execution.prepare_commit(BATCH_COMMIT_IDENTIFIER) + assert len(messages) == 2 + assert {tuple(message.partition) for message in messages} == {('a',), ('b',)} + assert all(message.bucket == 0 for message in messages) + commit.commit(messages) + finally: + execution.close() + commit.close() + read = table.new_read_builder() + actual = read.new_read().to_arrow(read.new_scan().plan().splits()).to_pylist() + assert sorted(actual, key=lambda row: row['id']) == rows diff --git a/paimon-python/pypaimon/write/backend_dispatch.py b/paimon-python/pypaimon/write/backend_dispatch.py index df811b68462d..a1f49e7fbc89 100644 --- a/paimon-python/pypaimon/write/backend_dispatch.py +++ b/paimon-python/pypaimon/write/backend_dispatch.py @@ -104,8 +104,22 @@ def prepare(): native = try_prepare_native(prepare) if native is not None: return native - from pypaimon.write.postpone_bucket import PostponeBucketPlanner - return PostponeBucketPlanner(table, known_num_buckets=known_num_buckets) + from pypaimon.execution.python.write.postpone_bucket import PythonPostponeBucketPlanner + return PythonPostponeBucketPlanner(table, known_num_buckets=known_num_buckets) + + +def load_postpone_bucket_metadata(table): + """Select metadata planning before public bucket arithmetic consumes input.""" + if table.options.native_write_enabled(): + def prepare(): + from pypaimon.execution.native.write.postpone_bucket import create_native_postpone_bucket_planner + planner = create_native_postpone_bucket_planner(table) + return None if planner is None else planner.current_plan().as_dict() + metadata = try_prepare_native(prepare) + if metadata is not None: + return metadata + from pypaimon.execution.python.write.postpone_bucket import PythonPostponeBucketPlanner + return PythonPostponeBucketPlanner._load_bucket_metadata(table) def dynamic_bucket_data_buckets(table, partition, snapshot): @@ -164,6 +178,35 @@ def prepare_native_merge_into(*args, **kwargs): return create_native_merge_into(*args, **kwargs) +def create_row_id_update_execution(owner, table, commit_user, commit_identifier, precomputed=None): + """Select one incremental updater before capturing target files.""" + if precomputed is not None: + from pypaimon.write.update_validation import _authorize_update + _authorize_update(table) + if precomputed is None: + native = prepare_native_write_operation('update_by_row_id', table, commit_user, commit_identifier) + if native is not None: + return native + from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId + return PythonTableUpdateByRowId().initialize(table, commit_user, commit_identifier, precomputed) + + +def create_row_id_delete_execution(owner, table, precomputed=None): + """Keep precomputed broadcast metadata within its selected Python executor.""" + if precomputed is not None: + from pypaimon.write.update_validation import _authorize_update + _authorize_update(table) + if precomputed is None and table.options.native_write_enabled(): + def prepare(): + from pypaimon.execution.native.write.native_update import create_native_row_id_delete + return create_native_row_id_delete(table) + native = try_prepare_native(prepare) + if native is not None: + return native + from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId + return PythonTableDeleteByRowId().initialize(table, precomputed) + + def python_commit(owner): """Create Python commit resources only after that backend is selected.""" if owner._python_commit is None: @@ -172,6 +215,13 @@ def python_commit(owner): return owner._python_commit +def set_python_commit(owner, commit): + """Install an explicitly supplied Python committer without selecting Native.""" + from pypaimon.execution.python.write.table_commit import PythonTableCommit + owner._python_commit = (None if commit is None else + PythonTableCommit(commit, owner.overwrite_partition)) + + def commit_table_messages( owner, commit_messages, diff --git a/paimon-python/pypaimon/write/postpone_batch_table_write.py b/paimon-python/pypaimon/write/postpone_batch_table_write.py index b6a6a6f37ff9..eff9fbc058ae 100644 --- a/paimon-python/pypaimon/write/postpone_batch_table_write.py +++ b/paimon-python/pypaimon/write/postpone_batch_table_write.py @@ -61,21 +61,6 @@ def _python_execution(self): from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWrite return PythonPostponeFixedBucketWrite(self) - def _create_file_store_write(self, commit_user): - return self._python_execution()._create_file_store_write(commit_user) - - def _create_row_key_extractor(self, static_partition): - return self._python_execution()._create_row_key_extractor(static_partition) - - def _write_partition_bucket_batch(self, partition, bucket, data): - return self._python_execution()._write_partition_bucket_batch(partition, bucket, data) - - def _write_partition_bucket_row(self, partition, bucket, row, values_by_name): - return self._python_execution()._write_partition_bucket_row(partition, bucket, row, values_by_name) - - def _buffer_input(self, data): - return self._python_execution()._buffer_input(data) - def write_arrow(self, table: pa.Table, bucket=None): return self._python_execution().write_arrow(table, bucket) @@ -85,9 +70,6 @@ def write_arrow_batch(self, data: pa.RecordBatch, bucket=None): def write_row(self, row): return self._python_execution().write_row(row) - def _flush_pending_inputs(self): - return self._python_execution()._flush_pending_inputs() - def prepare_commit(self) -> List[CommitMessage]: return super().prepare_commit() diff --git a/paimon-python/pypaimon/write/postpone_bucket.py b/paimon-python/pypaimon/write/postpone_bucket.py index 28b646a4e986..6d450e7e7370 100644 --- a/paimon-python/pypaimon/write/postpone_bucket.py +++ b/paimon-python/pypaimon/write/postpone_bucket.py @@ -26,5 +26,5 @@ class PostponeBucketPlanner(PostponeBucketPlanning): @staticmethod def _load_bucket_metadata(table): - from pypaimon.execution.python.write.postpone_bucket import PythonPostponeBucketPlanner - return PythonPostponeBucketPlanner._load_bucket_metadata(table) + from pypaimon.write.backend_dispatch import load_postpone_bucket_metadata + return load_postpone_bucket_metadata(table) diff --git a/paimon-python/pypaimon/write/table_commit.py b/paimon-python/pypaimon/write/table_commit.py index 30db71d07e7f..428df6c537fd 100644 --- a/paimon-python/pypaimon/write/table_commit.py +++ b/paimon-python/pypaimon/write/table_commit.py @@ -19,7 +19,7 @@ from pypaimon.write.backend_dispatch import ( abort_table_commit, close_table_commit, commit_table_messages, prepare_native_commit, - python_commit, truncate_partitions_commit, truncate_table_commit, + python_commit, set_python_commit, truncate_partitions_commit, truncate_table_commit, ) from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER from pypaimon.write.commit_callback import CommitCallback @@ -87,10 +87,7 @@ def file_store_commit(self): @file_store_commit.setter def file_store_commit(self, commit): - from pypaimon.execution.python.write.table_commit import PythonTableCommit - - self._python_commit = (None if commit is None else - PythonTableCommit(commit, self.overwrite_partition)) + set_python_commit(self, commit) class BatchTableCommit(TableCommit): diff --git a/paimon-python/pypaimon/write/table_delete.py b/paimon-python/pypaimon/write/table_delete.py index 5bb17f98ecad..827f468a67fb 100644 --- a/paimon-python/pypaimon/write/table_delete.py +++ b/paimon-python/pypaimon/write/table_delete.py @@ -27,50 +27,23 @@ class TableDeleteByRowId: """Prepare row deletion messages for a table.""" def __init__(self, table, _precomputed_anchor_ranges: Optional['_AnchorRangesInfo'] = None): + self.table = table + from pypaimon.write.backend_dispatch import create_row_id_delete_execution + self._execution = create_row_id_delete_execution(self, table, _precomputed_anchor_ranges) + + @classmethod + def _from_python(cls, table, _precomputed_anchor_ranges=None): + """Create within an operation that has already selected Python execution.""" + owner = cls.__new__(cls) + owner.table = table from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self).initialize(table, _precomputed_anchor_ranges) + owner._execution = PythonTableDeleteByRowId().initialize(table, _precomputed_anchor_ranges) + return owner def delete(self, row_ids: Sequence[int]) -> List[CommitMessage]: - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self).delete(row_ids) + return self._execution.delete(row_ids) def _snapshot_anchor_ranges(self): - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self)._snapshot_anchor_ranges() - - def _validate_delete(self): - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self)._validate_delete() - - @staticmethod - def _deduplicate_row_ids(row_ids): - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId._deduplicate_row_ids(row_ids) - - def _load_anchor_ranges(self): - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self)._load_anchor_ranges() - - def _scan_anchor_ranges(self): - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self)._scan_anchor_ranges() - - def _group_delete_targets(self, row_ids, anchors): - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self)._group_delete_targets(row_ids, anchors) - - def _build_commit_message(self, partition, bucket, file_positions, snapshot_id): - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self)._build_commit_message(partition, bucket, file_positions, snapshot_id) - - def _read_existing_deletion_vectors(self, partition, bucket, snapshot_id): - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self)._read_existing_deletion_vectors(partition, bucket, snapshot_id) - - def _write_deletion_vector_index(self, partition, bucket, deletion_vectors): - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self)._write_deletion_vector_index(partition, bucket, deletion_vectors) - - def _index_file_path(self, index_file, partition, bucket): - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId(self)._index_file_path(index_file, partition, bucket) + # Ray's metadata coordinator selects _from_python explicitly. Native + # writers must never initialize Python's anchor planner on demand. + return self._execution._snapshot_anchor_ranges() diff --git a/paimon-python/pypaimon/write/table_update.py b/paimon-python/pypaimon/write/table_update.py index 7b7301acb7fe..7e17467a4bc4 100644 --- a/paimon-python/pypaimon/write/table_update.py +++ b/paimon-python/pypaimon/write/table_update.py @@ -31,7 +31,6 @@ from pypaimon.write.commit_message import CommitMessage from pypaimon.write.commit_identifier import validate_commit_identifier from pypaimon.write.table_update_by_row_id import TableUpdateByRowId -from pypaimon.write.update_validation import predicate_fields __all__ = ["TableUpdate", "BatchTableUpdate", "StreamTableUpdate", "ShardTableUpdator", "SingleWriter"] @@ -76,11 +75,6 @@ def _update_by_arrow_with_row_id(self, table: pa.Table, commit_identifier: int) from pypaimon.write.update_dispatch import TableUpdateOperations return TableUpdateOperations(self)._update_by_arrow_with_row_id(table, commit_identifier) - def _update_by_arrow_batches_with_row_id( - self, tables: Iterable[pa.Table], commit_identifier: int) -> List[CommitMessage]: - from pypaimon.write.update_dispatch import TableUpdateOperations - return TableUpdateOperations(self)._update_by_arrow_batches_with_row_id(tables, commit_identifier) - def _upsert_by_arrow_with_key( self, table: pa.Table, upsert_keys: List[str], commit_identifier: int) -> List[CommitMessage]: """Shared implementation for ``upsert_by_arrow_with_key``. @@ -134,73 +128,10 @@ def _delete_by_row_id(self, row_ids: Sequence[int], commit_identifier: int) -> L from pypaimon.write.update_dispatch import TableUpdateOperations return TableUpdateOperations(self)._delete_by_row_id(row_ids, commit_identifier) - def _matched_delete_row_ids(self, predicate: Optional[Predicate]) -> List[int]: - from pypaimon.write.update_dispatch import TableUpdateOperations - return TableUpdateOperations(self)._matched_delete_row_ids(predicate) - - _predicate_fields = staticmethod(predicate_fields) - - @staticmethod - def _predicate_update_file_groups(splits): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate._predicate_update_file_groups(splits) - - def _matched_update_scan_table(self): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(self)._matched_update_scan_table() - def _validate_predicate_update(self, assignments, read_columns, has_callable, has_array): from pypaimon.write.update_validation import _validate_predicate_update return _validate_predicate_update(self, assignments, read_columns, has_callable, has_array) - def _build_predicate_update_table(self, assignments, matched): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(self)._build_predicate_update_table(assignments, matched) - - @staticmethod - def _assignment_to_array(value, data_type, row_count): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate._assignment_to_array(value, data_type, row_count) - - def _delete_by_partition_filter(self, partition_filter): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(self)._delete_by_partition_filter(partition_filter) - - @staticmethod - def _partition_delete_message(messages, partition, bucket, snapshot_id): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate._partition_delete_message(messages, partition, bucket, snapshot_id) - - def _partition_index_entries(self, snapshot, partition_filter): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(self)._partition_index_entries(snapshot, partition_filter) - - def _partition_only_delete_filter(self, predicate): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(self)._partition_only_delete_filter(predicate) - - def _rewrite_predicate_to_partition_indices(self, predicate, partition_index): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(self)._rewrite_predicate_to_partition_indices(predicate, partition_index) - - def _execute_predicate_update(self, predicate, assignments, commit_identifier, - read_columns, has_callable, has_array): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(self)._execute_predicate_update( - predicate, assignments, commit_identifier, read_columns, has_callable, has_array) - - def _execute_predicate_delete(self, predicate, commit_identifier): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(self)._execute_predicate_delete(predicate, commit_identifier) - - def _new_python_shard_updator(self, shard_num, total_shard_count): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(self)._new_python_shard_updator(shard_num, total_shard_count) - - def _delete_rows_with_python(self, row_ids): - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(self)._delete_rows_with_python(row_ids) - class BatchTableUpdate(TableUpdate): """Batch-mode table update; commit messages always use @@ -220,8 +151,8 @@ def update_by_arrow_batches_with_row_id(self, tables: Iterable[pa.Table]) -> Lis """Apply row-id updates from batches using one target-file index. For each updated column, batches must target disjoint ``first_row_id`` - file groups. Conflicting overlap is rejected and all files staged by - earlier batches are aborted. + file groups. Conflicting overlap is rejected. Files staged by earlier + batches are retained if a later batch fails. """ from pypaimon.write.update_dispatch import TableUpdateOperations return TableUpdateOperations(self).update_by_arrow_batches_with_row_id(tables) @@ -391,57 +322,63 @@ def merge_into( class ShardTableUpdator: + """Read and rewrite one shard using one selected execution backend.""" def __init__(self, table, projection: Optional[List[str]], write_cols: List[str], commit_user, shard_num: int, total_shard_count: int): + if total_shard_count <= 0 or shard_num < 0 or shard_num >= total_shard_count: + raise ValueError('Shard index must be smaller than a positive shard count') + if write_cols == []: + raise ValueError('Shard update columns must not be empty') + from pypaimon.write.backend_dispatch import prepare_native_write_operation + self._execution = prepare_native_write_operation( + 'shard_update', table, commit_user, projection, write_cols, shard_num, total_shard_count) + if self._execution is None: + self._initialize_python(table, projection, write_cols, commit_user, shard_num, total_shard_count) + + @classmethod + def _from_python(cls, table, projection, write_cols, commit_user, shard_num, total_shard_count): + instance = cls.__new__(cls) + instance._initialize_python(table, projection, write_cols, commit_user, shard_num, total_shard_count) + return instance + + def _initialize_python(self, table, projection, write_cols, commit_user, shard_num, total_shard_count): from pypaimon.execution.python.write.table_update import ShardTableUpdator - return ShardTableUpdator(self).initialize( - table, projection, write_cols, commit_user, shard_num, total_shard_count) + self._execution = ShardTableUpdator(self) + self._execution.initialize(table, projection, write_cols, commit_user, shard_num, total_shard_count) @staticmethod def compute_from_files(files: List[DataFileMeta]) -> List[Range]: - from pypaimon.execution.python.write.table_update import ShardTableUpdator - return ShardTableUpdator.compute_from_files(files) + return Range.sort_and_merge_overlap([file.row_id_range() for file in files], True, False) def arrow_reader(self) -> pyarrow.ipc.RecordBatchReader: - from pypaimon.execution.python.write.table_update import ShardTableUpdator - return ShardTableUpdator(self).arrow_reader() + return self._execution.arrow_reader() def prepare_commit(self) -> List[CommitMessage]: - from pypaimon.execution.python.write.table_update import ShardTableUpdator - return ShardTableUpdator(self).prepare_commit() + return self._execution.prepare_commit() def close(self): - from pypaimon.execution.python.write.table_update import ShardTableUpdator - return ShardTableUpdator(self).close() + return self._execution.close() def update_by_arrow_batch(self, data: pa.RecordBatch): - from pypaimon.execution.python.write.table_update import ShardTableUpdator - return ShardTableUpdator(self).update_by_arrow_batch(data) - - def _init_writer(self): - from pypaimon.execution.python.write.table_update import ShardTableUpdator - return ShardTableUpdator(self)._init_writer() + return self._execution.update_by_arrow_batch(data) class SingleWriter: def __init__(self, writer: 'DataWriter', partition, first_row_id: int, row_count: int): from pypaimon.execution.python.write.table_update import SingleWriter - return SingleWriter(self).initialize(writer, partition, first_row_id, row_count) + self._execution = SingleWriter(self) + self._execution.initialize(writer, partition, first_row_id, row_count) def capacity(self) -> int: - from pypaimon.execution.python.write.table_update import SingleWriter - return SingleWriter(self).capacity() + return self._execution.capacity() def write(self, data: pa.RecordBatch): - from pypaimon.execution.python.write.table_update import SingleWriter - return SingleWriter(self).write(data) + return self._execution.write(data) def partition(self) -> Tuple: - from pypaimon.execution.python.write.table_update import SingleWriter - return SingleWriter(self).partition() + return self._execution.partition() def end(self) -> DataFileMeta: - from pypaimon.execution.python.write.table_update import SingleWriter - return SingleWriter(self).end() + return self._execution.end() diff --git a/paimon-python/pypaimon/write/table_update_by_row_id.py b/paimon-python/pypaimon/write/table_update_by_row_id.py index cd51a098d303..8095aff1fa0f 100644 --- a/paimon-python/pypaimon/write/table_update_by_row_id.py +++ b/paimon-python/pypaimon/write/table_update_by_row_id.py @@ -35,117 +35,38 @@ class TableUpdateByRowId(RowIdUpdateInputMixin, metaclass=ABCMeta): def __init__(self, table, commit_user: str, commit_identifier: int, _precomputed_files_info: Optional['RowIdFileIndex'] = None): - commit_identifier = validate_commit_identifier(commit_identifier) - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self).initialize(table, commit_user, commit_identifier, _precomputed_files_info) - - def update_columns(self, data: pa.Table, column_names: List[str]) -> List[CommitMessage]: - """ - Add or update columns in the table. - - Args: - data: Input data containing row_id and columns to update - column_names: Names of columns to update (excluding row_id) - - Returns: - List of commit messages - """ - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self).update_columns(data, column_names) - - def _snapshot_files_info(self): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._snapshot_files_info() - - def _load_existing_files_info(self): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._load_existing_files_info() - - def _write_row_columns(self, update_data, column_names, blob_object_columns): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._write_row_columns(update_data, column_names, blob_object_columns) - - def _calculate_first_row_id(self, data): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._calculate_first_row_id(data) - - def _write_by_first_row_id(self, data, column_names, blob_object_columns=None): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._write_by_first_row_id(data, column_names, blob_object_columns) - - def _read_original_file_data(self, first_row_id, column_names): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._read_original_file_data(first_row_id, column_names) - - def _original_file_read(self, first_row_id, column_names): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._original_file_read(first_row_id, column_names) - - def _merged_batches(self, first_row_id, data, column_names): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._merged_batches(first_row_id, data, column_names) - - def _write_group_streaming(self, partition, first_row_id, data, column_names): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._write_group_streaming(partition, first_row_id, data, column_names) - - def _merge_update_with_original(self, original_data, update_data, column_names, - first_row_id, blob_object_columns=None, blob_columns_with_baseline=None): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._merge_update_with_original( - original_data, update_data, column_names, first_row_id, - blob_object_columns, blob_columns_with_baseline) - - def _normalize_write_data(self, data): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._normalize_write_data(data) - - def _check_write_not_null(self, column_name, has_null): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._check_write_not_null(column_name, has_null) + self.table = table + self.commit_user = commit_user + self.commit_identifier = validate_commit_identifier(commit_identifier) + from pypaimon.write.backend_dispatch import create_row_id_update_execution + self._execution = create_row_id_update_execution( + self, table, commit_user, self.commit_identifier, _precomputed_files_info) @classmethod - def _merge_chunked_column(cls, original_col, update_col, sorted_updates): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId._merge_chunked_column.__func__(cls, original_col, update_col, sorted_updates) - - @classmethod - def _merge_chunk_with_updates(cls, original, update_col, update_chunk_offsets, updates): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId._merge_chunk_with_updates.__func__( - cls, original, update_col, update_chunk_offsets, updates) - - @staticmethod - def _chunk_offsets(column): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId._chunk_offsets(column) + def _from_python(cls, table, commit_user, commit_identifier, _precomputed_files_info=None): + """Create within an operation that has already selected Python execution.""" + owner = cls.__new__(cls) + owner.table = table + owner.commit_user = commit_user + owner.commit_identifier = validate_commit_identifier(commit_identifier) + from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId + owner._execution = PythonTableUpdateByRowId().initialize( + table, commit_user, owner.commit_identifier, _precomputed_files_info) + return owner + + @property + def commit_messages(self): + """Messages accumulated by the selected row-ID execution backend.""" + return self._execution.commit_messages - @staticmethod - def _take_from_chunked_array(column, chunk_offsets, indices): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId._take_from_chunked_array(column, chunk_offsets, indices) - - @staticmethod - def _is_offset_overflow(error): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId._is_offset_overflow(error) - - @staticmethod - def _coerce_column(col, target_type): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId._coerce_column(col, target_type) - - def _blob_placeholder(self, column_name): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._blob_placeholder(column_name) + def update_columns(self, data: pa.Table, column_names: List[str]) -> List[CommitMessage]: + """Stage column updates from an Arrow table containing ``_ROW_ID``.""" + return self._execution.update_columns(data, column_names) - def _write_group(self, partition, first_row_id, data, column_names, blob_object_columns=None): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId(self)._write_group( - partition, first_row_id, data, column_names, blob_object_columns) + def _write_row_columns(self, update_data, column_names, blob_object_columns): + return self._execution._write_row_columns(update_data, column_names, blob_object_columns) - @staticmethod - def _assign_update_file_metadata(new_files, first_row_id, column_names, blob_columns): - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId._assign_update_file_metadata( - new_files, first_row_id, column_names, blob_columns) + def _snapshot_files_info(self): + # Internal Python metadata coordinators select _from_python before + # requesting the broadcast index. Never initialize a second backend. + return self._execution._snapshot_files_info() diff --git a/paimon-python/pypaimon/write/table_upsert_by_key.py b/paimon-python/pypaimon/write/table_upsert_by_key.py index 7c1c5ccb453e..1725230e4755 100644 --- a/paimon-python/pypaimon/write/table_upsert_by_key.py +++ b/paimon-python/pypaimon/write/table_upsert_by_key.py @@ -72,41 +72,11 @@ def _create_native_upsert(self, data, upsert_keys, columns): from pypaimon.write.update_dispatch import UpsertOperations return UpsertOperations(self)._create_native_upsert(data, upsert_keys, columns) - def _new_row_id_updater(self): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._new_row_id_updater() - @staticmethod def _normalize_rows(rows): from pypaimon.write.update_validation import _normalize_rows return _normalize_rows(rows) - def _group_rows_by_partition(self, row_items): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._group_rows_by_partition(row_items) - - def _upsert_row_partition(self, row_items, upsert_keys, partition_spec, update_cols): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._upsert_row_partition(row_items, upsert_keys, partition_spec, update_cols) - - @staticmethod - def _dedup_row_items_last_write_wins(row_items, input_key_tuples, partition_spec): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert._dedup_row_items_last_write_wins(row_items, input_key_tuples, partition_spec) - - def _group_by_partition(self, data): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._group_by_partition(data) - - def _upsert_partition(self, partition_data, upsert_keys, partition_spec, update_cols): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._upsert_partition(partition_data, upsert_keys, partition_spec, update_cols) - - @staticmethod - def _dedup_last_write_wins(partition_data, input_key_tuples, partition_spec): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert._dedup_last_write_wins(partition_data, input_key_tuples, partition_spec) - def _validate_inputs(self, data, upsert_keys, update_cols): from pypaimon.write.update_validation import _validate_inputs return _validate_inputs(self, data, upsert_keys, update_cols) @@ -114,31 +84,3 @@ def _validate_inputs(self, data, upsert_keys, update_cols): def _validate_row_inputs(self, values_by_name, upsert_keys, update_cols): from pypaimon.write.update_validation import _validate_row_inputs return _validate_row_inputs(self, values_by_name, upsert_keys, update_cols) - - def _build_key_to_row_ids_map(self, match_keys, partition_spec, input_key_set): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._build_key_to_row_ids_map(match_keys, partition_spec, input_key_set) - - def _do_updates(self, data, matched_indices, input_key_tuples, key_to_row_ids, update_cols): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._do_updates(data, matched_indices, input_key_tuples, key_to_row_ids, update_cols) - - def _do_appends(self, data, new_indices): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._do_appends(data, new_indices) - - def _append_rows(self, row_items): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._append_rows(row_items) - - def _append_row_column_names(self, row_items): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._append_row_column_names(row_items) - - def _upsert_arrow(self, data, upsert_keys, effective_update_cols): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._upsert_arrow(data, upsert_keys, effective_update_cols) - - def _upsert_rows(self, row_items, upsert_keys, effective_update_cols): - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - return PythonTableUpsert(self)._upsert_rows(row_items, upsert_keys, effective_update_cols) diff --git a/paimon-python/pypaimon/write/table_write.py b/paimon-python/pypaimon/write/table_write.py index 17c63fb6ab62..34eb19e36e1e 100644 --- a/paimon-python/pypaimon/write/table_write.py +++ b/paimon-python/pypaimon/write/table_write.py @@ -108,53 +108,12 @@ def close(self): def abort(self): return self._python_execution().abort() - def _create_file_store_write(self, commit_user): - return self._python_execution()._create_file_store_write(commit_user) - - def _create_row_key_extractor(self, static_partition): - return self._python_execution()._create_row_key_extractor(static_partition) - - def _write_partition_bucket_batch(self, partition, bucket, data): - return self._python_execution()._write_partition_bucket_batch(partition, bucket, data) - - def _write_precomputed_bucket(self, data, bucket): - return self._python_execution()._write_precomputed_bucket(data, bucket) - - def _write_partition_bucket_row(self, partition, bucket, row, values_by_name): - return self._python_execution()._write_partition_bucket_row(partition, bucket, row, values_by_name) - def _distributed_write_options(self, *, overwrite=False): return {} def _prepare_commit(self, commit_identifier): return self._python_execution().prepare_commit(commit_identifier) - def _release_prepared_indexes(self): - return self._python_execution()._release_prepared_indexes() - - def _prepare_arrow_data(self, data): - return self._python_execution()._prepare_arrow_data(data) - - def _prepare_arrow_schema(self, data): - return self._python_execution()._prepare_arrow_schema(data) - - def _normalize_arrow_write_schema(self, data): - return self._python_execution()._normalize_arrow_write_schema(data) - - def _validate_pyarrow_schema(self, data_schema, *, check_nested_nullability=True): - return self._python_execution()._validate_pyarrow_schema( - data_schema, check_nested_nullability=check_nested_nullability) - - def _is_compatible_pyarrow_schema(self, data_schema, expected_schema, *, check_nested_nullability=True): - return self._python_execution()._is_compatible_pyarrow_schema( - data_schema, expected_schema, check_nested_nullability=check_nested_nullability) - - def _write_cols_pyarrow_schema(self, write_cols): - return self._python_execution()._write_cols_pyarrow_schema(write_cols) - - def _raise_inconsistent_schema(self, data_schema): - return self._python_execution()._raise_inconsistent_schema(data_schema) - class BatchTableWrite(TableWrite): def __init__(self, table, commit_user, static_partition: Optional[dict] = None, diff --git a/paimon-python/pypaimon/write/update_dispatch.py b/paimon-python/pypaimon/write/update_dispatch.py index 38da462c6571..b1c861521494 100644 --- a/paimon-python/pypaimon/write/update_dispatch.py +++ b/paimon-python/pypaimon/write/update_dispatch.py @@ -45,8 +45,8 @@ def _new_row_id_updater(self, commit_identifier: int, _precomputed_files_info=No 'update_by_row_id', context.table, context.commit_user, commit_identifier) if native is not None: return native - return TableUpdateByRowId(context.table, context.commit_user, commit_identifier, - _precomputed_files_info=_precomputed_files_info) + return TableUpdateByRowId._from_python(context.table, context.commit_user, commit_identifier, + _precomputed_files_info=_precomputed_files_info) def new_shard_updator(self, shard_num: int, total_shard_count: int): context = self._context @@ -69,7 +69,8 @@ def new_shard_updator(self, shard_num: int, total_shard_count: int): StreamTableUpdate)) if native is not None: return native - return context._new_python_shard_updator(shard_num, total_shard_count) + from pypaimon.execution.python.write.table_update import PythonTableUpdate + return PythonTableUpdate(context)._new_python_shard_updator(shard_num, total_shard_count) def _update_by_arrow_with_row_id(self, table: pa.Table, commit_identifier: int) -> List[CommitMessage]: context = self._context From 0f5f53553330a24cad91d79382d27fc29d906569 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 22 Oct 2026 20:07:23 +0800 Subject: [PATCH 2/6] [Python] Expose shared commit identity configuration --- docs/docs/pypaimon/writing.md | 16 +++ .../execution/native/write/native_commit.py | 2 +- .../native/write/native_merge_into.py | 2 +- .../execution/native/write/native_update.py | 20 ++-- .../execution/native/write/native_write.py | 4 +- .../tests/merge_execution_boundary_test.py | 2 +- .../tests/native_branch_write_test.py | 2 +- .../native_merge_source_metadata_test.py | 2 +- .../native_write_inactive_options_test.py | 2 +- .../tests/write/execution_lifecycle_test.py | 2 +- .../write/write_builder_identity_test.py | 107 ++++++++++++++++++ paimon-python/pypaimon/write/write_builder.py | 14 ++- 12 files changed, 152 insertions(+), 23 deletions(-) create mode 100644 paimon-python/pypaimon/tests/write/write_builder_identity_test.py diff --git a/docs/docs/pypaimon/writing.md b/docs/docs/pypaimon/writing.md index 8deec81f9bcb..99d77a1f2e41 100644 --- a/docs/docs/pypaimon/writing.md +++ b/docs/docs/pypaimon/writing.md @@ -72,6 +72,22 @@ Use `new_postpone_fixed_bucket_write_builder()` to write real buckets. New partitions are buffered until `prepare_commit()`; existing ones are incremental. This builder currently supports `bucket-function.type=default` only. +### Commit Identity + +Write builders generate a commit user by default. Use `with_commit_user()` before +creating writers, updates or committers to share an application identity across +distributed tasks or restore a stream application: + +```python +write_builder = table.new_stream_write_builder().with_commit_user('application-1') +table_write = write_builder.new_write() +table_commit = write_builder.new_commit() +``` + +Writers and their committer must use the same identity. Use different identities +for independent applications. Batch and postpone fixed-bucket builders expose +the same method. Changing a builder affects only operations created afterward. + ### Native Write and Commit With the `pypaimon-rust` extension installed, set `write.native.enabled=true` diff --git a/paimon-python/pypaimon/execution/native/write/native_commit.py b/paimon-python/pypaimon/execution/native/write/native_commit.py index 75b6a1a5a8bc..5e4e77d52937 100644 --- a/paimon-python/pypaimon/execution/native/write/native_commit.py +++ b/paimon-python/pypaimon/execution/native/write/native_commit.py @@ -56,7 +56,7 @@ def create_native_commit(table, commit_user, overwrite_partition=None, *, batch= if native_table is None: return None if batch or overwrite_partition is not None: - builder = native_table.new_batch_write_builder()._with_commit_user(commit_user) + builder = native_table.new_batch_write_builder().with_commit_user(commit_user) if overwrite_partition is not None: builder = builder.with_overwrite(overwrite_partition) return builder.new_commit() diff --git a/paimon-python/pypaimon/execution/native/write/native_merge_into.py b/paimon-python/pypaimon/execution/native/write/native_merge_into.py index 41eca96baf11..cd92aca1e3c7 100644 --- a/paimon-python/pypaimon/execution/native/write/native_merge_into.py +++ b/paimon-python/pypaimon/execution/native/write/native_merge_into.py @@ -120,7 +120,7 @@ def encode(clause): encoded_not_matched = [encode(clause) for clause in not_matched] if any(clause is None for clause in encoded_matched + encoded_not_matched): return None - builder = (native_table.new_batch_write_builder()._with_commit_user(commit_user) + builder = (native_table.new_batch_write_builder().with_commit_user(commit_user) if commit_identifier == BATCH_COMMIT_IDENTIFIER else native_table.new_stream_write_builder().with_commit_user(commit_user)) writer = builder.new_update() diff --git a/paimon-python/pypaimon/execution/native/write/native_update.py b/paimon-python/pypaimon/execution/native/write/native_update.py index 60b600230d64..f7e58eab7e46 100644 --- a/paimon-python/pypaimon/execution/native/write/native_update.py +++ b/paimon-python/pypaimon/execution/native/write/native_update.py @@ -77,7 +77,7 @@ def create_native_update(table, commit_user, columns): if native_table is None: return None writer = (native_table.new_batch_write_builder() - ._with_commit_user(commit_user) + .with_commit_user(commit_user) .new_update()) if columns is not None: writer.with_update_type(columns) @@ -94,7 +94,7 @@ def create_native_update_by_row_id(table, commit_user, commit_identifier): if native_table is None: return None if commit_identifier == BATCH_COMMIT_IDENTIFIER: - writer = (native_table.new_batch_write_builder()._with_commit_user(commit_user) + writer = (native_table.new_batch_write_builder().with_commit_user(commit_user) .new_update().new_update_by_row_id()) else: writer = (native_table.new_stream_write_builder().with_commit_user(commit_user) @@ -111,7 +111,7 @@ def create_native_shard_updator(table, commit_user, projection, columns, shard, if native_table is None: return None builder = (native_table.new_stream_write_builder().with_commit_user(commit_user) if stream else - native_table.new_batch_write_builder()._with_commit_user(commit_user)) + native_table.new_batch_write_builder().with_commit_user(commit_user)) update = builder.new_update() read_builder = table.new_read_builder() if projection is not None: @@ -379,11 +379,11 @@ def create_native_upsert(table, commit_user, data, keys, columns): for name in key_names): return None writer = (native_table.new_batch_write_builder() - ._with_commit_user(commit_user) + .with_commit_user(commit_user) .new_update() .with_update_type(columns)) if blob_rows is not None: - writer._with_blob_uri_reader_factory(blob_rows) + writer.with_blob_uri_reader_factory(blob_rows) return NativeTableUpsert(table, writer, keys, data, blob_rows) @@ -401,7 +401,7 @@ def create_native_predicate_update(table, commit_user, predicate, columns=None): # reading and callback execution belong to the core operation below. native_table.new_read_builder().with_filter(native_predicate) writer = (native_table.new_batch_write_builder() - ._with_commit_user(commit_user).new_update()) + .with_commit_user(commit_user).new_update()) return NativePredicateTableUpdate(table, writer, native_predicate) @@ -434,7 +434,7 @@ def canonical_paths(node): # Resolve all fields/operators before selecting the mutating operation. native_table.new_read_builder().with_filter(native_predicate) writer = (native_table.new_batch_write_builder() - ._with_commit_user(commit_user).new_update()) + .with_commit_user(commit_user).new_update()) return NativePredicateTableUpdate(table, writer, native_predicate) @@ -446,7 +446,7 @@ def create_native_delete(table, commit_user): if native_table is None: return None writer = (native_table.new_batch_write_builder() - ._with_commit_user(commit_user) + .with_commit_user(commit_user) .new_update()) return NativeBatchTableUpdate(table, writer) @@ -548,11 +548,11 @@ def _write_row_columns(self, data, column_names, blob_object_columns): arrow_field = pa.field(name, pa.list_(entries), nullable=arrow_field.nullable) data = data.append_column(arrow_field, pa.array( [blobs.encode(value, field.type) for value in values], type=arrow_field.type)) - self.writer._with_blob_uri_reader_factory(blobs) + self.writer.with_blob_uri_reader_factory(blobs) try: return self.update_columns(data, column_names) finally: - self.writer._with_blob_uri_reader_factory(None) + self.writer.with_blob_uri_reader_factory(None) class NativeTableUpsert: diff --git a/paimon-python/pypaimon/execution/native/write/native_write.py b/paimon-python/pypaimon/execution/native/write/native_write.py index c88f9c79806e..9f29670158f8 100644 --- a/paimon-python/pypaimon/execution/native/write/native_write.py +++ b/paimon-python/pypaimon/execution/native/write/native_write.py @@ -55,7 +55,7 @@ def create_native_write(table, commit_user, static_partition=None, stream=False, if native_table is None: return None if fixed_bucket: - builder = native_table.new_postpone_fixed_bucket_write_builder()._with_commit_user(commit_user) + builder = native_table.new_postpone_fixed_bucket_write_builder().with_commit_user(commit_user) if bucket_plan is not None: builder = builder.with_bucket_plan(bucket_plan.to_arrow(table)) if static_partition is not None: @@ -65,7 +65,7 @@ def create_native_write(table, commit_user, static_partition=None, stream=False, if stream: builder = native_table.new_stream_write_builder().with_commit_user(commit_user) else: - builder = native_table.new_batch_write_builder()._with_commit_user(commit_user) + builder = native_table.new_batch_write_builder().with_commit_user(commit_user) if static_partition is not None: builder = builder.with_overwrite(static_partition) if restore_snapshot_id is not None: diff --git a/paimon-python/pypaimon/tests/merge_execution_boundary_test.py b/paimon-python/pypaimon/tests/merge_execution_boundary_test.py index a3140f97d2c3..92a524e3ab36 100644 --- a/paimon-python/pypaimon/tests/merge_execution_boundary_test.py +++ b/paimon-python/pypaimon/tests/merge_execution_boundary_test.py @@ -79,7 +79,7 @@ def find_spec(self, fullname, path=None, target=None): else: core = Mock() builder = core.new_batch_write_builder.return_value - builder._with_commit_user.return_value = builder + builder.with_commit_user.return_value = builder writer = builder.new_update.return_value writer.merge_into.return_value = [] def assignment(batch): diff --git a/paimon-python/pypaimon/tests/native_branch_write_test.py b/paimon-python/pypaimon/tests/native_branch_write_test.py index eb71cc56e318..6e10f6d576e6 100644 --- a/paimon-python/pypaimon/tests/native_branch_write_test.py +++ b/paimon-python/pypaimon/tests/native_branch_write_test.py @@ -189,7 +189,7 @@ def test_native_branch_stream_checkpoint_histories(tmp_path, native_rest_catalog _write(main, [{'id': 1, 'value': 10, 'p': 'a'}]) branch = _branch(catalog, main) for target, value in ((main, 100), (branch, 20)): - builder = target.new_stream_write_builder()._with_commit_user('same-writer') + builder = target.new_stream_write_builder().with_commit_user('same-writer') messages = _prepare(builder, [{'id': 1, 'value': value, 'p': 'a'}], checkpoint=7) _commit(builder, messages, checkpoint=7) snapshot = target.snapshot_manager().get_latest_snapshot() diff --git a/paimon-python/pypaimon/tests/native_merge_source_metadata_test.py b/paimon-python/pypaimon/tests/native_merge_source_metadata_test.py index fc15dfa8349d..9a5251026db1 100644 --- a/paimon-python/pypaimon/tests/native_merge_source_metadata_test.py +++ b/paimon-python/pypaimon/tests/native_merge_source_metadata_test.py @@ -134,7 +134,7 @@ def test_native_condition_transport_does_not_load_python_datafusion(native_rest_ pa.table({'key': [0], 'value': [91]}, schema=_SCHEMA)) source_key = '_ROW_ID' if table_source else 'key' native_table, native_source = Mock(), Mock() - core = native_table.new_batch_write_builder.return_value._with_commit_user.return_value.new_update.return_value + core = native_table.new_batch_write_builder.return_value.with_commit_user.return_value.new_update.return_value core.merge_into.return_value = [] with patch('pypaimon.execution.native.write.native_merge_into._native_row_id_table', return_value=native_table), \ patch( diff --git a/paimon-python/pypaimon/tests/native_write_inactive_options_test.py b/paimon-python/pypaimon/tests/native_write_inactive_options_test.py index d2a2fcb6532e..16d763da340a 100644 --- a/paimon-python/pypaimon/tests/native_write_inactive_options_test.py +++ b/paimon-python/pypaimon/tests/native_write_inactive_options_test.py @@ -65,7 +65,7 @@ def test_inactive_changelog_format_qualifies_for_native_write( assert isinstance(writer, NativeWrite) create.assert_called_once_with(table) builder = (native_table.new_stream_write_builder.return_value.with_commit_user.return_value if stream - else native_table.new_batch_write_builder.return_value._with_commit_user.return_value) + else native_table.new_batch_write_builder.return_value.with_commit_user.return_value) if partition is not None: builder.with_overwrite.assert_called_once_with(partition) builder = builder.with_overwrite.return_value diff --git a/paimon-python/pypaimon/tests/write/execution_lifecycle_test.py b/paimon-python/pypaimon/tests/write/execution_lifecycle_test.py index dedd3d6b6b5c..1c3938cfb580 100644 --- a/paimon-python/pypaimon/tests/write/execution_lifecycle_test.py +++ b/paimon-python/pypaimon/tests/write/execution_lifecycle_test.py @@ -187,7 +187,7 @@ def _data_evolution_table(tmp_path): def _core_builder(): table = Mock() builder = table.new_batch_write_builder.return_value - builder._with_commit_user.return_value = builder + builder.with_commit_user.return_value = builder return table, builder diff --git a/paimon-python/pypaimon/tests/write/write_builder_identity_test.py b/paimon-python/pypaimon/tests/write/write_builder_identity_test.py new file mode 100644 index 000000000000..cba427d46dee --- /dev/null +++ b/paimon-python/pypaimon/tests/write/write_builder_identity_test.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Public commit identities survive builder reuse and both write modes.""" + +import pyarrow as pa +import pytest + +from pypaimon import CatalogFactory, Schema +from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER + +pytestmark = [pytest.mark.python_plan, pytest.mark.python_read, + pytest.mark.python_write, pytest.mark.python_commit] + + +def _table(tmp_path, postpone=False): + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + options = { + 'bucket': '-2' if postpone else '-1', 'postpone.default-bucket-num': '2', + 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', + 'write.native.enabled': 'false', 'commit.native.enabled': 'false', + 'read.native.enabled': 'false', 'scan.native-plan.enabled': 'false', + } + if postpone: + options.pop('row-tracking.enabled') + options.pop('data-evolution.enabled') + schema = pa.schema([('id', pa.int32()), ('value', pa.int32())]) + catalog.create_table('default.data', Schema.from_pyarrow_schema( + schema, primary_keys=['id'] if postpone else [], options=options), False) + return catalog.get_table('default.data'), schema + + +@pytest.mark.parametrize('stream', [False, True]) +def test_public_builder_commit_user_is_captured_by_write_update_and_commit(tmp_path, stream): + table, schema = _table(tmp_path) + create_builder = table.new_stream_write_builder if stream else table.new_batch_write_builder + builder = create_builder() + assert builder.with_commit_user('application-1') is builder + writer, commit, update = builder.new_write(), builder.new_commit(), builder.new_update() + assert builder.with_commit_user('application-2') is builder + # Reconfiguring a reusable builder does not change existing operations. + assert writer.commit_user == commit.commit_user == update.commit_user == 'application-1' + assert builder.new_update().commit_user == 'application-2' + next_commit = builder.new_commit() + try: + assert next_commit.commit_user == 'application-2' + finally: + next_commit.close() + + try: + writer.write_arrow(pa.table({'id': [1], 'value': [10]}, schema=schema)) + messages = writer.prepare_commit(7) if stream else writer.prepare_commit() + commit.commit(messages, 7) if stream else commit.commit(messages) + finally: + writer.close() + commit.close() + snapshot = table.snapshot_manager().get_latest_snapshot() + assert snapshot.commit_user == 'application-1' + assert snapshot.commit_identifier == (7 if stream else BATCH_COMMIT_IDENTIFIER) + + updater = update.new_update_by_row_id(8) if stream else update.new_update_by_row_id() + assert updater.commit_user == 'application-1' + messages = updater.update_columns(pa.table({'_ROW_ID': [0], 'value': [11]}), ['value']) + recovered = create_builder().with_commit_user('application-1').new_commit() + try: + recovered.commit(messages, 8) if stream else recovered.commit(messages) + finally: + recovered.close() + snapshot = table.snapshot_manager().get_latest_snapshot() + assert snapshot.commit_user == 'application-1' + assert snapshot.commit_identifier == (8 if stream else BATCH_COMMIT_IDENTIFIER) + read = table.new_read_builder() + assert read.new_read().to_arrow(read.new_scan().plan().splits()).to_pylist() == [{'id': 1, 'value': 11}] + + +def test_postpone_builder_public_commit_user_reaches_the_committed_snapshot(tmp_path): + table, schema = _table(tmp_path, postpone=True) + builder = table.new_postpone_fixed_bucket_write_builder() + assert builder.with_commit_user('postpone-job') is builder + writer, commit = builder.new_write(), builder.new_commit() + assert writer.commit_user == commit.commit_user == 'postpone-job' + try: + writer.write_arrow(pa.table({'id': [1, 2], 'value': [10, 20]}, schema=schema)) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + snapshot = table.snapshot_manager().get_latest_snapshot() + assert snapshot.commit_user == 'postpone-job' + read = table.new_read_builder() + assert read.new_read().to_arrow(read.new_scan().plan().splits()).sort_by('id').to_pylist() == [ + {'id': 1, 'value': 10}, {'id': 2, 'value': 20}] diff --git a/paimon-python/pypaimon/write/write_builder.py b/paimon-python/pypaimon/write/write_builder.py index cbab080aaa0d..ca8a15ceceec 100644 --- a/paimon-python/pypaimon/write/write_builder.py +++ b/paimon-python/pypaimon/write/write_builder.py @@ -55,8 +55,14 @@ def with_restore_snapshot(self, snapshot_id: int): self.restore_snapshot_id = snapshot_id return self - def _with_commit_user(self, commit_user: str): - """Reuse the identity of an existing write or commit operation.""" + def with_commit_user(self, commit_user: str): + """Set the identity shared by new writers, updates and committers. + + The default is a generated UUID. Reuse the same identity when restoring + a stream application, and use different identities for independent + applications. Configure it before creating operations; existing + operations retain the identity they were created with. + """ self.commit_user = commit_user return self @@ -114,11 +120,11 @@ def _new_update_by_row_id( if commit_identifier == BATCH_COMMIT_IDENTIFIER: return (table.new_batch_write_builder() - ._with_commit_user(commit_user) + .with_commit_user(commit_user) .new_update() .new_update_by_row_id(_precomputed_files_info)) return (table.new_stream_write_builder() - ._with_commit_user(commit_user) + .with_commit_user(commit_user) .new_update() .new_update_by_row_id( commit_identifier, _precomputed_files_info)) From 83348d67cbc4dabfc6baf89a60603659e4a1a044 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 22 Oct 2026 20:17:16 +0800 Subject: [PATCH 3/6] [Python] Use resolve_bucket_plan for Native bucket metadata --- .../pypaimon/execution/native/write/postpone_bucket.py | 2 +- .../tests/write/postpone_execution_boundary_test.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/paimon-python/pypaimon/execution/native/write/postpone_bucket.py b/paimon-python/pypaimon/execution/native/write/postpone_bucket.py index 5ed29178bf10..2bb38944b5f5 100644 --- a/paimon-python/pypaimon/execution/native/write/postpone_bucket.py +++ b/paimon-python/pypaimon/execution/native/write/postpone_bucket.py @@ -38,7 +38,7 @@ def _load_bucket_metadata(table): native_table = create_native_write_table(table) if native_table is None: raise RuntimeError('Native bucket metadata planning is unavailable') - plan = native_table.new_postpone_fixed_bucket_write_builder().bucket_plan() + plan = native_table.new_postpone_fixed_bucket_write_builder().resolve_bucket_plan() # A partition key may itself be called total_buckets. Keep the # positional schema used by the Core plan and with_bucket_plan input. columns = [column.to_pylist() for column in plan.columns] diff --git a/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py b/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py index ce9a50a21003..8453fecbd5d4 100644 --- a/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py +++ b/paimon-python/pypaimon/tests/write/postpone_execution_boundary_test.py @@ -97,7 +97,7 @@ def _table(tmp_path, partition_name='p'): def _native_table(table): native = Mock() - native.new_postpone_fixed_bucket_write_builder.return_value.bucket_plan.return_value = pa.record_batch([ + native.new_postpone_fixed_bucket_write_builder.return_value.resolve_bucket_plan.return_value = pa.record_batch([ pa.array(['existing']), pa.array([3], type=pa.int32())], names=['p', 'total_buckets']) return native @@ -220,7 +220,7 @@ def test_native_bucket_metadata_reports_core_conflicts(tmp_path, forbid_python_e table = _table(tmp_path) native = _native_table(table) failure = ValueError('Existing postpone files contain conflicting total bucket counts') - native.new_postpone_fixed_bucket_write_builder.return_value.bucket_plan.side_effect = failure + native.new_postpone_fixed_bucket_write_builder.return_value.resolve_bucket_plan.side_effect = failure with patch('pypaimon.execution.native.write.postpone_bucket.create_native_write_table', return_value=native): with pytest.raises(ValueError, match='conflicting total bucket counts') as raised: NativePostponeBucketPlanner(table) @@ -232,7 +232,7 @@ def test_native_bucket_plan_keeps_a_partition_named_total_buckets(tmp_path): table = _table(tmp_path, 'total_buckets') native = _native_table(table) - native.new_postpone_fixed_bucket_write_builder.return_value.bucket_plan.return_value = pa.record_batch([ + native.new_postpone_fixed_bucket_write_builder.return_value.resolve_bucket_plan.return_value = pa.record_batch([ pa.array(['west', 'east']), pa.array([3, 7], type=pa.int32())], names=['total_buckets', 'total_buckets']) with patch('pypaimon.execution.native.write.postpone_bucket.create_native_write_table', return_value=native): From be02b4b4ea224f7cdec987536f993b2153562a0d Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 22 Oct 2026 20:53:15 +0800 Subject: [PATCH 4/6] [Python] Centralize writer construction and execution dispatch --- docs/docs/pypaimon/writing.md | 5 + paimon-python/pypaimon/execution/README.md | 5 + .../execution/python/write/merge_into.py | 6 +- .../execution/python/write/table_update.py | 7 - .../python/write/table_upsert_by_key.py | 6 +- .../execution/python/write/table_write.py | 9 +- .../pypaimon/tests/arrow_schema_test.py | 3 + .../tests/shard_table_updator_test.py | 5 +- .../write/direct_writer_execution_test.py | 369 ++++++++++++++++++ .../write/factory_execution_boundary_test.py | 122 ++++++ .../write/row_id_execution_boundary_test.py | 4 +- .../pypaimon/tests/write/table_write_test.py | 4 + .../pypaimon/write/backend_dispatch.py | 170 +++++++- .../write/postpone_batch_table_write.py | 22 +- paimon-python/pypaimon/write/table_delete.py | 6 +- paimon-python/pypaimon/write/table_update.py | 22 +- .../pypaimon/write/table_update_by_row_id.py | 6 +- paimon-python/pypaimon/write/table_write.py | 49 +-- .../pypaimon/write/update_dispatch.py | 49 +-- .../pypaimon/write/write_dispatch.py | 15 +- 20 files changed, 754 insertions(+), 130 deletions(-) create mode 100644 paimon-python/pypaimon/tests/write/direct_writer_execution_test.py create mode 100644 paimon-python/pypaimon/tests/write/factory_execution_boundary_test.py diff --git a/docs/docs/pypaimon/writing.md b/docs/docs/pypaimon/writing.md index 99d77a1f2e41..9925af72f94e 100644 --- a/docs/docs/pypaimon/writing.md +++ b/docs/docs/pypaimon/writing.md @@ -127,6 +127,11 @@ does not prevent Native updates on tables without VECTOR fields. These operation continue to write Parquet/BLOB files; an active dedicated format must still be supported by the Native path. +The same Native selection applies to direct construction of `TableWrite`, +`BatchTableWrite`, `StreamTableWrite` and `PostponeFixedBucketBatchTableWrite`. +Their public classes, constructor signatures and Arrow/Pandas batch overrides +are preserved. Python execution is initialized only when that backend is selected. + `write_ray()` uses the shared Python coordinator and lets workers select Native writers. For postpone fixed-bucket writes, overwrite plans use the new input unless an explicit bucket plan was supplied. A complete primary-key diff --git a/paimon-python/pypaimon/execution/README.md b/paimon-python/pypaimon/execution/README.md index 822fe5e157ac..88845fe0aa8a 100644 --- a/paimon-python/pypaimon/execution/README.md +++ b/paimon-python/pypaimon/execution/README.md @@ -107,6 +107,11 @@ private scanner and retains its write authorization boundary. Direct `TableUpdateByRowId`, `TableDeleteByRowId` and `ShardTableUpdator` construction also selects one execution backend before loading target files. +Direct `TableWrite`, batch/stream writers and postpone fixed-bucket writers +use the same selection boundary as builders while retaining their public class +and subclass hooks. Writer resources and update executors are created in +`write/backend_dispatch.py`; the public classes only delegate to the selected +implementation. Python-only nested writers use explicit Python factories. An operation that selected Python keeps its nested update/delete writers in Python, including MERGE actions. Private schema, routing and rewrite algorithms call their executor's helpers; public method overrides and user callbacks remain diff --git a/paimon-python/pypaimon/execution/python/write/merge_into.py b/paimon-python/pypaimon/execution/python/write/merge_into.py index 04ce05a52966..9d7d109db87c 100644 --- a/paimon-python/pypaimon/execution/python/write/merge_into.py +++ b/paimon-python/pypaimon/execution/python/write/merge_into.py @@ -35,7 +35,7 @@ from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER from pypaimon.table.special_fields import SpecialFields from pypaimon.write.commit_message import CommitMessage -from pypaimon.write.table_write import BatchTableWrite, StreamTableWrite +from pypaimon.write.backend_dispatch import create_python_write from pypaimon.write.table_delete import TableDeleteByRowId from pypaimon.write.merge_inputs import ( _PrepareCtx, @@ -513,9 +513,7 @@ def _prepare_commit_messages( def _new_table_write(table, commit_user: str, commit_identifier: int): - if commit_identifier == BATCH_COMMIT_IDENTIFIER: - return BatchTableWrite(table, commit_user) - return StreamTableWrite(table, commit_user) + return create_python_write(table, commit_user, stream=commit_identifier != BATCH_COMMIT_IDENTIFIER) def _normalize_source(source: Any, on_columns=()) -> pa.Table: diff --git a/paimon-python/pypaimon/execution/python/write/table_update.py b/paimon-python/pypaimon/execution/python/write/table_update.py index 3074842f5c0b..ce54a83348ef 100644 --- a/paimon-python/pypaimon/execution/python/write/table_update.py +++ b/paimon-python/pypaimon/execution/python/write/table_update.py @@ -485,13 +485,6 @@ def _execute_predicate_delete(self, predicate, commit_identifier): row_ids = self._matched_delete_row_ids(predicate) return self._delete_rows_with_python(row_ids) - def _new_python_shard_updator(self, shard_num, total_shard_count): - context = self._context - from pypaimon.write.table_update import ShardTableUpdator - return ShardTableUpdator._from_python( - context.table, context.projection, context.update_cols, - context.commit_user, shard_num, total_shard_count) - def _delete_rows_with_python(self, row_ids): context = self._context return TableDeleteByRowId._from_python(context.table).delete(list(row_ids)) diff --git a/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py b/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py index 41ae859ca338..2e1fe4fade2e 100644 --- a/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py +++ b/paimon-python/pypaimon/execution/python/write/table_upsert_by_key.py @@ -34,7 +34,7 @@ from pypaimon.write.commit_message import CommitMessage from pypaimon.write.row_utils import require_columns from pypaimon.write.update_validation import _authorize_update -from pypaimon.write.table_write import StreamTableWrite +from pypaimon.write.backend_dispatch import create_python_write # Composite key is represented as a tuple of values _KeyTuple = Tuple[Any, ...] @@ -454,7 +454,7 @@ def _do_appends( all_ordered_cols = [c for c in context.table.field_names if c in new_data.column_names] new_data = new_data.select(all_ordered_cols) - table_write = StreamTableWrite(context.table, context.commit_user) + table_write = create_python_write(context.table, context.commit_user, stream=True) try: table_write.with_write_type(all_ordered_cols) table_write.write_arrow(new_data) @@ -469,7 +469,7 @@ def _append_rows( context = self._context all_ordered_cols = self._append_row_column_names(row_items) - table_write = StreamTableWrite(context.table, context.commit_user) + table_write = create_python_write(context.table, context.commit_user, stream=True) try: table_write.with_write_type(all_ordered_cols) for row, _ in row_items: diff --git a/paimon-python/pypaimon/execution/python/write/table_write.py b/paimon-python/pypaimon/execution/python/write/table_write.py index c893807b0aaa..36fc14855540 100644 --- a/paimon-python/pypaimon/execution/python/write/table_write.py +++ b/paimon-python/pypaimon/execution/python/write/table_write.py @@ -39,8 +39,9 @@ class PythonTableWrite: """Bound Python writer implementing the common write execution protocol.""" - def __init__(self, writer): + def __init__(self, writer, public_writer=None): self.writer = writer + self.public_writer = public_writer if public_writer is not None else writer def initialize(self, table, commit_user, static_partition: Optional[dict] = None, *, restore_snapshot_id=None): @@ -87,9 +88,9 @@ def write_arrow(self, data, bucket: Optional[int] = None): # Public batch overrides remain observable for Arrow and pandas input. # Schema, routing and preparation helpers stay within this executor. if isinstance(data, pa.RecordBatch): - return self.writer.write_arrow_batch(data, bucket) + return self.public_writer.write_arrow_batch(data, bucket) for batch in data.to_batches(): - self.writer.write_arrow_batch(batch, bucket) + self.public_writer.write_arrow_batch(batch, bucket) def write_arrow_batch(self, data: pa.RecordBatch, bucket: Optional[int] = None): writer = self.writer @@ -204,7 +205,7 @@ def write_pandas(self, dataframe): pa_schema = writer.table_pyarrow_schema record_batch = _pandas_write_batch( getattr(writer, 'table', None), dataframe, pa_schema, writer.table_pyarrow_schema) - return writer.write_arrow_batch(record_batch) + return self.public_writer.write_arrow_batch(record_batch) def with_write_type(self, write_cols: List[str]): writer = self.writer diff --git a/paimon-python/pypaimon/tests/arrow_schema_test.py b/paimon-python/pypaimon/tests/arrow_schema_test.py index a0f26438aa85..02a0a069198c 100644 --- a/paimon-python/pypaimon/tests/arrow_schema_test.py +++ b/paimon-python/pypaimon/tests/arrow_schema_test.py @@ -125,6 +125,7 @@ def test_core_writes_canonical_strings(self, entry): @parameterized.expand([('table',), ('batch',)]) def test_schema_only_writer_normalizes_input(self, entry): writer = object.__new__(TableWrite) + writer._execution = PythonTableWrite(writer) writer.file_store_write = Mock(write_cols=None) writer.table_pyarrow_schema = pa.schema([('text', pa.string())]) source = pa.Table.from_pydict({'text': ['中文', None]}, schema=pa.schema([('text', pa.large_string())])) @@ -141,6 +142,7 @@ def test_schema_only_writer_normalizes_input(self, entry): @parameterized.expand([('table',), ('batch',)]) def test_normalization_failure_fails_before_routing(self, entry): writer = object.__new__(TableWrite) + writer._execution = PythonTableWrite(writer) writer.file_store_write = Mock(write_cols=None) writer.row_key_extractor = Mock() writer.table_pyarrow_schema = pa.schema([('text', pa.string())]) @@ -159,6 +161,7 @@ def test_normalization_failure_fails_before_routing(self, entry): @unittest.skipUnless(int(pa.__version__.split('.')[0]) == 6, 'Arrow 6 lacks struct cast kernels') def test_unsupported_nested_cast_fails_before_routing(self): writer = object.__new__(TableWrite) + writer._execution = PythonTableWrite(writer) writer.file_store_write = Mock(write_cols=None) writer.row_key_extractor = Mock() writer.table_pyarrow_schema = pa.schema([('nested', pa.struct([('text', pa.string())]))]) diff --git a/paimon-python/pypaimon/tests/shard_table_updator_test.py b/paimon-python/pypaimon/tests/shard_table_updator_test.py index 8e39fed553ef..fb3968f81db5 100644 --- a/paimon-python/pypaimon/tests/shard_table_updator_test.py +++ b/paimon-python/pypaimon/tests/shard_table_updator_test.py @@ -25,7 +25,6 @@ import pyarrow as pa from pypaimon import CatalogFactory, Schema -from pypaimon.execution.python.write.table_update import PythonTableUpdate class ShardUpdateValidationTest(unittest.TestCase): @@ -42,8 +41,8 @@ def test_empty_update_columns_are_rejected_before_selecting_a_backend(self): update = update_type(table, 'writer').with_update_type([]) with patch('pypaimon.write.update_dispatch.prepare_native_write_operation', side_effect=AssertionError('Prepared a backend for invalid columns')) as prepare, \ - patch.object(PythonTableUpdate, '_new_python_shard_updator', - side_effect=AssertionError('Created a Python shard updater')) as python: + patch('pypaimon.write.backend_dispatch.create_python_shard_updator', + side_effect=AssertionError('Created a Python shard updater')) as python: with self.assertRaisesRegex(ValueError, 'Shard update columns must not be empty'): update.new_shard_updator(0, 1) prepare.assert_not_called() diff --git a/paimon-python/pypaimon/tests/write/direct_writer_execution_test.py b/paimon-python/pypaimon/tests/write/direct_writer_execution_test.py new file mode 100644 index 000000000000..f66c2c4807aa --- /dev/null +++ b/paimon-python/pypaimon/tests/write/direct_writer_execution_test.py @@ -0,0 +1,369 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Direct public writer construction shares the builder execution boundary.""" + +from contextlib import ExitStack, contextmanager +from pathlib import Path +import subprocess +import sys +from unittest.mock import Mock, call, patch + +import pyarrow as pa +import pytest + +from pypaimon import CatalogFactory, Schema +from pypaimon.schema.data_types import PyarrowFieldParser +from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER +from pypaimon.write.bucket_plan import PostponeBucketPlan +from pypaimon.write.postpone_batch_table_write import PostponeFixedBucketBatchTableWrite +from pypaimon.write.table_commit import BatchTableCommit, StreamTableCommit +from pypaimon.write.table_write import BatchTableWrite, StreamTableWrite, TableWrite + + +WRITERS = [TableWrite, BatchTableWrite, StreamTableWrite, PostponeFixedBucketBatchTableWrite] + +pytestmark = [pytest.mark.python_plan, pytest.mark.python_read, pytest.mark.python_commit] + + +def _table(tmp_path, writer_class): + postpone = issubclass(writer_class, PostponeFixedBucketBatchTableWrite) + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + schema = pa.schema([('id', pa.int32()), ('value', pa.int32())]) + catalog.create_table('default.direct', Schema.from_pyarrow_schema( + schema, primary_keys=['id'] if postpone else [], options={ + 'bucket': '-2' if postpone else '-1', 'postpone.default-bucket-num': '2', + 'write.native.enabled': 'true', 'commit.native.enabled': 'false', + 'scan.native-plan.enabled': 'false', 'read.native.enabled': 'false', + }), False) + return catalog.get_table('default.direct'), schema + + +def _construct(writer_class, table): + if issubclass(writer_class, PostponeFixedBucketBatchTableWrite): + return writer_class(table, 'direct-writer', bucket_plan=PostponeBucketPlan({(): 2})) + return writer_class(table, 'direct-writer') + + +def _prepare(writer): + if isinstance(writer, StreamTableWrite): + return writer.prepare_commit(7) + if isinstance(writer, BatchTableWrite): + return writer.prepare_commit() + return writer._prepare_commit(BATCH_COMMIT_IDENTIFIER) + + +@contextmanager +def _native_backend(forbid_python=True): + """Use the real thin Native adapter without loading or building the SDK.""" + native_table, builder, core = Mock(), Mock(), Mock() + for name in ('new_batch_write_builder', 'new_stream_write_builder', + 'new_postpone_fixed_bucket_write_builder'): + getattr(native_table, name).return_value = builder + for name in ('with_commit_user', 'with_bucket_plan', 'with_overwrite'): + getattr(builder, name).return_value = builder + builder.new_write.return_value = core + core.prepare_commit.return_value = ['wire-message'] + with ExitStack() as stack: + stack.enter_context(patch( + 'pypaimon.execution.native.write.native_write.native_runtime_available', return_value=True)) + create = stack.enter_context(patch( + 'pypaimon.execution.native.write.native_write.create_native_write_table', return_value=native_table)) + stack.enter_context(patch( + 'pypaimon.execution.native.write.native_write.from_native_commit_messages', return_value=['message'])) + if forbid_python: + stack.enter_context(patch( + 'pypaimon.execution.python.write.table_write.PythonTableWrite.initialize', + side_effect=AssertionError('Native initialized Python execution'))) + yield core, create + + +@pytest.mark.parametrize('writer_class', WRITERS) +def test_direct_writer_selects_native_without_python_initialization(tmp_path, writer_class): + table, schema = _table(tmp_path, writer_class) + batch = pa.record_batch([[1], [10]], schema=schema) + with _native_backend() as (core, create): + writer = _construct(writer_class, table) + try: + assert type(writer) is writer_class + writer.write_arrow_batch(batch) + assert _prepare(writer) == ['message'] + finally: + writer.close() + create.assert_called_once_with(table) + core.write_arrow.assert_called_once_with(batch, None) + core.prepare_commit.assert_called_once_with(*((True, 7) if writer_class is StreamTableWrite else ())) + core.close.assert_called_once_with() + + +@pytest.mark.parametrize('writer_class', WRITERS) +def test_direct_writer_metadata_inspection_does_not_select_python(tmp_path, writer_class): + table, _ = _table(tmp_path, writer_class) + schema = PyarrowFieldParser.from_paimon_schema(table.fields) + with _native_backend() as (core, create): + writer = _construct(writer_class, table) + try: + assert writer.table is table + assert writer.commit_user == 'direct-writer' + assert writer.static_partition is None + assert writer.table_pyarrow_schema == schema + assert writer.restore_snapshot_id is None + assert not hasattr(writer, 'missing_writer_attribute') + with pytest.raises(AttributeError): + getattr(writer, 'missing_writer_attribute') + if isinstance(writer, BatchTableWrite): + assert writer.batch_committed is False + create.assert_called_once_with(table) + core.close.assert_not_called() + finally: + writer.close() + + +@pytest.mark.parametrize('writer_class', WRITERS) +def test_direct_writer_native_write_failure_is_not_replayed(tmp_path, writer_class): + table, schema = _table(tmp_path, writer_class) + failure = OSError('file write may already have succeeded') + with _native_backend() as (core, create): + core.write_arrow.side_effect = failure + writer = _construct(writer_class, table) + try: + with pytest.raises(OSError) as caught: + writer.write_arrow_batch(pa.record_batch([[1], [10]], schema=schema)) + assert caught.value is failure + create.assert_called_once_with(table) + finally: + writer.close() + core.write_arrow.assert_called_once() + + +@pytest.mark.parametrize('writer_class', WRITERS) +def test_direct_native_writer_configuration_keeps_public_receiver(tmp_path, writer_class): + table, _ = _table(tmp_path, writer_class) + with _native_backend() as (core, _): + writer = _construct(writer_class, table) + try: + assert writer.with_write_type(['id', 'value']) is writer + assert writer.with_blob_consumer(None) is writer + assert writer.with_blob_uri_reader_factory(None) is writer + core.with_blob_consumer.assert_called_once_with(None) + core.with_blob_uri_reader_factory.assert_called_once_with(None) + finally: + writer.close() + + +@pytest.mark.parametrize('writer_class', WRITERS) +def test_direct_writer_python_selection_is_not_revisited(tmp_path, writer_class): + table, schema = _table(tmp_path, writer_class) + data = pa.table({'id': [1], 'value': [10]}, schema=schema) + with patch('pypaimon.execution.native.write.native_write.create_native_write', return_value=None) as prepare: + writer = _construct(writer_class, table) + try: + assert type(writer) is writer_class + writer.write_arrow(data) + messages = _prepare(writer) + finally: + writer.close() + prepare.assert_called_once() + commit = (StreamTableCommit(table, 'direct-writer') if writer_class is StreamTableWrite else + BatchTableCommit(table, 'direct-writer', None)) + try: + commit.commit(messages, 7) if writer_class is StreamTableWrite else commit.commit(messages) + finally: + commit.close() + read = table.new_read_builder() + assert read.new_read().to_arrow(read.new_scan().plan().splits()).to_pylist() == data.to_pylist() + + +@pytest.mark.parametrize('writer_class', WRITERS) +@pytest.mark.parametrize('pandas', [False, True]) +def test_direct_native_writer_preserves_public_batch_override(tmp_path, writer_class, pandas): + table, schema = _table(tmp_path, writer_class) + + class ObservingWriter(writer_class): + def write_arrow_batch(self, data, bucket=None): + self.observed.append(data) + return super().write_arrow_batch(data, bucket) + + with _native_backend() as (core, _): + writer = _construct(ObservingWriter, table) + writer.observed = [] + data = pa.table({'id': [1, 2], 'value': [10, 20]}, schema=schema) + try: + writer.write_pandas(data.to_pandas()) if pandas else writer.write_arrow(data) + assert isinstance(writer, ObservingWriter) + assert len(writer.observed) == 1 + assert writer.observed[0].to_pydict() == data.to_pydict() + core.write_arrow.assert_called_once() + finally: + writer.close() + + +@pytest.mark.parametrize('writer_class', WRITERS) +@pytest.mark.parametrize('pandas', [False, True]) +def test_direct_writer_diagnostic_switch_preserves_outer_batch_override(tmp_path, writer_class, pandas): + table, schema = _table(tmp_path, writer_class) + + class ObservingWriter(writer_class): + def write_arrow_batch(self, data, bucket=None): + self.observed.append(data) + return super().write_arrow_batch(data, bucket) + + with _native_backend(forbid_python=False) as (core, create): + writer = _construct(ObservingWriter, table) + writer.observed = [] + try: + # Explicit Python diagnostics select that backend before mutation. + # The outer public subclass must continue to observe batch writes. + assert writer.file_store_write is writer.file_store_write + core.close.assert_called_once_with() + data = pa.table({'id': [1], 'value': [10]}, schema=schema) + writer.write_pandas(data.to_pandas()) if pandas else writer.write_arrow(data) + assert len(writer.observed) == 1 + assert writer.observed[0].to_pydict() == data.to_pydict() + assert _prepare(writer) + create.assert_called_once_with(table) + core.write_arrow.assert_not_called() + core.prepare_commit.assert_not_called() + finally: + writer.close() + core.close.assert_called_once_with() + + +@pytest.mark.parametrize('overwrite', [False, True]) +def test_direct_postpone_diagnostic_switch_reuses_python_distributed_planner(tmp_path, overwrite): + table, _ = _table(tmp_path, PostponeFixedBucketBatchTableWrite) + with _native_backend(forbid_python=False): + writer = _construct(PostponeFixedBucketBatchTableWrite, table) + try: + assert writer.file_store_write is writer.file_store_write + pure = writer._execution._python_writer + planner = pure._planner + with patch('pypaimon.write.backend_dispatch.create_postpone_bucket_planner', + side_effect=AssertionError('distributed write selected a second planner')): + options = writer._distributed_write_options(overwrite=overwrite) + assert options['postpone_bucket_planner'] is planner + finally: + writer.close() + + +@pytest.mark.parametrize('writer_class', [BatchTableWrite, PostponeFixedBucketBatchTableWrite]) +def test_direct_native_batch_preparation_failure_consumes_one_attempt(tmp_path, writer_class): + table, _ = _table(tmp_path, writer_class) + failure = OSError('preparation may already have succeeded') + with _native_backend() as (core, _): + core.prepare_commit.side_effect = failure + writer = _construct(writer_class, table) + try: + with pytest.raises(TypeError): + writer.prepare_commit(1) + core.prepare_commit.assert_not_called() + assert writer.batch_committed is False + with pytest.raises(OSError) as caught: + writer.prepare_commit() + assert caught.value is failure + assert writer.batch_committed is True + with pytest.raises(RuntimeError, match='one-time committing'): + writer.prepare_commit() + core.prepare_commit.assert_called_once_with() + finally: + writer.close() + + +def test_direct_native_stream_preparation_validates_before_io_and_supports_multiple_rounds(tmp_path): + table, _ = _table(tmp_path, StreamTableWrite) + with _native_backend() as (core, _): + writer = StreamTableWrite(table, 'direct-writer') + try: + with pytest.raises(TypeError): + writer.prepare_commit() + for identifier in (None, 7.5, '7'): + with pytest.raises(TypeError): + writer.prepare_commit(identifier) + core.prepare_commit.assert_not_called() + assert writer.prepare_commit(7) == ['message'] + assert writer.prepare_commit(8) == ['message'] + assert core.prepare_commit.call_args_list == [call(True, 7), call(True, 8)] + finally: + writer.close() + + +@pytest.mark.parametrize('writer_class', WRITERS) +def test_direct_native_writer_never_imports_python_execution(tmp_path, writer_class): + source = r''' +import importlib.abc +import sys +from unittest.mock import Mock, patch + +class BlockPythonExecution(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == 'pypaimon.execution.python' or fullname.startswith('pypaimon.execution.python.'): + raise AssertionError('Native imported Python execution: ' + fullname) + +sys.meta_path.insert(0, BlockPythonExecution()) +import pyarrow as pa +from pypaimon import CatalogFactory, Schema +from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER +from pypaimon.write.bucket_plan import PostponeBucketPlan +from pypaimon.write.postpone_batch_table_write import PostponeFixedBucketBatchTableWrite +from pypaimon.write.table_write import BatchTableWrite, StreamTableWrite, TableWrite + +writer_class = globals()[sys.argv[1]] +postpone = writer_class is PostponeFixedBucketBatchTableWrite +catalog = CatalogFactory.create({'warehouse': sys.argv[2]}) +catalog.create_database('default', True) +schema = pa.schema([('id', pa.int32()), ('value', pa.int32())]) +catalog.create_table('default.direct', Schema.from_pyarrow_schema( + schema, primary_keys=['id'] if postpone else [], options={ + 'bucket': '-2' if postpone else '-1', 'postpone.default-bucket-num': '2', + 'write.native.enabled': 'true', 'commit.native.enabled': 'false', + 'scan.native-plan.enabled': 'false', 'read.native.enabled': 'false', + }), False) +table = catalog.get_table('default.direct') +native_table, builder, core = Mock(), Mock(), Mock() +for name in ('new_batch_write_builder', 'new_stream_write_builder', 'new_postpone_fixed_bucket_write_builder'): + getattr(native_table, name).return_value = builder +for name in ('with_commit_user', 'with_bucket_plan'): + getattr(builder, name).return_value = builder +builder.new_write.return_value = core +core.prepare_commit.return_value = [] +with patch('pypaimon.execution.native.write.native_write.native_runtime_available', return_value=True), \ + patch('pypaimon.execution.native.write.native_write.create_native_write_table', return_value=native_table), \ + patch('pypaimon.execution.native.write.native_write.from_native_commit_messages', return_value=[]): + kwargs = {'bucket_plan': PostponeBucketPlan({(): 2})} if postpone else {} + writer = writer_class(table, 'native-direct', **kwargs) + try: + assert type(writer) is writer_class + writer.write_arrow(pa.table({'id': [1], 'value': [10]}, schema=schema)) + if writer_class is StreamTableWrite: + assert writer.prepare_commit(7) == [] + elif writer_class is TableWrite: + assert writer._prepare_commit(BATCH_COMMIT_IDENTIFIER) == [] + else: + assert writer.prepare_commit() == [] + finally: + writer.close() +core.write_arrow.assert_called_once() +assert core.write_arrow.call_args[0][0].to_pydict() == {'id': [1], 'value': [10]} +core.close.assert_called_once_with() +assert not any(name.startswith('pypaimon.execution.python') for name in sys.modules) +''' + result = subprocess.run( + [sys.executable, '-c', source, writer_class.__name__, str(tmp_path / 'warehouse')], + cwd=str(Path(__file__).resolve().parents[3]), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/paimon-python/pypaimon/tests/write/factory_execution_boundary_test.py b/paimon-python/pypaimon/tests/write/factory_execution_boundary_test.py new file mode 100644 index 000000000000..8008a21be24e --- /dev/null +++ b/paimon-python/pypaimon/tests/write/factory_execution_boundary_test.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Selected Python factories retain state without reopening Native execution.""" + +from unittest.mock import Mock, patch + +import pyarrow as pa +import pytest + +from pypaimon import CatalogFactory, Schema +from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER +from pypaimon.write.table_delete import TableDeleteByRowId +from pypaimon.write.table_update import ShardTableUpdator, SingleWriter +from pypaimon.write.table_update_by_row_id import TableUpdateByRowId + +pytestmark = [pytest.mark.python_plan, pytest.mark.python_read, + pytest.mark.python_write, pytest.mark.python_commit] + + +@pytest.fixture +def table(tmp_path): + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + schema = pa.schema([('id', pa.int32()), ('value', pa.int32())]) + catalog.create_table('default.data', Schema.from_pyarrow_schema(schema, options={ + 'data-evolution.enabled': 'true', 'row-tracking.enabled': 'true', + 'deletion-vectors.enabled': 'true', 'write.native.enabled': 'false', + 'commit.native.enabled': 'false', 'read.native.enabled': 'false', 'scan.native-plan.enabled': 'false', + }), False) + table = catalog.get_table('default.data') + builder = table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + try: + writer.write_arrow(pa.table({'id': [1, 2], 'value': [10, 20]}, schema=schema)) + commit.commit(writer.prepare_commit()) + finally: + writer.close() + commit.close() + return table.copy({'write.native.enabled': 'true'}) + + +@pytest.mark.parametrize('operation', ['row-id', 'delete', 'shard']) +def test_explicit_python_factory_keeps_captured_state_and_committable_results(table, operation): + with patch('pypaimon.write.backend_dispatch.prepare_native_write_operation', + side_effect=AssertionError('Selected Python operation reopened Native')): + if operation == 'row-id': + updater = TableUpdateByRowId._from_python(table, 'writer', BATCH_COMMIT_IDENTIFIER) + captured = updater._snapshot_files_info() + messages = updater.update_columns(pa.table({'_ROW_ID': [0], 'value': [11]}), ['value']) + # Different columns of the same file group share the captured + # snapshot and append to the same executor's accumulated messages. + assert updater.update_columns(pa.table({'_ROW_ID': [1], 'id': [3]}), ['id']) is messages + assert updater._snapshot_files_info().snapshot_id == captured.snapshot_id + assert all(message.check_from_snapshot == captured.snapshot_id for message in messages) + expected = [{'id': 1, 'value': 11}, {'id': 3, 'value': 20}] + elif operation == 'delete': + coordinator = TableDeleteByRowId._from_python(table) + captured = coordinator._snapshot_anchor_ranges() + worker = TableDeleteByRowId._from_python(table, captured) + messages = worker.delete([0, 0]) + assert all(message.check_from_snapshot == captured.snapshot_id for message in messages) + expected = [{'id': 2, 'value': 20}] + else: + shard = ShardTableUpdator._from_python(table, ['value'], ['value'], 'writer', 0, 1) + try: + reader = shard.arrow_reader() + for batch in reader: + shard.update_by_arrow_batch(pa.record_batch([ + pa.array([11] * batch.num_rows, type=pa.int32())], names=['value'])) + messages = shard.prepare_commit() + assert shard.prepare_commit() == messages + finally: + shard.close() + expected = [{'id': 1, 'value': 11}, {'id': 2, 'value': 11}] + assert messages + commit = table.new_batch_write_builder().new_commit() + try: + commit.commit(messages) + finally: + commit.close() + read = table.new_read_builder() + assert read.new_read().to_arrow(read.new_scan().plan().splits()).sort_by('id').to_pylist() == expected + + +def test_single_writer_factory_preserves_capacity_and_metadata_across_calls(table): + data_writer = Mock() + read = table.new_read_builder() + metadata = read.new_scan().plan().splits()[0].files[0] + data_writer.prepare_commit.return_value = [metadata] + writer = SingleWriter(data_writer, (), 100, 2) + assert writer.capacity() == 2 + with pytest.raises(Exception, match='exceeds capacity'): + writer.write(pa.record_batch([pa.array([1, 2, 3])], names=['value'])) + assert writer.capacity() == 2 + data_writer.write.assert_not_called() + for value in [10, 20]: + writer.write(pa.record_batch([pa.array([value])], names=['value'])) + if writer.capacity(): + with pytest.raises(Exception, match='capacity left'): + writer.end() + data_writer.prepare_commit.assert_not_called() + assert writer.capacity() == 0 + result = writer.end() + assert result.first_row_id == 100 + assert result.row_count == 2 + assert data_writer.write.call_count == 2 + data_writer.prepare_commit.assert_called_once_with() diff --git a/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py b/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py index c8fc3052a44f..2db9cd18db8e 100644 --- a/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py +++ b/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py @@ -41,7 +41,7 @@ def test_direct_row_id_update_caches_executor_and_delegates_messages(): assert update.update_columns(data, ['value']) is messages assert update.commit_messages is execution.commit_messages update._write_row_columns(data, ['value'], {'value': [10]}) - prepare.assert_called_once_with(update, table, 'user', BATCH_COMMIT_IDENTIFIER, None) + prepare.assert_called_once_with(table, 'user', BATCH_COMMIT_IDENTIFIER, None) execution.update_columns.assert_called_once_with(data, ['value']) execution._write_row_columns.assert_called_once_with(data, ['value'], {'value': [10]}) assert not hasattr(update, '_merge_update_with_original') @@ -68,7 +68,7 @@ def test_direct_delete_caches_executor_and_preserves_execution_error(): execution.delete.side_effect = RuntimeError('already staged') with pytest.raises(RuntimeError, match='already staged'): delete.delete([2]) - prepare.assert_called_once_with(delete, table, None) + prepare.assert_called_once_with(table, None) assert not hasattr(delete, '_scan_anchor_ranges') assert not hasattr(delete, '_write_deletion_vector_index') diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py b/paimon-python/pypaimon/tests/write/table_write_test.py index fffae7605dd2..8ab45cb3ac91 100644 --- a/paimon-python/pypaimon/tests/write/table_write_test.py +++ b/paimon-python/pypaimon/tests/write/table_write_test.py @@ -44,6 +44,7 @@ class TableWriteTest(unittest.TestCase): def test_large_string_compatibility_rejects_other_type_changes(self): writer = object.__new__(TableWrite) + writer._execution = PythonTableWrite(writer) writer.file_store_write = Mock(write_cols=None) writer.table_pyarrow_schema = pa.schema([ ('id', pa.int32()), @@ -232,6 +233,7 @@ def _groups_from_rows(partitions, buckets, num_rows): @staticmethod def _mock_table_write(partitions, buckets, schema): table_write = object.__new__(TableWrite) + table_write._execution = PythonTableWrite(table_write) table_write.table = Mock( is_primary_key_table=False, options=CoreOptions(Options({})), fields=Schema.from_pyarrow_schema(schema).fields) @@ -309,6 +311,7 @@ def test_write_arrow_batch_noncontiguous_group_uses_take(self): 'payload': [b'a', b'b', b'c', b'd'], }) table_write = object.__new__(TableWrite) + table_write._execution = PythonTableWrite(table_write) table_write.table = Mock( is_primary_key_table=False, options=CoreOptions(Options({})), fields=Schema.from_pyarrow_schema(data.schema).fields) @@ -336,6 +339,7 @@ def test_write_arrow_batch_contiguous_group_uses_zero_copy_slice(self): 'payload': [b'a', b'b', b'c', b'd'], }) table_write = object.__new__(TableWrite) + table_write._execution = PythonTableWrite(table_write) table_write.table = Mock( is_primary_key_table=False, options=CoreOptions(Options({})), fields=Schema.from_pyarrow_schema(data.schema).fields) diff --git a/paimon-python/pypaimon/write/backend_dispatch.py b/paimon-python/pypaimon/write/backend_dispatch.py index a1f49e7fbc89..cfd039adc27e 100644 --- a/paimon-python/pypaimon/write/backend_dispatch.py +++ b/paimon-python/pypaimon/write/backend_dispatch.py @@ -55,18 +55,38 @@ def try_prepare_native(factory, *args, **kwargs): def create_python_write(table, commit_user, static_partition=None, stream=False, - restore_snapshot_id=None, fixed_bucket=False, bucket_plan=None): + restore_snapshot_id=None, fixed_bucket=False, bucket_plan=None, + public_writer=None): if fixed_bucket: from pypaimon.write.postpone_batch_table_write import PostponeFixedBucketBatchTableWrite - return PostponeFixedBucketBatchTableWrite(table, commit_user, static_partition, bucket_plan) - from pypaimon.write.table_write import BatchTableWrite, StreamTableWrite - if stream: - return StreamTableWrite(table, commit_user, restore_snapshot_id=restore_snapshot_id) - return BatchTableWrite(table, commit_user, static_partition, restore_snapshot_id=restore_snapshot_id) + writer_type = PostponeFixedBucketBatchTableWrite + else: + from pypaimon.write.table_write import BatchTableWrite, StreamTableWrite + writer_type = StreamTableWrite if stream else BatchTableWrite + writer = object.__new__(writer_type) + _initialize_python_write(writer, table, commit_user, static_partition, stream, + restore_snapshot_id, fixed_bucket, bucket_plan, public_writer) + return writer -def create_write(table, commit_user, static_partition=None, stream=False, - restore_snapshot_id=None, fixed_bucket=False, bucket_plan=None): +def _initialize_python_write(owner, table, commit_user, static_partition, stream, + restore_snapshot_id, fixed_bucket, bucket_plan, public_writer=None): + if fixed_bucket: + from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWrite + execution = PythonPostponeFixedBucketWrite(owner, public_writer) + execution.initialize(table, commit_user, static_partition, bucket_plan) + else: + from pypaimon.execution.python.write.table_write import PythonTableWrite + execution = PythonTableWrite(owner, public_writer) + execution.initialize(table, commit_user, static_partition, + restore_snapshot_id=restore_snapshot_id) + owner._execution = execution + if not stream: + owner.__dict__['batch_committed'] = False + + +def _prepare_write(table, commit_user, static_partition, stream, + restore_snapshot_id, fixed_bucket, bucket_plan): from pypaimon.read.merge_engine_support import check_sequence_field_supported check_sequence_field_supported(table) if table.options.native_write_enabled(): @@ -79,10 +99,79 @@ def prepare(): if native is not None: from pypaimon.write.write_dispatch import NativeTableWrite return NativeTableWrite.from_native(native) + + +def initialize_write(owner, table, commit_user, static_partition=None, stream=False, + restore_snapshot_id=None, fixed_bucket=False, bucket_plan=None): + """Select once while retaining the caller's public class and batch hooks.""" + execution = _prepare_write(table, commit_user, static_partition, stream, + restore_snapshot_id, fixed_bucket, bucket_plan) + if execution is None: + _initialize_python_write(owner, table, commit_user, static_partition, stream, + restore_snapshot_id, fixed_bucket, bucket_plan) + else: + execution._public_writer = owner + owner._execution = execution + + +def create_write(table, commit_user, static_partition=None, stream=False, + restore_snapshot_id=None, fixed_bucket=False, bucket_plan=None): + execution = _prepare_write(table, commit_user, static_partition, stream, + restore_snapshot_id, fixed_bucket, bucket_plan) + if execution is not None: + return execution return create_python_write( table, commit_user, static_partition, stream, restore_snapshot_id, fixed_bucket, bucket_plan) +def get_write_attribute(owner, name): + from pypaimon.write.write_dispatch import NativeTableWrite + execution = owner.__dict__.get('_execution') + if isinstance(execution, NativeTableWrite): + return getattr(execution, name) + try: + return owner.__dict__[name] + except KeyError: + raise AttributeError(name) from None + + +def write_property(name): + """Preserve public writer attributes without selecting a second backend.""" + def get(owner): + return get_write_attribute(owner, name) + + def set_(owner, value): + from pypaimon.write.write_dispatch import NativeTableWrite + execution = owner.__dict__.get('_execution') + if isinstance(execution, NativeTableWrite): + setattr(execution, name, value) + else: + owner.__dict__[name] = value + + return property(get, set_) + + +def prepare_write_commit(owner, commit_identifier): + from pypaimon.write.write_dispatch import NativeTableWrite + execution = owner._execution + if isinstance(execution, NativeTableWrite): + return (execution.prepare_commit(commit_identifier) if execution.stream else + execution.prepare_commit()) + return execution.prepare_commit(commit_identifier) + + +def prepare_batch_write(owner): + from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER + from pypaimon.write.write_dispatch import NativeTableWrite + execution = owner._execution + if isinstance(execution, NativeTableWrite): + return execution.prepare_commit() + if owner.batch_committed: + raise RuntimeError('BatchTableWrite only supports one-time committing.') + owner.batch_committed = True + return owner._prepare_commit(BATCH_COMMIT_IDENTIFIER) + + def switch_writer_to_python(owner): """Switch only before the native writer has accepted its first mutation.""" if owner._python_writer is not None: @@ -178,7 +267,62 @@ def prepare_native_merge_into(*args, **kwargs): return create_native_merge_into(*args, **kwargs) -def create_row_id_update_execution(owner, table, commit_user, commit_identifier, precomputed=None): +def create_python_row_id_update(table, commit_user, commit_identifier, precomputed=None): + """Create an updater after Python execution has already been selected.""" + from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId + return PythonTableUpdateByRowId().initialize(table, commit_user, commit_identifier, precomputed) + + +def create_python_row_id_delete(table, precomputed=None): + """Create a deleter without preparing another execution backend.""" + from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId + return PythonTableDeleteByRowId().initialize(table, precomputed) + + +def create_python_shard_updator(owner, table, projection, columns, commit_user, shard, shard_count): + """Initialize the Python shard resources only after backend selection.""" + from pypaimon.execution.python.write.table_update import ShardTableUpdator + execution = ShardTableUpdator(owner) + execution.initialize(table, projection, columns, commit_user, shard, shard_count) + return execution + + +def create_python_single_writer(owner, writer, partition, first_row_id, row_count): + """Wrap an already selected Python data writer.""" + from pypaimon.execution.python.write.table_update import SingleWriter + execution = SingleWriter(owner) + execution.initialize(writer, partition, first_row_id, row_count) + return execution + + +def create_python_update_operation(context, commit_identifier, predicate=None, *, has_callable=False, has_array=False): + from pypaimon.execution.python.write.table_update import PythonTableUpdate + return PythonTableUpdate(context, commit_identifier, predicate, has_callable=has_callable, has_array=has_array) + + +def create_python_upsert_operation(context, data, upsert_keys, update_columns, *, rows=False): + from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert + return PythonTableUpsert(context, data, upsert_keys, update_columns, rows=rows) + + +def create_python_merge_into_operation( + table, source, *, on, when_matched, when_not_matched, commit_user, commit_identifier): + from pypaimon.execution.python.write.merge_into import PythonTableMergeInto + return PythonTableMergeInto( + table, source, on=on, when_matched=when_matched, when_not_matched=when_not_matched, + commit_user=commit_user, commit_identifier=commit_identifier) + + +def create_shard_update_execution(owner, table, projection, columns, commit_user, shard, shard_count): + """Select the direct shard constructor's backend before capturing files.""" + native = prepare_native_write_operation( + 'shard_update', table, commit_user, projection, columns, shard, shard_count) + if native is not None: + return native + return create_python_shard_updator(owner, table, projection, columns, commit_user, shard, shard_count) + + +def create_row_id_update_execution(table, commit_user, commit_identifier, precomputed=None): """Select one incremental updater before capturing target files.""" if precomputed is not None: from pypaimon.write.update_validation import _authorize_update @@ -187,11 +331,10 @@ def create_row_id_update_execution(owner, table, commit_user, commit_identifier, native = prepare_native_write_operation('update_by_row_id', table, commit_user, commit_identifier) if native is not None: return native - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - return PythonTableUpdateByRowId().initialize(table, commit_user, commit_identifier, precomputed) + return create_python_row_id_update(table, commit_user, commit_identifier, precomputed) -def create_row_id_delete_execution(owner, table, precomputed=None): +def create_row_id_delete_execution(table, precomputed=None): """Keep precomputed broadcast metadata within its selected Python executor.""" if precomputed is not None: from pypaimon.write.update_validation import _authorize_update @@ -203,8 +346,7 @@ def prepare(): native = try_prepare_native(prepare) if native is not None: return native - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - return PythonTableDeleteByRowId().initialize(table, precomputed) + return create_python_row_id_delete(table, precomputed) def python_commit(owner): diff --git a/paimon-python/pypaimon/write/postpone_batch_table_write.py b/paimon-python/pypaimon/write/postpone_batch_table_write.py index eff9fbc058ae..c7034bff879e 100644 --- a/paimon-python/pypaimon/write/postpone_batch_table_write.py +++ b/paimon-python/pypaimon/write/postpone_batch_table_write.py @@ -21,7 +21,7 @@ from pypaimon.write.commit_message import CommitMessage -from pypaimon.write.backend_dispatch import create_write +from pypaimon.write.backend_dispatch import create_write, initialize_write from pypaimon.table.bucket_mode import BucketMode from pypaimon.write.table_write import BatchTableWrite from pypaimon.write.write_builder import BatchWriteBuilder @@ -54,30 +54,26 @@ class PostponeFixedBucketBatchTableWrite(BatchTableWrite): """Batch writer which plans postpone rows before routing new partitions.""" def __init__(self, table, commit_user, static_partition: Optional[dict] = None, bucket_plan=None): - self._python_execution().initialize(table, commit_user, static_partition, bucket_plan) - self.batch_committed = False - - def _python_execution(self): - from pypaimon.execution.python.write.postpone_batch_table_write import PythonPostponeFixedBucketWrite - return PythonPostponeFixedBucketWrite(self) + initialize_write(self, table, commit_user, static_partition, + fixed_bucket=True, bucket_plan=bucket_plan) def write_arrow(self, table: pa.Table, bucket=None): - return self._python_execution().write_arrow(table, bucket) + return self._execution.write_arrow(table, bucket) def write_arrow_batch(self, data: pa.RecordBatch, bucket=None): - return self._python_execution().write_arrow_batch(data, bucket) + return self._execution.write_arrow_batch(data, bucket) def write_row(self, row): - return self._python_execution().write_row(row) + return self._execution.write_row(row) def prepare_commit(self) -> List[CommitMessage]: return super().prepare_commit() def _distributed_write_options(self, *, overwrite=False): - return self._python_execution()._distributed_write_options(overwrite=overwrite) + return self._execution._distributed_write_options(overwrite=overwrite) def close(self): - return self._python_execution().close() + return self._execution.close() def abort(self): - return self._python_execution().abort() + return self._execution.abort() diff --git a/paimon-python/pypaimon/write/table_delete.py b/paimon-python/pypaimon/write/table_delete.py index 827f468a67fb..c00bafaa8143 100644 --- a/paimon-python/pypaimon/write/table_delete.py +++ b/paimon-python/pypaimon/write/table_delete.py @@ -29,15 +29,15 @@ class TableDeleteByRowId: def __init__(self, table, _precomputed_anchor_ranges: Optional['_AnchorRangesInfo'] = None): self.table = table from pypaimon.write.backend_dispatch import create_row_id_delete_execution - self._execution = create_row_id_delete_execution(self, table, _precomputed_anchor_ranges) + self._execution = create_row_id_delete_execution(table, _precomputed_anchor_ranges) @classmethod def _from_python(cls, table, _precomputed_anchor_ranges=None): """Create within an operation that has already selected Python execution.""" owner = cls.__new__(cls) owner.table = table - from pypaimon.execution.python.write.table_delete import PythonTableDeleteByRowId - owner._execution = PythonTableDeleteByRowId().initialize(table, _precomputed_anchor_ranges) + from pypaimon.write.backend_dispatch import create_python_row_id_delete + owner._execution = create_python_row_id_delete(table, _precomputed_anchor_ranges) return owner def delete(self, row_ids: Sequence[int]) -> List[CommitMessage]: diff --git a/paimon-python/pypaimon/write/table_update.py b/paimon-python/pypaimon/write/table_update.py index 7e17467a4bc4..a97863e98909 100644 --- a/paimon-python/pypaimon/write/table_update.py +++ b/paimon-python/pypaimon/write/table_update.py @@ -330,23 +330,18 @@ def __init__(self, table, projection: Optional[List[str]], raise ValueError('Shard index must be smaller than a positive shard count') if write_cols == []: raise ValueError('Shard update columns must not be empty') - from pypaimon.write.backend_dispatch import prepare_native_write_operation - self._execution = prepare_native_write_operation( - 'shard_update', table, commit_user, projection, write_cols, shard_num, total_shard_count) - if self._execution is None: - self._initialize_python(table, projection, write_cols, commit_user, shard_num, total_shard_count) + from pypaimon.write.backend_dispatch import create_shard_update_execution + self._execution = create_shard_update_execution( + self, table, projection, write_cols, commit_user, shard_num, total_shard_count) @classmethod def _from_python(cls, table, projection, write_cols, commit_user, shard_num, total_shard_count): instance = cls.__new__(cls) - instance._initialize_python(table, projection, write_cols, commit_user, shard_num, total_shard_count) + from pypaimon.write.backend_dispatch import create_python_shard_updator + instance._execution = create_python_shard_updator( + instance, table, projection, write_cols, commit_user, shard_num, total_shard_count) return instance - def _initialize_python(self, table, projection, write_cols, commit_user, shard_num, total_shard_count): - from pypaimon.execution.python.write.table_update import ShardTableUpdator - self._execution = ShardTableUpdator(self) - self._execution.initialize(table, projection, write_cols, commit_user, shard_num, total_shard_count) - @staticmethod def compute_from_files(files: List[DataFileMeta]) -> List[Range]: return Range.sort_and_merge_overlap([file.row_id_range() for file in files], True, False) @@ -367,9 +362,8 @@ def update_by_arrow_batch(self, data: pa.RecordBatch): class SingleWriter: def __init__(self, writer: 'DataWriter', partition, first_row_id: int, row_count: int): - from pypaimon.execution.python.write.table_update import SingleWriter - self._execution = SingleWriter(self) - self._execution.initialize(writer, partition, first_row_id, row_count) + from pypaimon.write.backend_dispatch import create_python_single_writer + self._execution = create_python_single_writer(self, writer, partition, first_row_id, row_count) def capacity(self) -> int: return self._execution.capacity() diff --git a/paimon-python/pypaimon/write/table_update_by_row_id.py b/paimon-python/pypaimon/write/table_update_by_row_id.py index 8095aff1fa0f..602cc17686d8 100644 --- a/paimon-python/pypaimon/write/table_update_by_row_id.py +++ b/paimon-python/pypaimon/write/table_update_by_row_id.py @@ -40,7 +40,7 @@ def __init__(self, table, commit_user: str, commit_identifier: int, self.commit_identifier = validate_commit_identifier(commit_identifier) from pypaimon.write.backend_dispatch import create_row_id_update_execution self._execution = create_row_id_update_execution( - self, table, commit_user, self.commit_identifier, _precomputed_files_info) + table, commit_user, self.commit_identifier, _precomputed_files_info) @classmethod def _from_python(cls, table, commit_user, commit_identifier, _precomputed_files_info=None): @@ -49,8 +49,8 @@ def _from_python(cls, table, commit_user, commit_identifier, _precomputed_files_ owner.table = table owner.commit_user = commit_user owner.commit_identifier = validate_commit_identifier(commit_identifier) - from pypaimon.execution.python.write.table_update_by_row_id import PythonTableUpdateByRowId - owner._execution = PythonTableUpdateByRowId().initialize( + from pypaimon.write.backend_dispatch import create_python_row_id_update + owner._execution = create_python_row_id_update( table, commit_user, owner.commit_identifier, _precomputed_files_info) return owner diff --git a/paimon-python/pypaimon/write/table_write.py b/paimon-python/pypaimon/write/table_write.py index 34eb19e36e1e..371900f188ae 100644 --- a/paimon-python/pypaimon/write/table_write.py +++ b/paimon-python/pypaimon/write/table_write.py @@ -19,8 +19,10 @@ import pyarrow as pa -from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER from pypaimon.table.row.blob import BlobConsumer +from pypaimon.write.backend_dispatch import ( + get_write_attribute, initialize_write, prepare_batch_write, prepare_write_commit, write_property, +) from pypaimon.write.commit_identifier import validate_commit_identifier from pypaimon.write.commit_message import CommitMessage from pypaimon.write.row_utils import validate_write_columns @@ -33,41 +35,46 @@ class TableWrite: """Public writer API backed by the selected execution implementation.""" def __init__(self, table, commit_user, static_partition: Optional[dict] = None, *, restore_snapshot_id=None): - return self._python_execution().initialize( - table, commit_user, static_partition, restore_snapshot_id=restore_snapshot_id) + initialize_write(self, table, commit_user, static_partition, + restore_snapshot_id=restore_snapshot_id) + + def __getattr__(self, name): + return get_write_attribute(self, name) - def _python_execution(self): - from pypaimon.execution.python.write.table_write import PythonTableWrite - return PythonTableWrite(self) + file_store_write = write_property('file_store_write') + row_key_extractor = write_property('row_key_extractor') + table_pyarrow_schema = write_property('table_pyarrow_schema') + restore_snapshot_id = write_property('restore_snapshot_id') + batch_committed = write_property('batch_committed') def write_arrow(self, data, bucket: Optional[int] = None): """Write Arrow data, optionally to a bucket assigned upstream.""" - return self._python_execution().write_arrow(data, bucket) + return self._execution.write_arrow(data, bucket) def write_arrow_batch(self, data: pa.RecordBatch, bucket: Optional[int] = None): - return self._python_execution().write_arrow_batch(data, bucket) + return self._execution.write_arrow_batch(data, bucket) def roll_before_group_if_needed(self, row_count: int): """Roll current files before the next logical write group if needed.""" - return self._python_execution().roll_before_group_if_needed(row_count) + return self._execution.roll_before_group_if_needed(row_count) def write_row(self, row): - return self._python_execution().write_row(row) + return self._execution.write_row(row) def write_pandas(self, dataframe): - return self._python_execution().write_pandas(dataframe) + return self._execution.write_pandas(dataframe) def with_write_type(self, write_cols: List[str]): write_cols = validate_write_columns(write_cols) - self._python_execution().with_write_type(write_cols) + self._execution.with_write_type(write_cols) return self def with_blob_consumer(self, blob_consumer: BlobConsumer): - self._python_execution().with_blob_consumer(blob_consumer) + self._execution.with_blob_consumer(blob_consumer) return self def with_blob_uri_reader_factory(self, uri_reader_factory): - self._python_execution().with_blob_uri_reader_factory(uri_reader_factory) + self._execution.with_blob_uri_reader_factory(uri_reader_factory) return self def write_ray( @@ -103,16 +110,16 @@ def write_ray( hash_fixed_precluster, static_partition) def close(self): - return self._python_execution().close() + return self._execution.close() def abort(self): - return self._python_execution().abort() + return self._execution.abort() def _distributed_write_options(self, *, overwrite=False): return {} def _prepare_commit(self, commit_identifier): - return self._python_execution().prepare_commit(commit_identifier) + return prepare_write_commit(self, commit_identifier) class BatchTableWrite(TableWrite): @@ -120,19 +127,15 @@ def __init__(self, table, commit_user, static_partition: Optional[dict] = None, *, restore_snapshot_id=None): super().__init__(table, commit_user, static_partition, restore_snapshot_id=restore_snapshot_id) - self.batch_committed = False def prepare_commit(self) -> List[CommitMessage]: - if self.batch_committed: - raise RuntimeError("BatchTableWrite only supports one-time committing.") - self.batch_committed = True - return self._prepare_commit(BATCH_COMMIT_IDENTIFIER) + return prepare_batch_write(self) class StreamTableWrite(TableWrite): def __init__(self, table, commit_user, *, restore_snapshot_id=None): - super().__init__(table, commit_user, None, + initialize_write(self, table, commit_user, stream=True, restore_snapshot_id=restore_snapshot_id) def prepare_commit(self, commit_identifier) -> List[CommitMessage]: diff --git a/paimon-python/pypaimon/write/update_dispatch.py b/paimon-python/pypaimon/write/update_dispatch.py index b1c861521494..e53afdca1c40 100644 --- a/paimon-python/pypaimon/write/update_dispatch.py +++ b/paimon-python/pypaimon/write/update_dispatch.py @@ -22,7 +22,10 @@ import pyarrow as pa from pypaimon.common.predicate import Predicate -from pypaimon.write.backend_dispatch import prepare_native_write_operation +from pypaimon.write.backend_dispatch import ( + create_python_merge_into_operation, create_python_update_operation, create_python_upsert_operation, + prepare_native_write_operation, +) from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER from pypaimon.table.special_fields import SpecialFields from pypaimon.write.commit_message import CommitMessage @@ -69,8 +72,9 @@ def new_shard_updator(self, shard_num: int, total_shard_count: int): StreamTableUpdate)) if native is not None: return native - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(context)._new_python_shard_updator(shard_num, total_shard_count) + from pypaimon.write.table_update import ShardTableUpdator + return ShardTableUpdator._from_python( + context.table, context.projection, context.update_cols, context.commit_user, shard_num, total_shard_count) def _update_by_arrow_with_row_id(self, table: pa.Table, commit_identifier: int) -> List[CommitMessage]: context = self._context @@ -80,16 +84,9 @@ def _update_by_arrow_with_row_id(self, table: pa.Table, commit_identifier: int) if table.num_rows and cols and (SpecialFields.ROW_ID.name in table.column_names): operation = prepare_native_write_operation('update', context.table, context.commit_user, cols) if operation is None: - from pypaimon.execution.python.write.table_update import PythonTableUpdate - operation = PythonTableUpdate(context, commit_identifier) + operation = create_python_update_operation(context, commit_identifier) return operation.update_by_arrow_with_row_id(table) - def _update_by_arrow_batches_with_row_id( - self, tables: Iterable[pa.Table], commit_identifier: int) -> List[CommitMessage]: - context = self._context - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(context)._update_by_arrow_batches_with_row_id(tables, commit_identifier) - def _upsert_by_arrow_with_key( self, table: pa.Table, upsert_keys: List[str], commit_identifier: int) -> List[CommitMessage]: """Shared implementation for ``upsert_by_arrow_with_key``. @@ -150,17 +147,15 @@ def _update_by_predicate( predicate, list(assignments)) if operation is None: - from pypaimon.execution.python.write.table_update import PythonTableUpdate - operation = PythonTableUpdate(context, commit_identifier, predicate, - has_callable=has_callable, has_array=has_array) + operation = create_python_update_operation( + context, commit_identifier, predicate, has_callable=has_callable, has_array=has_array) return operation.update(assignments, read_columns) def _delete_by_predicate(self, predicate: Optional[Predicate], commit_identifier: int) -> List[CommitMessage]: context = self._context operation = prepare_native_write_operation('predicate_delete', context.table, context.commit_user, predicate) if operation is None: - from pypaimon.execution.python.write.table_update import PythonTableUpdate - operation = PythonTableUpdate(context, commit_identifier, predicate) + operation = create_python_update_operation(context, commit_identifier, predicate) return operation.delete() def _delete_by_row_id(self, row_ids: Sequence[int], commit_identifier: int) -> List[CommitMessage]: @@ -169,15 +164,9 @@ def _delete_by_row_id(self, row_ids: Sequence[int], commit_identifier: int) -> L if len(row_ids): operation = prepare_native_write_operation('delete', context.table, context.commit_user) if operation is None: - from pypaimon.execution.python.write.table_update import PythonTableUpdate - operation = PythonTableUpdate(context, commit_identifier) + operation = create_python_update_operation(context, commit_identifier) return operation.delete_by_row_id(row_ids) - def _matched_delete_row_ids(self, predicate: Optional[Predicate]) -> List[int]: - context = self._context - from pypaimon.execution.python.write.table_update import PythonTableUpdate - return PythonTableUpdate(context)._matched_delete_row_ids(predicate) - def update_by_arrow_batches_with_row_id(self, tables: Iterable[pa.Table]) -> List[CommitMessage]: """Apply row-id updates from batches using one target-file index. @@ -188,8 +177,7 @@ def update_by_arrow_batches_with_row_id(self, tables: Iterable[pa.Table]) -> Lis context = self._context operation = prepare_native_write_operation('update', context.table, context.commit_user, context.update_cols) if operation is None: - from pypaimon.execution.python.write.table_update import PythonTableUpdate - operation = PythonTableUpdate(context) + operation = create_python_update_operation(context, BATCH_COMMIT_IDENTIFIER) return operation.update_by_arrow_batches_with_row_id(tables) @@ -234,8 +222,7 @@ def upsert(self, data: pa.Table, upsert_keys: List[str], columns = list(effective_update_cols) if effective_update_cols else list(context.table.field_names) operation = context._create_native_upsert(data, upsert_keys, columns) if operation is None: - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - operation = PythonTableUpsert(context, data, upsert_keys, effective_update_cols) + operation = create_python_upsert_operation(context, data, upsert_keys, effective_update_cols) return operation.upsert() def upsert_rows(self, rows, upsert_keys: List[str], update_cols: Optional[List[str]] = None) -> List[CommitMessage]: @@ -253,8 +240,8 @@ def upsert_rows(self, rows, upsert_keys: List[str], update_cols: Optional[List[s columns = list(effective_update_cols) if effective_update_cols is not None else list(context.table.field_names) operation = context._create_native_upsert([values for _, values in row_items], upsert_keys, columns) if operation is None: - from pypaimon.execution.python.write.table_upsert_by_key import PythonTableUpsert - operation = PythonTableUpsert(context, row_items, upsert_keys, effective_update_cols, rows=True) + operation = create_python_upsert_operation( + context, row_items, upsert_keys, effective_update_cols, rows=True) return operation.upsert() def _create_native_upsert(self, data, upsert_keys, columns): @@ -276,9 +263,7 @@ def merge_into( if native is not None: return native.prepare_commit() - from pypaimon.execution.python.write.merge_into import PythonTableMergeInto - - return PythonTableMergeInto( + return create_python_merge_into_operation( target_table, source, on=on, when_matched=when_matched, when_not_matched=when_not_matched, commit_user=commit_user, commit_identifier=commit_identifier, diff --git a/paimon-python/pypaimon/write/write_dispatch.py b/paimon-python/pypaimon/write/write_dispatch.py index fb0d2910caf3..536939840dd3 100644 --- a/paimon-python/pypaimon/write/write_dispatch.py +++ b/paimon-python/pypaimon/write/write_dispatch.py @@ -43,6 +43,7 @@ def __init__(self, table, commit_user, static_partition, stream, native_writer, self._execution = NativeWrite(table, commit_user, static_partition, stream, native_writer, restore_snapshot_id) self._python_writer = None + self._public_writer = self @classmethod def from_native(cls, native): @@ -57,6 +58,7 @@ def from_native(cls, native): result = object.__new__(selected) result._execution = native result._python_writer = None + result._public_writer = result return result def __getattr__(self, name): @@ -81,7 +83,7 @@ def _new_python_writer(self): writer = create_python_write( self.table, self.commit_user, self.static_partition, self.stream, - restore_snapshot_id=self._restore_snapshot_id) + restore_snapshot_id=self._restore_snapshot_id, public_writer=self._public_writer) return self._configure_python_writer(writer) def _configure_python_writer(self, writer): @@ -178,9 +180,9 @@ def write_arrow(self, data, bucket=None): return self._python_writer.write_arrow(data, bucket) data = self._execution._prepare_native_arrow_data(data) if isinstance(data, pa.RecordBatch): - return self.write_arrow_batch(data, bucket) + return self._public_writer.write_arrow_batch(data, bucket) for batch in data.to_batches(): - self.write_arrow_batch(batch, bucket) + self._public_writer.write_arrow_batch(batch, bucket) def write_arrow_batch(self, data, bucket=None): execution = self._python_writer if self._python_writer is not None else self._execution @@ -191,7 +193,7 @@ def write_pandas(self, dataframe): return self._python_writer.write_pandas(dataframe) from pypaimon.write.input_adapter import _pandas_write_batch - return self.write_arrow_batch(_pandas_write_batch( + return self._public_writer.write_arrow_batch(_pandas_write_batch( self.table, dataframe, self._schema, self._table_schema)) def write_row(self, row): @@ -258,15 +260,18 @@ def __init__(self, table, commit_user, static_partition, bucket_plan, native_wri self._execution = NativePostponeWrite( table, commit_user, static_partition, bucket_plan, native_writer) self._python_writer = None + self._public_writer = self def _new_python_writer(self): from pypaimon.write.backend_dispatch import create_python_write return self._configure_python_writer(create_python_write( self.table, self.commit_user, self.static_partition, - fixed_bucket=True, bucket_plan=self._bucket_plan)) + fixed_bucket=True, bucket_plan=self._bucket_plan, public_writer=self._public_writer)) def _distributed_write_options(self, *, overwrite=False): + if self._python_writer is not None: + return self._python_writer._distributed_write_options(overwrite=overwrite) from pypaimon.write.input_adapter import postpone_distributed_write_options return postpone_distributed_write_options(self.table, self._bucket_plan, overwrite) From 405a67d0fb38184307bde8afa9ae48c8b986f78f Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 22 Oct 2026 21:14:03 +0800 Subject: [PATCH 5/6] [Python] Keep direct writer APIs independent of execution backend --- .../write/direct_writer_execution_test.py | 39 +++++++++++++++++++ .../pypaimon/write/backend_dispatch.py | 6 ++- paimon-python/pypaimon/write/table_write.py | 8 ++-- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/paimon-python/pypaimon/tests/write/direct_writer_execution_test.py b/paimon-python/pypaimon/tests/write/direct_writer_execution_test.py index f66c2c4807aa..48378cd1ee11 100644 --- a/paimon-python/pypaimon/tests/write/direct_writer_execution_test.py +++ b/paimon-python/pypaimon/tests/write/direct_writer_execution_test.py @@ -18,6 +18,7 @@ """Direct public writer construction shares the builder execution boundary.""" from contextlib import ExitStack, contextmanager +import inspect from pathlib import Path import subprocess import sys @@ -134,6 +135,44 @@ def test_direct_writer_metadata_inspection_does_not_select_python(tmp_path, writ writer.close() +@pytest.mark.parametrize('writer_class', WRITERS) +@pytest.mark.parametrize('native', [False, True]) +def test_direct_writer_public_api_is_backend_independent(tmp_path, writer_class, native): + table, _ = _table(tmp_path, writer_class) + schema = PyarrowFieldParser.from_paimon_schema(table.fields) + with ExitStack() as stack: + if native: + stack.enter_context(_native_backend()) + else: + stack.enter_context(patch( + 'pypaimon.execution.native.write.native_write.create_native_write', return_value=None)) + writer = _construct(writer_class, table) + try: + assert type(writer) is writer_class + assert writer.table is table + assert writer.commit_user == 'direct-writer' + assert writer.static_partition is None + assert writer.table_pyarrow_schema == schema + assert writer.restore_snapshot_id is None + if isinstance(writer, BatchTableWrite): + assert writer.batch_committed is False + else: + assert not hasattr(writer, 'batch_committed') + if writer_class is TableWrite: + # The base API intentionally leaves preparation to its subclass. + # Discovering optional methods must not leak the selected executor. + assert not hasattr(writer, 'prepare_commit') + else: + parameters = list(inspect.signature(writer.prepare_commit).parameters.values()) + if writer_class is StreamTableWrite: + assert [parameter.name for parameter in parameters] == ['commit_identifier'] + assert parameters[0].default is inspect.Parameter.empty + else: + assert not parameters + finally: + writer.close() + + @pytest.mark.parametrize('writer_class', WRITERS) def test_direct_writer_native_write_failure_is_not_replayed(tmp_path, writer_class): table, schema = _table(tmp_path, writer_class) diff --git a/paimon-python/pypaimon/write/backend_dispatch.py b/paimon-python/pypaimon/write/backend_dispatch.py index cfd039adc27e..0490bc106e00 100644 --- a/paimon-python/pypaimon/write/backend_dispatch.py +++ b/paimon-python/pypaimon/write/backend_dispatch.py @@ -81,7 +81,8 @@ def _initialize_python_write(owner, table, commit_user, static_partition, stream execution.initialize(table, commit_user, static_partition, restore_snapshot_id=restore_snapshot_id) owner._execution = execution - if not stream: + from pypaimon.write.table_write import BatchTableWrite + if isinstance(owner, BatchTableWrite): owner.__dict__['batch_committed'] = False @@ -110,6 +111,9 @@ def initialize_write(owner, table, commit_user, static_partition=None, stream=Fa _initialize_python_write(owner, table, commit_user, static_partition, stream, restore_snapshot_id, fixed_bucket, bucket_plan) else: + owner.table = table + owner.commit_user = commit_user + owner.static_partition = static_partition execution._public_writer = owner owner._execution = execution diff --git a/paimon-python/pypaimon/write/table_write.py b/paimon-python/pypaimon/write/table_write.py index 371900f188ae..b9e4fa38cd3e 100644 --- a/paimon-python/pypaimon/write/table_write.py +++ b/paimon-python/pypaimon/write/table_write.py @@ -21,7 +21,7 @@ from pypaimon.table.row.blob import BlobConsumer from pypaimon.write.backend_dispatch import ( - get_write_attribute, initialize_write, prepare_batch_write, prepare_write_commit, write_property, + initialize_write, prepare_batch_write, prepare_write_commit, write_property, ) from pypaimon.write.commit_identifier import validate_commit_identifier from pypaimon.write.commit_message import CommitMessage @@ -38,14 +38,10 @@ def __init__(self, table, commit_user, static_partition: Optional[dict] = None, initialize_write(self, table, commit_user, static_partition, restore_snapshot_id=restore_snapshot_id) - def __getattr__(self, name): - return get_write_attribute(self, name) - file_store_write = write_property('file_store_write') row_key_extractor = write_property('row_key_extractor') table_pyarrow_schema = write_property('table_pyarrow_schema') restore_snapshot_id = write_property('restore_snapshot_id') - batch_committed = write_property('batch_committed') def write_arrow(self, data, bucket: Optional[int] = None): """Write Arrow data, optionally to a bucket assigned upstream.""" @@ -123,6 +119,8 @@ def _prepare_commit(self, commit_identifier): class BatchTableWrite(TableWrite): + batch_committed = write_property('batch_committed') + def __init__(self, table, commit_user, static_partition: Optional[dict] = None, *, restore_snapshot_id=None): super().__init__(table, commit_user, static_partition, From 94cc3a2c5b466bd12688047f67c67c6d122598d9 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 22 Oct 2026 22:20:48 +0800 Subject: [PATCH 6/6] [Python] Fix single-bucket routing and authorization dispatch regressions --- .../execution/python/write/table_write.py | 2 +- .../tests/e2e/java_py_read_write_test.py | 1 - .../pypaimon/tests/multimodal_lerobot_test.py | 26 +++--- .../tests/native_append_index_search_test.py | 19 ++++- .../tests/native_authorized_update_test.py | 80 +++++++++++++++++++ .../tests/native_blob_consumer_test.py | 29 +++++-- .../tests/native_pk_full_text_search_test.py | 3 +- .../tests/native_stream_table_scan_test.py | 3 +- .../tests/native_temporal_zone_cast_test.py | 40 +++++++++- .../tests/native_write_fork_safety_test.py | 66 +++++++++++---- .../pypaimon/tests/table_update_test.py | 4 + .../tests/write/append_row_kind_test.py | 38 +++++++++ .../write/row_id_execution_boundary_test.py | 3 +- .../write/update_backend_selection_test.py | 4 +- .../pypaimon/write/backend_dispatch.py | 15 ++-- .../pypaimon/write/update_dispatch.py | 1 + 16 files changed, 283 insertions(+), 51 deletions(-) diff --git a/paimon-python/pypaimon/execution/python/write/table_write.py b/paimon-python/pypaimon/execution/python/write/table_write.py index 36fc14855540..ac0d892000ce 100644 --- a/paimon-python/pypaimon/execution/python/write/table_write.py +++ b/paimon-python/pypaimon/execution/python/write/table_write.py @@ -343,7 +343,7 @@ def _project_write_input(data, columns): def _write_routing_columns(self): table = self.writer.table routing = set(table.partition_keys) - if table.options.bucket() > 1: + if table.options.bucket() > 0: routing.update(table.table_schema.bucket_keys) row_kind = table.options.options.to_map().get('rowkind.field') if row_kind is not None: diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py index 439647a0c8e5..505fa2e00126 100644 --- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py +++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py @@ -1909,7 +1909,6 @@ def test_compact_conflict_shard_update(self): update.with_read_projection(['f0']) update.with_update_type(['f2']) upd = update.new_shard_updator(shard_num=0, total_shard_count=3) - print(f"Shard 0 row_ranges: {[(r[1].from_, r[1].to) for r in upd.row_ranges]}") reader = upd.arrow_reader() import pyarrow as pa diff --git a/paimon-python/pypaimon/tests/multimodal_lerobot_test.py b/paimon-python/pypaimon/tests/multimodal_lerobot_test.py index 7d48264a3a88..1aea33b2db8a 100644 --- a/paimon-python/pypaimon/tests/multimodal_lerobot_test.py +++ b/paimon-python/pypaimon/tests/multimodal_lerobot_test.py @@ -4061,10 +4061,7 @@ def counted_plan(scan): "pypaimon.multimodal.blob_read.fetch_blob_bodies", wraps=fetch_blob_bodies) as fetch: last, first = dataset.__getitems__([4, 0]) - scanner = dataset._frame_locator._scanner - dataset._frame_locator.locate([2]) self.assertEqual(0, len(scan_calls)) - self.assertIs(scanner, dataset._frame_locator._scanner) self.assertEqual(1, read.call_count) self.assertEqual([0, 1, 3, 4], read.call_args.args[0]) self.assertEqual(1, fetch.call_count) @@ -4085,15 +4082,13 @@ def counted_plan(scan): self.assertEqual([True, False, False], first["action_is_pad"].tolist()) - restored_locator = pickle.loads(pickle.dumps( - dataset._frame_locator)) - with patch.object(TableScan, "plan", new=counted_plan): - restored_splits, restored_needs_filter = \ - restored_locator.locate([2]) - self.assertEqual(0, len(scan_calls)) - self.assertTrue(restored_splits) - self.assertFalse(restored_needs_filter) - restored_locator.close() + restored_reader = pickle.loads(pickle.dumps(dataset.reader)) + try: + # Both execution backends preserve indexed lookup after transport. + self.assertEqual( + [2], restored_reader.read_indices([2], ["index"])["index"].to_pylist()) + finally: + restored_reader.close() worker_indices = [] for batch in DataLoader( @@ -4176,8 +4171,13 @@ def counted_plan(scan): def test_paimon_dataset_rejects_unavailable_frame_index(self): self.connection.load_from_lerobot( "missing_frame_index", self.image_source) + # Exercise the Python locator's failures explicitly; Native lookup + # does not open this scanner or invoke its internal hooks. dataset = pmm.PaimonLeRobotDataset( - self.connection.get_table("missing_frame_index")) + self.connection.get_table("missing_frame_index").copy({ + "read.native.enabled": "false", + "scan.native-plan.enabled": "false", + })) with patch.object( dataset._frame_locator, "_index_scanner", diff --git a/paimon-python/pypaimon/tests/native_append_index_search_test.py b/paimon-python/pypaimon/tests/native_append_index_search_test.py index 249cda690a2a..48387089d5a7 100644 --- a/paimon-python/pypaimon/tests/native_append_index_search_test.py +++ b/paimon-python/pypaimon/tests/native_append_index_search_test.py @@ -130,8 +130,21 @@ def test_exact_scalar_index_filters_work_on_append_vector_routes(rest_catalog, f @pytest.mark.parametrize('family', ['full-text', 'hybrid-text', 'hybrid-mixed']) -@pytest.mark.parametrize('native', [False, True]) -def test_append_full_text_row_filters_remain_unsupported(rest_catalog, family, native): - table = _table(rest_catalog[0], True).copy({'read.native.enabled': str(native).lower()}) +def test_python_append_full_text_row_filters_remain_unsupported(rest_catalog, family): + table = _table(rest_catalog[0], True) with pytest.raises(Exception, match='[Ff]ull-text row filters require a data-evolution table'): _builder(table, family, row_filter=True).execute_local() + + +@pytest.mark.parametrize('family', ['full-text', 'hybrid-text', 'hybrid-mixed']) +@pytest.mark.parametrize('partition', [False, True]) +def test_native_append_full_text_routes_apply_scalar_filters(rest_catalog, family, partition): + table = _table(rest_catalog[0], True).copy({'read.native.enabled': 'true'}) + with _native_only(): + actual = _scores(_builder(table, family, partition, row_filter=True).execute_local()) + wanted = {3} if partition else {0, 3} + if family == 'hybrid-mixed': + wanted = {2, 3} if partition else {0, 2, 3} + assert set(actual) == wanted + # A matching text document still has to satisfy the scalar predicate. + assert 1 not in actual diff --git a/paimon-python/pypaimon/tests/native_authorized_update_test.py b/paimon-python/pypaimon/tests/native_authorized_update_test.py index 366990c3793b..aaba80f3bc21 100644 --- a/paimon-python/pypaimon/tests/native_authorized_update_test.py +++ b/paimon-python/pypaimon/tests/native_authorized_update_test.py @@ -29,6 +29,7 @@ from pypaimon import Schema from pypaimon.api.api_response import AuthTableQueryResponse, ErrorResponse +from pypaimon.catalog.catalog_exception import TableNoPermissionException from pypaimon.execution.native.runtime import (native_runtime_available) from pypaimon.table.data_evolution_merge_into import WhenMatched, WhenNotMatched, source_col from pypaimon.table.row.generic_row import GenericRow @@ -256,6 +257,85 @@ def _restricted(server, table, policy): return patch.object(server, '_table_query_auth_handle', wraps=server._table_query_auth_handle) +def _new_update_executor(table, entrypoint): + from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER + from pypaimon.write.table_delete import TableDeleteByRowId + from pypaimon.write.table_update import ShardTableUpdator + from pypaimon.write.table_update_by_row_id import TableUpdateByRowId + + builder = table.new_stream_write_builder() if entrypoint == 'stream' else table.new_batch_write_builder() + if entrypoint == 'direct-update': + return TableUpdateByRowId(table, builder.commit_user, BATCH_COMMIT_IDENTIFIER) + if entrypoint == 'direct-delete': + return TableDeleteByRowId(table) + if entrypoint == 'direct-shard': + return ShardTableUpdator(table, ['id'], ['value'], builder.commit_user, 0, 1) + update = builder.new_update().with_update_type(['value']) + update.with_read_projection(['id']) + return update.new_shard_updator(0, 1) + + +@pytest.mark.parametrize('entrypoint', ['batch', 'stream', 'direct-shard', 'direct-update', 'direct-delete']) +@pytest.mark.parametrize('policy', ['denied', 'filter', 'mask']) +def test_update_authorization_refusal_precedes_backend_selection(rest_catalog, entrypoint, policy): + catalog, server = rest_catalog + table = _table(catalog, True, seed=False) + before = _files(table) + with ExitStack() as stack: + authorize = stack.enter_context(_restricted(server, table, policy)) + prepare = [stack.enter_context(patch(path, side_effect=AssertionError( + 'Backend prepared before authorization: ' + path))) for path in ( + 'pypaimon.write.update_dispatch.prepare_native_write_operation', + 'pypaimon.write.backend_dispatch.prepare_native_write_operation', + 'pypaimon.execution.native.write.native_update.create_native_row_id_delete', + 'pypaimon.write.backend_dispatch.create_python_shard_updator', + 'pypaimon.write.backend_dispatch.create_python_row_id_update', + 'pypaimon.write.backend_dispatch.create_python_row_id_delete')] + snapshot = stack.enter_context(patch.object(table, 'snapshot_manager', + side_effect=AssertionError('Unauthorized snapshot read'))) + with pytest.raises(TableNoPermissionException): + _new_update_executor(table, entrypoint) + for factory in prepare: + factory.assert_not_called() + snapshot.assert_not_called() + authorize.assert_called_once() + assert json.loads(authorize.call_args.args[1]).get('select') is None + assert _files(table) == before + + +@pytest.mark.parametrize('entrypoint,native_factory', [ + ('batch', 'create_native_shard_updator'), ('stream', 'create_native_shard_updator'), + ('direct-shard', 'create_native_shard_updator'), ('direct-update', 'create_native_update_by_row_id'), + ('direct-delete', 'create_native_row_id_delete')]) +def test_native_preparation_permission_refusal_is_not_retried_in_python(rest_catalog, entrypoint, native_factory): + catalog, server = rest_catalog + table = _table(catalog, True, seed=False) + failure = TableNoPermissionException(table.identifier) + before = _files(table) + with ExitStack() as stack: + authorize = stack.enter_context(patch.object( + server, '_table_query_auth_handle', wraps=server._table_query_auth_handle)) + native = stack.enter_context(patch( + 'pypaimon.execution.native.write.native_update.' + native_factory, side_effect=failure)) + python = [stack.enter_context(patch( + 'pypaimon.write.backend_dispatch.' + name, + side_effect=AssertionError('Permission refusal selected Python'))) + for name in ('create_python_shard_updator', 'create_python_row_id_update', + 'create_python_row_id_delete')] + snapshot = stack.enter_context(patch.object(table, 'snapshot_manager', + side_effect=AssertionError('Unauthorized snapshot read'))) + with pytest.raises(TableNoPermissionException) as raised: + _new_update_executor(table, entrypoint) + assert raised.value is failure + native.assert_called_once() + for factory in python: + factory.assert_not_called() + snapshot.assert_not_called() + authorize.assert_called_once() + assert json.loads(authorize.call_args.args[1]).get('select') is None + assert _files(table) == before + + @pytest.mark.parametrize('use_native', _ENGINES) @pytest.mark.parametrize('policy', ['denied', 'filter', 'mask']) @pytest.mark.parametrize('operation', _OPERATIONS) diff --git a/paimon-python/pypaimon/tests/native_blob_consumer_test.py b/paimon-python/pypaimon/tests/native_blob_consumer_test.py index 018e91f80909..f7517ac66fb6 100644 --- a/paimon-python/pypaimon/tests/native_blob_consumer_test.py +++ b/paimon-python/pypaimon/tests/native_blob_consumer_test.py @@ -224,20 +224,37 @@ def test_blob_consumer_flush_makes_local_payload_visible_before_prepare(tmp_path writer.close() -def test_blob_consumer_can_be_replaced_cleared_and_cannot_change_after_write(tmp_path): - table = _table(tmp_path) - writer = table.new_batch_write_builder().new_write() +@pytest.mark.parametrize('native', [False, True]) +@pytest.mark.parametrize('stream', [False, True]) +def test_blob_consumer_can_be_replaced_and_cleared_before_and_after_write(tmp_path, native, stream): + table = _table(tmp_path, {'write.native.enabled': str(native).lower()}) + builder = table.new_stream_write_builder() if stream else table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + received = [] try: + if native: + assert isinstance(writer, NativeTableWrite) writer.with_blob_consumer(lambda name, descriptor: pytest.fail('replaced consumer invoked')) with pytest.raises(TypeError, match='callable'): writer.with_blob_consumer(123) writer.with_blob_consumer(None) writer.write_arrow(_data()) - with pytest.raises(RuntimeError, match='before any write'): - writer.with_blob_consumer(None) - assert writer._python_writer is None + assert writer.with_blob_consumer(lambda name, descriptor: received.append((name, descriptor))) is writer + writer.write_arrow(_data(2)) + # Java updates the factory setting; the existing bucket retains the + # cleared consumer it captured when its writer was first created. + assert not received + assert writer.with_blob_consumer(None) is writer + if stream: + commit.commit(writer.prepare_commit(1), 1) + else: + commit.commit(writer.prepare_commit()) + assert _read(table, True, True) == _data().to_pylist() + _data(2).to_pylist() + if native: + assert writer._python_writer is None finally: writer.close() + commit.close() def test_blob_consumer_survives_switch_to_row_writer(tmp_path): diff --git a/paimon-python/pypaimon/tests/native_pk_full_text_search_test.py b/paimon-python/pypaimon/tests/native_pk_full_text_search_test.py index 288600e147c7..0d2e1adbc134 100644 --- a/paimon-python/pypaimon/tests/native_pk_full_text_search_test.py +++ b/paimon-python/pypaimon/tests/native_pk_full_text_search_test.py @@ -155,7 +155,8 @@ def test_pk_plan_validates_source_family_and_reader_requirements(source, case): elif case == 'filter': reader = _native(table) reader.with_filter(_predicate_to_native(table.new_read_builder().new_predicate_builder().equal('id', 1))) - with pytest.raises(Exception, match='non-partition filters'): + with pytest.raises(NotImplementedError, + match='full-text row filters require a row-tracked Parquet append table'): reader.new_full_text_read() elif case == 'kind': changed = table.copy({'pk-full-text.index.columns': ''}) diff --git a/paimon-python/pypaimon/tests/native_stream_table_scan_test.py b/paimon-python/pypaimon/tests/native_stream_table_scan_test.py index 1e3b7713e380..a1274bfc38a7 100644 --- a/paimon-python/pypaimon/tests/native_stream_table_scan_test.py +++ b/paimon-python/pypaimon/tests/native_stream_table_scan_test.py @@ -157,7 +157,8 @@ def test_bucket_filter_is_applied_to_every_streaming_frame(tmp_path, mode, selec python_plan, expected = _frame(table, snapshot, mode, False, callback, predicate=predicate) native_plan, actual = _frame(table, snapshot, mode, True, callback, predicate=predicate) assert actual == expected - splits = python_plan.splits() + (native_plan.splits() if native_plan is not None else []) + splits = ((python_plan.splits() if python_plan is not None else []) + + (native_plan.splits() if native_plan is not None else [])) assert all(split.bucket in selected for split in splits) if not selected: assert not actual and (native_plan is None or not native_plan.splits()) diff --git a/paimon-python/pypaimon/tests/native_temporal_zone_cast_test.py b/paimon-python/pypaimon/tests/native_temporal_zone_cast_test.py index 752fc45de0ab..7a3eaabda309 100644 --- a/paimon-python/pypaimon/tests/native_temporal_zone_cast_test.py +++ b/paimon-python/pypaimon/tests/native_temporal_zone_cast_test.py @@ -18,6 +18,7 @@ """Existing temporal changes read natively from old REST Parquet files.""" import os +import time import pyarrow as pa import pytest @@ -81,12 +82,49 @@ def _zone(): # Run non-UTC cases in separate processes so Native's cached system zone # and Python's libc zone are initialized under the same environment. - name = os.environ.get('TZ', 'UTC') + name = os.environ.get('TZ') + if name is None: + # TZ may be unset on a non-UTC host. The standard Linux/macOS link + # retains the IANA name; abbreviations such as CST are ambiguous. + name = os.path.realpath('/etc/localtime').partition('/zoneinfo/')[2] + if not name: + # GMT can also name London's winter offset, not a UTC zone. + name = 'UTC' if time.tzname[0] == 'UTC' else None + if name in ('Etc/UTC', 'Etc/GMT', 'GMT'): + name = 'UTC' if name not in _JAVA_CASTS: pytest.skip('Java golden values are defined for UTC, New York and Shanghai') return name +@pytest.mark.parametrize('system_zone,expected', [ + ('Asia/Shanghai', 'Asia/Shanghai'), + ('America/New_York', 'America/New_York'), + ('Etc/UTC', 'UTC'), +]) +def test_zone_goldens_resolve_unset_tz_from_system_iana_name(monkeypatch, system_zone, expected): + monkeypatch.delenv('TZ', raising=False) + monkeypatch.setattr(os.path, 'realpath', lambda path: '/usr/share/zoneinfo/' + system_zone) + monkeypatch.setattr(time, 'tzname', ('CST', 'CST')) + assert _zone() == expected + + +@pytest.mark.parametrize('zone,expected', [('America/New_York', 'America/New_York'), ('GMT', 'UTC')]) +def test_zone_goldens_prefer_explicit_process_tz(monkeypatch, zone, expected): + monkeypatch.setenv('TZ', zone) + monkeypatch.setattr(os.path, 'realpath', lambda path: '/usr/share/zoneinfo/Asia/Shanghai') + assert _zone() == expected + + +@pytest.mark.parametrize('abbreviation', ['CST', 'GMT']) +def test_zone_goldens_do_not_assume_utc_for_ambiguous_system_abbreviation(monkeypatch, abbreviation): + monkeypatch.delenv('TZ', raising=False) + monkeypatch.setattr(os.path, 'realpath', lambda path: path) + monkeypatch.setattr(time, 'tzname', (abbreviation, abbreviation)) + with pytest.raises(pytest.skip.Exception): + _zone() + + def _case(operation): date_values = {'UTC': [0, -86400000, 1717200000000, 1325203200000, None], 'Asia/Shanghai': [-28800000, -115200000, 1717171200000, 1325174400000, None], diff --git a/paimon-python/pypaimon/tests/native_write_fork_safety_test.py b/paimon-python/pypaimon/tests/native_write_fork_safety_test.py index 32398fa8c3b5..b9e16a77e5ee 100644 --- a/paimon-python/pypaimon/tests/native_write_fork_safety_test.py +++ b/paimon-python/pypaimon/tests/native_write_fork_safety_test.py @@ -31,7 +31,7 @@ pytestmark = [pytest.mark.python_write, pytest.mark.python_commit] _OPERATIONS = ['batch-write', 'stream-write', 'commit', 'upsert', 'row-id-updater', 'update', 'batch-update', - 'predicate-update', 'delete', 'predicate-delete'] + 'predicate-update', 'delete', 'predicate-delete', 'direct-row-id-updater', 'direct-row-id-delete'] class _ForkSafetyError(RuntimeError): @@ -52,15 +52,14 @@ def _operation(tmp_path, operation): if operation.endswith('write'): streaming = operation == 'stream-write' builder = table.new_stream_write_builder() if streaming else table.new_batch_write_builder() - name = 'StreamTableWrite' if streaming else 'BatchTableWrite' - context.enter_context(patch('pypaimon.write.table_write.' + name, pure)) + context.enter_context(patch('pypaimon.write.backend_dispatch.create_python_write', pure)) factory = 'create_native_write' module = 'pypaimon.execution.native.write.native_write.' invoke = builder.new_write elif operation == 'commit': committer = table.new_batch_write_builder().new_commit() context.callback(committer.close) - context.enter_context(patch.object(committer.file_store_commit, 'commit', pure)) + context.enter_context(patch('pypaimon.execution.python.write.table_commit.create_python_commit', pure)) context.enter_context(patch( 'pypaimon.execution.native.write.native_commit.native_messages_supported', return_value=True)) message = Mock() @@ -71,31 +70,40 @@ def _operation(tmp_path, operation): module = 'pypaimon.execution.native.write.native_update.' if operation == 'upsert': upsert = TableUpsertByKey(table, 'test', 7) - context.enter_context(patch.object(upsert, '_group_by_partition', pure)) + context.enter_context(patch('pypaimon.write.update_dispatch.create_python_upsert_operation', pure)) factory = 'create_native_upsert' invoke = lambda: upsert.upsert(data, ['id']) - elif operation == 'row-id-updater': - context.enter_context(patch('pypaimon.write.update_dispatch.TableUpdateByRowId', pure)) - factory, invoke = 'create_native_update_by_row_id', updater.new_update_by_row_id + elif operation in ('row-id-updater', 'direct-row-id-updater'): + from pypaimon.write.table_update_by_row_id import TableUpdateByRowId + + context.enter_context(patch('pypaimon.write.backend_dispatch.create_python_row_id_update', pure)) + factory = 'create_native_update_by_row_id' + invoke = (updater.new_update_by_row_id if operation == 'row-id-updater' else + lambda: TableUpdateByRowId(table, 'test', 7)) + elif operation == 'direct-row-id-delete': + from pypaimon.write.table_delete import TableDeleteByRowId + + context.enter_context(patch('pypaimon.write.backend_dispatch.create_python_row_id_delete', pure)) + factory = 'create_native_row_id_delete' + invoke = lambda: TableDeleteByRowId(table) elif operation in ('update', 'batch-update'): data = data.append_column('_ROW_ID', pa.array([0], pa.int64())) + context.enter_context(patch('pypaimon.write.update_dispatch.create_python_update_operation', pure)) if operation == 'update': - context.enter_context(patch.object(updater, '_new_row_id_updater', pure)) invoke = lambda: updater.update_by_arrow_with_row_id(data) else: - context.enter_context(patch.object(updater, '_update_by_arrow_batches_with_row_id', pure)) invoke = lambda: updater.update_by_arrow_batches_with_row_id([data]) factory = 'create_native_update' elif operation == 'predicate-update': - context.enter_context(patch.object(updater, '_matched_update_scan_table', pure)) + context.enter_context(patch('pypaimon.write.update_dispatch.create_python_update_operation', pure)) factory = 'create_native_predicate_update' invoke = lambda: updater.update_by_predicate(None, {'value': 'new'}) elif operation == 'delete': - context.enter_context(patch('pypaimon.execution.python.write.table_update.TableDeleteByRowId', pure)) + context.enter_context(patch('pypaimon.write.update_dispatch.create_python_update_operation', pure)) factory = 'create_native_delete' invoke = lambda: updater.delete_by_row_id([0]) elif operation == 'predicate-delete': - context.enter_context(patch.object(updater, '_partition_only_delete_filter', pure)) + context.enter_context(patch('pypaimon.write.update_dispatch.create_python_update_operation', pure)) factory = 'create_native_predicate_delete' invoke = lambda: updater.delete_by_predicate(None) yield invoke, module + factory, pure @@ -114,13 +122,33 @@ def test_native_preparation_propagates_fork_errors_without_python_mutations(tmp_ with _operation(tmp_path, operation) as (invoke, factory, pure), patch.dict( sys.modules, {'pypaimon_rust': module}), patch( 'pypaimon.execution.native.runtime.native_runtime_available', return_value=True), patch( - factory, side_effect=error): + factory, side_effect=error) as prepare: with pytest.raises(_ForkSafetyError) as raised: invoke() assert raised.value is fork + prepare.assert_called_once() pure.assert_not_called() +@pytest.mark.parametrize('chain', ['cause', 'context']) +def test_permission_wrapper_preserves_fork_error_priority(chain): + from pypaimon.catalog.catalog_exception import TableNoPermissionException + from pypaimon.common.identifier import Identifier + from pypaimon.write.backend_dispatch import try_prepare_native + + module = ModuleType('pypaimon_rust') + module.ForkSafetyError = _ForkSafetyError + fork = _ForkSafetyError('The inherited SDK cannot be used after fork') + permission = TableNoPermissionException(Identifier('default', 't')) + setattr(permission, '__cause__' if chain == 'cause' else '__context__', fork) + prepare = Mock(side_effect=permission) + with patch.dict(sys.modules, {'pypaimon_rust': module}): + with pytest.raises(_ForkSafetyError) as raised: + try_prepare_native(prepare) + assert raised.value is fork + prepare.assert_called_once() + + @pytest.mark.parametrize('operation', _OPERATIONS) @pytest.mark.parametrize('missing_runtime', [False, True], ids=['ordinary-error', 'missing-runtime']) def test_ordinary_native_preparation_failures_keep_the_safe_python_fallback(tmp_path, operation, missing_runtime): @@ -130,9 +158,10 @@ def test_ordinary_native_preparation_failures_keep_the_safe_python_fallback(tmp_ with _operation(tmp_path, operation) as (invoke, factory, pure), patch.dict( sys.modules, {'pypaimon_rust': module}), patch( 'pypaimon.execution.native.runtime.native_runtime_available', return_value=not missing_runtime), patch( - factory, side_effect=error): + factory, side_effect=error) as prepare: with pytest.raises(_PythonFallbackEntered): invoke() + prepare.assert_called_once() pure.assert_called_once() @@ -146,8 +175,13 @@ def test_merge_native_failures_never_restart_python_execution(tmp_path, phase): native.prepare_commit.side_effect = failure with patch('pypaimon.execution.native.write.native_merge_into.create_native_merge_into', side_effect=failure if phase == 'prepare' else None, return_value=native), patch( - 'pypaimon.write.merge_inputs._prepare', side_effect=_PythonFallbackEntered) as pure: + 'pypaimon.write.update_dispatch.create_python_merge_into_operation', + side_effect=_PythonFallbackEntered) as pure: with pytest.raises(_ForkSafetyError) as raised: merge_into(table, pa.table({'id': [1]}), on='id', commit_user='job') assert raised.value is failure pure.assert_not_called() + if phase == 'prepare': + native.prepare_commit.assert_not_called() + else: + native.prepare_commit.assert_called_once_with() diff --git a/paimon-python/pypaimon/tests/table_update_test.py b/paimon-python/pypaimon/tests/table_update_test.py index 6987c931664d..1756668728e2 100644 --- a/paimon-python/pypaimon/tests/table_update_test.py +++ b/paimon-python/pypaimon/tests/table_update_test.py @@ -85,6 +85,8 @@ def test_python_row_id_delete_reports_unsupported_table_capabilities(missing_met from pypaimon.write.table_delete import TableDeleteByRowId table = mock.Mock() + table.options.native_write_enabled.return_value = False + table.catalog_environment.table_query_auth.return_value = None table.options.data_evolution_enabled.return_value = True table.options.row_tracking_enabled.return_value = True table.options.deletion_vectors_enabled.return_value = True @@ -98,6 +100,8 @@ def test_python_row_id_delete_keeps_invalid_input_as_value_error(): from pypaimon.write.table_delete import TableDeleteByRowId table = mock.Mock() + table.options.native_write_enabled.return_value = False + table.catalog_environment.table_query_auth.return_value = None table.options.data_evolution_enabled.return_value = True table.options.row_tracking_enabled.return_value = True table.options.deletion_vectors_enabled.return_value = True diff --git a/paimon-python/pypaimon/tests/write/append_row_kind_test.py b/paimon-python/pypaimon/tests/write/append_row_kind_test.py index 8c9abd7776a0..27829d713515 100644 --- a/paimon-python/pypaimon/tests/write/append_row_kind_test.py +++ b/paimon-python/pypaimon/tests/write/append_row_kind_test.py @@ -342,6 +342,44 @@ def test_pandas_partial_write_keeps_selected_named_index(tmp_path, native): assert source.index.tolist() == [42] +@pytest.mark.python_plan +@pytest.mark.python_read +@pytest.mark.parametrize('buckets', [1, 2]) +@pytest.mark.parametrize('stream', [False, True], ids=['batch', 'stream']) +@pytest.mark.parametrize('input_', ['row', 'batch', 'table', 'pandas']) +def test_partial_write_keeps_bucket_key_until_routing(tmp_path, buckets, stream, input_): + schema = pa.schema([('id', pa.int32()), ('payload', pa.int32()), ('pt', pa.string())]) + catalog = CatalogFactory.create({'warehouse': str(tmp_path)}) + catalog.create_database('default', True) + catalog.create_table('default.t', Schema.from_pyarrow_schema( + schema, partition_keys=['pt'], options={ + 'bucket': str(buckets), 'bucket-key': 'id', 'write.native.enabled': 'false', + 'commit.native.enabled': 'false', 'read.native.enabled': 'false', + 'scan.native-plan.enabled': 'false', + }), False) + table = catalog.get_table('default.t') + builder = table.new_stream_write_builder() if stream else table.new_batch_write_builder() + writer, commit = builder.new_write(), builder.new_commit() + writer.with_write_type(['payload']) + values = [dict(id=1, payload=10, pt='a'), dict(id=2, payload=20, pt='b')] + try: + _write(writer, table, schema, values, input_, True) + messages = writer.prepare_commit(1) if stream else writer.prepare_commit() + assert all(file.write_cols == ['payload'] for message in messages for file in message.new_files) + assert all(0 <= message.bucket < buckets for message in messages) + if stream: + commit.commit(messages, 1) + else: + commit.commit(messages) + finally: + writer.close() + commit.close() + read = table.new_read_builder() + actual = read.new_read().to_arrow(read.new_scan().plan().splits()).to_pylist() + assert sorted(actual, key=lambda row: row['payload']) == [ + dict(id=None, payload=10, pt='a'), dict(id=None, payload=20, pt='b')] + + @pytest.mark.parametrize('columns', [['payload', 'id'], []]) def test_partial_batch_projection_uses_columns_and_preserves_schema_metadata(columns): from unittest.mock import Mock diff --git a/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py b/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py index 2db9cd18db8e..b0a4d9d3faa6 100644 --- a/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py +++ b/paimon-python/pypaimon/tests/write/row_id_execution_boundary_test.py @@ -146,7 +146,8 @@ def delete(self, row_ids): native_update.create_native_update_by_row_id = lambda *args: Execution() native_update.create_native_row_id_delete = lambda *args: Execution() -table = SimpleNamespace(options=SimpleNamespace(native_write_enabled=lambda: True)) +table = SimpleNamespace(options=SimpleNamespace(native_write_enabled=lambda: True), identifier=object(), + catalog_environment=SimpleNamespace(table_query_auth=lambda *args: None)) if sys.argv[1] == 'update': update = TableUpdateByRowId(table, 'user', BATCH_COMMIT_IDENTIFIER) assert update.update_columns(pa.table({'_ROW_ID': [3], 'value': [10]}), ['value']) == ['message'] diff --git a/paimon-python/pypaimon/tests/write/update_backend_selection_test.py b/paimon-python/pypaimon/tests/write/update_backend_selection_test.py index 0c7e5e934907..8c3f500113ad 100644 --- a/paimon-python/pypaimon/tests/write/update_backend_selection_test.py +++ b/paimon-python/pypaimon/tests/write/update_backend_selection_test.py @@ -31,6 +31,7 @@ def test_direct_shard_constructor_never_loads_or_replays_python(tmp_path, fails) source = r''' import importlib.abc import sys +from types import SimpleNamespace from unittest.mock import Mock, patch class BlockPythonExecution(importlib.abc.MetaPathFinder): @@ -46,7 +47,8 @@ def find_spec(self, fullname, path=None, target=None): core.prepare_commit.return_value = ['prepared'] core.update_by_arrow_batch.side_effect = failure if sys.argv[1] == 'True' else None with patch('pypaimon.write.backend_dispatch.prepare_native_write_operation', return_value=core) as prepare: - table = object() + table = SimpleNamespace(options=object(), identifier=object(), + catalog_environment=SimpleNamespace(table_query_auth=lambda *args: None)) shard = ShardTableUpdator(table, ['value'], ['value'], 'writer', 0, 2) prepare.assert_called_once_with('shard_update', table, 'writer', ['value'], ['value'], 0, 2) assert shard.arrow_reader() is core.arrow_reader.return_value diff --git a/paimon-python/pypaimon/write/backend_dispatch.py b/paimon-python/pypaimon/write/backend_dispatch.py index 0490bc106e00..f30ccd7c1a37 100644 --- a/paimon-python/pypaimon/write/backend_dispatch.py +++ b/paimon-python/pypaimon/write/backend_dispatch.py @@ -25,6 +25,7 @@ import logging import sys +from pypaimon.catalog.catalog_exception import TableNoPermissionException from pypaimon.write.commit_identifier import validate_commit_identifier logger = logging.getLogger(__name__) @@ -50,6 +51,8 @@ def try_prepare_native(factory, *args, **kwargs): return factory(*args, **kwargs) except Exception as error: propagate_fork_safety_error(error) + if isinstance(error, TableNoPermissionException): + raise logger.debug('Native execution preparation failed; using Python: %s', error) return None @@ -319,6 +322,8 @@ def create_python_merge_into_operation( def create_shard_update_execution(owner, table, projection, columns, commit_user, shard, shard_count): """Select the direct shard constructor's backend before capturing files.""" + from pypaimon.write.update_validation import _authorize_update + _authorize_update(table) native = prepare_native_write_operation( 'shard_update', table, commit_user, projection, columns, shard, shard_count) if native is not None: @@ -328,9 +333,8 @@ def create_shard_update_execution(owner, table, projection, columns, commit_user def create_row_id_update_execution(table, commit_user, commit_identifier, precomputed=None): """Select one incremental updater before capturing target files.""" - if precomputed is not None: - from pypaimon.write.update_validation import _authorize_update - _authorize_update(table) + from pypaimon.write.update_validation import _authorize_update + _authorize_update(table) if precomputed is None: native = prepare_native_write_operation('update_by_row_id', table, commit_user, commit_identifier) if native is not None: @@ -340,9 +344,8 @@ def create_row_id_update_execution(table, commit_user, commit_identifier, precom def create_row_id_delete_execution(table, precomputed=None): """Keep precomputed broadcast metadata within its selected Python executor.""" - if precomputed is not None: - from pypaimon.write.update_validation import _authorize_update - _authorize_update(table) + from pypaimon.write.update_validation import _authorize_update + _authorize_update(table) if precomputed is None and table.options.native_write_enabled(): def prepare(): from pypaimon.execution.native.write.native_update import create_native_row_id_delete diff --git a/paimon-python/pypaimon/write/update_dispatch.py b/paimon-python/pypaimon/write/update_dispatch.py index e53afdca1c40..0b170d9855c1 100644 --- a/paimon-python/pypaimon/write/update_dispatch.py +++ b/paimon-python/pypaimon/write/update_dispatch.py @@ -59,6 +59,7 @@ def new_shard_updator(self, shard_num: int, total_shard_count: int): raise ValueError('Shard index must be smaller than a positive shard count') if context.update_cols == []: raise ValueError('Shard update columns must not be empty') + _authorize_update(context.table) native = prepare_native_write_operation( 'shard_update', context.table,