From 89cdae5dbe0fb3c53f1cac385b52ec5a7a11be5f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 4 Sep 2026 05:06:38 -0600 Subject: [PATCH 1/2] test: enable the Iceberg split-operator and native write by default Both toggles have defaulted to false since they landed, so no CI job has ever exercised the two-operator plan or the iceberg-rust writer outside the handful of suites that set them explicitly. Flip the defaults to find out what the Iceberg Spark SQL suites and Comet's own Iceberg suites say when every write goes through them. Exploratory: `iceberg-writes.md` still documents both as off by default and is deliberately not updated here. --- spark/src/main/scala/org/apache/comet/CometConf.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index f4aeacc847..2fc049521e 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -129,7 +129,7 @@ object CometConf extends ShimCometConf { "into Comet's two-operator shape: a file writer exec (inside AQE) and a committer " + "(outside AQE).") .booleanConf - .createWithDefault(false) + .createWithDefault(true) val COMET_ICEBERG_NATIVE_WRITE_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.iceberg.write.enabled") @@ -137,9 +137,9 @@ object CometConf extends ShimCometConf { .doc( "Whether to delegate the executor-side Parquet write to Comet's native (iceberg-rust) " + "writer when the table's properties allow it. Requires " + - "`spark.comet.write.iceberg.splitOperator.enabled = true`. Off by default.") + "`spark.comet.write.iceberg.splitOperator.enabled = true`.") .booleanConf - .createWithDefault(false) + .createWithDefault(true) val COMET_ICEBERG_DATA_FILE_CONCURRENCY_LIMIT: ConfigEntry[Int] = conf("spark.comet.scan.icebergNative.dataFileConcurrencyLimit") From 62c63eb8818d3c7e2cdf398ef92dcd2730140502 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 4 Sep 2026 06:18:58 -0600 Subject: [PATCH 2/2] test: enable local table scan in the Iceberg test diffs The Iceberg test sessions already configure the write split operator, but they leave `spark.comet.exec.localTableScan.enabled` off. Spark emits a row-based `LocalTableScanExec` for an inline `VALUES` list, and `CometIcebergNativeWrite` sets `requiresNativeChildren`, so those writes are declined and silently run on the JVM writer. Iceberg's suites seed data that way constantly, so the native writer stayed hidden from most of the write surface even with `spark.comet.iceberg.write.enabled` on. Measured on Spark 4.1.3 with the Iceberg 1.11 runtime and the Iceberg CI session config: unpartitioned `INSERT INTO ... VALUES` and `df.writeTo(...).append()` both selected the JVM writer before this change and the native writer after it. Regenerated all four diffs against their own tags per the contributor guide. Also document the flag and the existing `spark.comet.explainFallback.enabled` setting, which reports every declined operator with its reason, and drop the stale claim that the split-operator flag is off by default. --- dev/diffs/iceberg/1.10.0.diff | 335 ++++++++++------- dev/diffs/iceberg/1.11.0.diff | 173 +++++---- dev/diffs/iceberg/1.8.1.diff | 339 +++++++++++------- dev/diffs/iceberg/1.9.1.diff | 339 +++++++++++------- .../contributor-guide/iceberg-spark-tests.md | 11 +- 5 files changed, 711 insertions(+), 486 deletions(-) diff --git a/dev/diffs/iceberg/1.10.0.diff b/dev/diffs/iceberg/1.10.0.diff index 7bbf584c31..4d9bf0c3a5 100644 --- a/dev/diffs/iceberg/1.10.0.diff +++ b/dev/diffs/iceberg/1.10.0.diff @@ -24,10 +24,10 @@ index 714be0831d..dede65f213 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 4c1a509591..186a264a88 100644 +index 4c1a509591..51f9244f10 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -59,7 +59,25 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -59,7 +59,27 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -38,6 +38,7 @@ index 4c1a509591..186a264a88 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -48,16 +49,17 @@ index 4c1a509591..186a264a88 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestBase.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index 58d054bd05..ef1c1cb6d1 100644 +index 58d054bd05..096985a6b5 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,15 @@ public class TestCallStatementParser { +@@ -56,6 +56,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -68,6 +70,7 @@ index 58d054bd05..ef1c1cb6d1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -102,10 +105,10 @@ index 5a0a04edba..afe92a8b72 100644 } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index b6ade2bff3..90a4839dad 100644 +index b6ade2bff3..af28e6bb33 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -170,6 +170,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -116,16 +119,17 @@ index b6ade2bff3..90a4839dad 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index 7cd41bd7e7..0b6d490529 100644 +index 7cd41bd7e7..b3fa94f225 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { +@@ -386,6 +386,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -136,16 +140,17 @@ index 7cd41bd7e7..0b6d490529 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -160,6 +165,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -947,10 +953,10 @@ index a0f45e7610..78e7845911 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -961,16 +967,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 9361c63176..16ce1c614f 100644 +index 9361c63176..5c9fb10e62 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -69,6 +69,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -981,16 +988,17 @@ index 9361c63176..16ce1c614f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1001,16 +1009,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index acd4688440..33732e6654 100644 +index acd4688440..4f7a9040a7 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting +@@ -59,6 +59,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1021,16 +1030,17 @@ index acd4688440..33732e6654 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index 3c32b46936..2a39c4605c 100644 +index 3c32b46936..1b14b79784 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -79,6 +79,15 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -79,6 +79,16 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -1041,16 +1051,17 @@ index 3c32b46936..2a39c4605c 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 6647a1b483..aeb16f85ac 100644 +index 6647a1b483..92ece0d94f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -62,6 +62,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { +@@ -62,6 +62,16 @@ public abstract class ScanTestBase extends AvroDataTestBase { SparkSession.builder() .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .master("local[2]") @@ -1061,16 +1072,17 @@ index 6647a1b483..aeb16f85ac 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 24a14bb64d..9329cd99b3 100644 +index 24a14bb64d..4cab0ae7e1 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,20 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -1085,6 +1097,7 @@ index 24a14bb64d..9329cd99b3 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1092,10 +1105,10 @@ index 24a14bb64d..9329cd99b3 100644 @BeforeEach diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index 99d3f38ee7..f1a74280dd 100644 +index 99d3f38ee7..db87f1904b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -74,7 +74,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -74,7 +74,20 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -1110,6 +1123,7 @@ index 99d3f38ee7..f1a74280dd 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1117,10 +1131,10 @@ index 99d3f38ee7..f1a74280dd 100644 @AfterAll diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index a70aeb1ba6..fae2359d01 100644 +index a70aeb1ba6..ed82c3a8b2 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -123,6 +123,15 @@ public class TestFilteredScan { +@@ -123,6 +123,16 @@ public class TestFilteredScan { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1131,16 +1145,17 @@ index a70aeb1ba6..fae2359d01 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); // define UDFs used by partition tests diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index c03f7b94ec..30032fa107 100644 +index c03f7b94ec..52c3728a77 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -99,6 +99,15 @@ public class TestForwardCompatibility { +@@ -99,6 +99,16 @@ public class TestForwardCompatibility { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1151,16 +1166,17 @@ index c03f7b94ec..30032fa107 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index f4f57157e4..418364e3da 100644 +index f4f57157e4..d3fb378fec 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -51,6 +51,15 @@ public class TestIcebergSpark { +@@ -51,6 +51,16 @@ public class TestIcebergSpark { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1171,16 +1187,17 @@ index f4f57157e4..418364e3da 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index e1402396fa..3092902873 100644 +index e1402396fa..e007e8e35b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -118,6 +118,15 @@ public class TestPartitionPruning { +@@ -118,6 +118,16 @@ public class TestPartitionPruning { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1191,16 +1208,17 @@ index e1402396fa..3092902873 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index b0ad930487..6764bf645a 100644 +index b0ad930487..886726d908 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -112,6 +112,15 @@ public class TestPartitionValues { +@@ -112,6 +112,16 @@ public class TestPartitionValues { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1211,16 +1229,17 @@ index b0ad930487..6764bf645a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 11865db7fc..c7c1e6a02f 100644 +index 11865db7fc..e09c0fc74e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -91,6 +91,15 @@ public class TestSnapshotSelection { +@@ -91,6 +91,16 @@ public class TestSnapshotSelection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1231,16 +1250,17 @@ index 11865db7fc..c7c1e6a02f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 833a0b6b5e..eb29e23aac 100644 +index 833a0b6b5e..6dbcc8d985 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -125,6 +125,15 @@ public class TestSparkDataFile { +@@ -125,6 +125,16 @@ public class TestSparkDataFile { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1251,16 +1271,17 @@ index 833a0b6b5e..eb29e23aac 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 8884654fe0..fde607663e 100644 +index 8884654fe0..fe01afe653 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -100,6 +100,15 @@ public class TestSparkDataWrite { +@@ -100,6 +100,16 @@ public class TestSparkDataWrite { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1271,12 +1292,13 @@ index 8884654fe0..fde607663e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } -@@ -144,7 +153,7 @@ public class TestSparkDataWrite { +@@ -144,7 +154,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1285,7 +1307,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); for (ManifestFile manifest : SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { -@@ -213,7 +222,7 @@ public class TestSparkDataWrite { +@@ -213,7 +223,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1294,7 +1316,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -258,7 +267,7 @@ public class TestSparkDataWrite { +@@ -258,7 +268,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1303,7 +1325,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -310,7 +319,7 @@ public class TestSparkDataWrite { +@@ -310,7 +320,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1312,7 +1334,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -352,7 +361,7 @@ public class TestSparkDataWrite { +@@ -352,7 +362,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1321,7 +1343,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -391,7 +400,7 @@ public class TestSparkDataWrite { +@@ -391,7 +401,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1330,7 +1352,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); List files = Lists.newArrayList(); -@@ -459,7 +468,7 @@ public class TestSparkDataWrite { +@@ -459,7 +469,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1339,7 +1361,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -707,7 +716,7 @@ public class TestSparkDataWrite { +@@ -707,7 +717,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1349,10 +1371,10 @@ index 8884654fe0..fde607663e 100644 .hasSize(records.size() + records2.size()) .containsExactlyInAnyOrder( diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 596d05d30b..acbacf65e0 100644 +index 596d05d30b..b6ed4688ce 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -88,6 +88,15 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -88,6 +88,16 @@ public class TestSparkReadProjection extends TestReadProjection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1363,16 +1385,17 @@ index 596d05d30b..acbacf65e0 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); ImmutableMap config = ImmutableMap.of( diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 16fa726032..ce27c5cd4d 100644 +index 16fa726032..be4813379e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -132,7 +132,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -132,7 +132,27 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1383,6 +1406,7 @@ index 16fa726032..ce27c5cd4d 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1393,12 +1417,13 @@ index 16fa726032..ce27c5cd4d 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = -@@ -200,7 +218,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -200,7 +220,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -1409,10 +1434,10 @@ index 16fa726032..ce27c5cd4d 100644 @Override diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..fb677d7114 100644 +index baf7fa8f88..69181efc94 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,7 +182,25 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,7 +182,27 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1423,6 +1448,7 @@ index baf7fa8f88..fb677d7114 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1433,16 +1459,17 @@ index baf7fa8f88..fb677d7114 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index dc4fc7e187..2ab6dd12c2 100644 +index dc4fc7e187..b56dc6db7b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -69,6 +69,15 @@ public class TestStructuredStreaming { +@@ -69,6 +69,16 @@ public class TestStructuredStreaming { .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .config("spark.sql.shuffle.partitions", 4) @@ -1453,16 +1480,17 @@ index dc4fc7e187..2ab6dd12c2 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 8b1e3fbfc7..c519a2be0f 100644 +index 8b1e3fbfc7..8f774766cc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,20 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -1477,6 +1505,7 @@ index 8b1e3fbfc7..c519a2be0f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1484,10 +1513,10 @@ index 8b1e3fbfc7..c519a2be0f 100644 @AfterAll diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index c3fac70dd3..d3ffd034d2 100644 +index c3fac70dd3..5491a84de3 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -84,6 +84,15 @@ public class TestWriteMetricsConfig { +@@ -84,6 +84,16 @@ public class TestWriteMetricsConfig { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1498,16 +1527,17 @@ index c3fac70dd3..d3ffd034d2 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 5ce56b4fec..3b614e3fc9 100644 +index 5ce56b4fec..ee6ce0cbb1 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -63,7 +63,25 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -63,7 +63,27 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -1518,6 +1548,7 @@ index 5ce56b4fec..3b614e3fc9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1528,6 +1559,7 @@ index 5ce56b4fec..3b614e3fc9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1561,10 +1593,10 @@ index 69700d8436..49ea338a45 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 4c1a509591..3b4e44cc13 100644 +index 4c1a509591..2477b29cba 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -59,6 +59,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -59,6 +59,16 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -1575,6 +1607,7 @@ index 4c1a509591..3b4e44cc13 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1595,10 +1628,10 @@ index 893f9931cf..17e5fef217 100644 protected void createTableWithDeleteGranularity( diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index ecf9e6f8a5..beca5f8521 100644 +index ecf9e6f8a5..42591bd9b2 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,15 @@ public class TestCallStatementParser { +@@ -56,6 +56,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -1609,6 +1642,7 @@ index ecf9e6f8a5..beca5f8521 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1643,10 +1677,10 @@ index 8cb92224fc..777b9bb1dd 100644 } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 64edb1002e..57767da086 100644 +index 64edb1002e..fc1d0f01ae 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -179,6 +179,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -1657,16 +1691,17 @@ index 64edb1002e..57767da086 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index 77b79384a6..8708ffa72a 100644 +index 77b79384a6..0ac69d5487 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { +@@ -392,6 +392,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -1677,16 +1712,17 @@ index 77b79384a6..8708ffa72a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index c6794e43c6..4c1a165a44 100644 +index c6794e43c6..90bde88cd5 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -239,6 +239,15 @@ public class DVReaderBenchmark { +@@ -239,6 +239,16 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1697,16 +1733,17 @@ index c6794e43c6..4c1a165a44 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index ac74fb5a10..4aa5d5b361 100644 +index ac74fb5a10..07b8c34f8a 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -223,6 +223,15 @@ public class DVWriterBenchmark { +@@ -223,6 +223,16 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1717,16 +1754,17 @@ index ac74fb5a10..4aa5d5b361 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -1741,6 +1779,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -2528,10 +2567,10 @@ index a0f45e7610..78e7845911 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2542,16 +2581,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 659507e4c5..2a3a87f079 100644 +index 659507e4c5..d44910e809 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2562,16 +2602,17 @@ index 659507e4c5..2a3a87f079 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2582,16 +2623,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index 2665d7ba8d..c24c82341a 100644 +index 2665d7ba8d..ef17afe769 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2602,16 +2644,17 @@ index 2665d7ba8d..c24c82341a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index daf4e29ac0..235cf60557 100644 +index daf4e29ac0..7f21607315 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -79,6 +79,15 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -79,6 +79,16 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -2622,6 +2665,7 @@ index daf4e29ac0..235cf60557 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -2653,10 +2697,10 @@ index 0db6a65fd3..9ee8b9e084 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -index 973a17c9a3..4e4d3c02f1 100644 +index 973a17c9a3..9d3852665f 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -@@ -65,6 +65,15 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -65,6 +65,16 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2667,16 +2711,17 @@ index 973a17c9a3..4e4d3c02f1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 1c5905744a..b9cf548290 100644 +index 1c5905744a..08b0e0a361 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -61,6 +61,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { +@@ -61,6 +61,16 @@ public abstract class ScanTestBase extends AvroDataTestBase { ScanTestBase.spark = SparkSession.builder() .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2687,16 +2732,17 @@ index 1c5905744a..b9cf548290 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[2]") .getOrCreate(); ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 19ec6d13dd..c5f73bf666 100644 +index 19ec6d13dd..125e3b2ccb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,20 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -2711,6 +2757,7 @@ index 19ec6d13dd..c5f73bf666 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2718,10 +2765,10 @@ index 19ec6d13dd..c5f73bf666 100644 @BeforeEach diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index a7702b169a..67fabf3cdc 100644 +index a7702b169a..61359553cb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -74,7 +74,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -74,7 +74,20 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -2736,6 +2783,7 @@ index a7702b169a..67fabf3cdc 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2743,10 +2791,10 @@ index a7702b169a..67fabf3cdc 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index fd7d52178f..36831cfb07 100644 +index fd7d52178f..fb806b7023 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -114,6 +114,15 @@ public class TestFilteredScan { +@@ -114,6 +114,16 @@ public class TestFilteredScan { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2757,16 +2805,17 @@ index fd7d52178f..36831cfb07 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 153564f7d1..973eac1878 100644 +index 153564f7d1..b9eb74e443 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -98,6 +98,15 @@ public class TestForwardCompatibility { +@@ -98,6 +98,16 @@ public class TestForwardCompatibility { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2777,16 +2826,17 @@ index 153564f7d1..973eac1878 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index f4f57157e4..418364e3da 100644 +index f4f57157e4..d3fb378fec 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -51,6 +51,15 @@ public class TestIcebergSpark { +@@ -51,6 +51,16 @@ public class TestIcebergSpark { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2797,16 +2847,17 @@ index f4f57157e4..418364e3da 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index e1402396fa..3092902873 100644 +index e1402396fa..e007e8e35b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -118,6 +118,15 @@ public class TestPartitionPruning { +@@ -118,6 +118,16 @@ public class TestPartitionPruning { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2817,16 +2868,17 @@ index e1402396fa..3092902873 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 0b6ab2052b..8682b4a7da 100644 +index 0b6ab2052b..5ff42b3182 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -112,6 +112,15 @@ public class TestPartitionValues { +@@ -112,6 +112,16 @@ public class TestPartitionValues { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2837,16 +2889,17 @@ index 0b6ab2052b..8682b4a7da 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 11865db7fc..c7c1e6a02f 100644 +index 11865db7fc..e09c0fc74e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -91,6 +91,15 @@ public class TestSnapshotSelection { +@@ -91,6 +91,16 @@ public class TestSnapshotSelection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2857,16 +2910,17 @@ index 11865db7fc..c7c1e6a02f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 3051e27d72..cb3011b222 100644 +index 3051e27d72..7f8b806351 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -125,6 +125,15 @@ public class TestSparkDataFile { +@@ -125,6 +125,16 @@ public class TestSparkDataFile { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2877,16 +2931,17 @@ index 3051e27d72..cb3011b222 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 4ccbf86f12..f915f22b21 100644 +index 4ccbf86f12..24f7ea1979 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -100,6 +100,15 @@ public class TestSparkDataWrite { +@@ -100,6 +100,16 @@ public class TestSparkDataWrite { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2897,12 +2952,13 @@ index 4ccbf86f12..f915f22b21 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } -@@ -144,7 +153,7 @@ public class TestSparkDataWrite { +@@ -144,7 +154,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2911,7 +2967,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); for (ManifestFile manifest : SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { -@@ -213,7 +222,7 @@ public class TestSparkDataWrite { +@@ -213,7 +223,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2920,7 +2976,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -258,7 +267,7 @@ public class TestSparkDataWrite { +@@ -258,7 +268,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2929,7 +2985,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -310,7 +319,7 @@ public class TestSparkDataWrite { +@@ -310,7 +320,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2938,7 +2994,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -352,7 +361,7 @@ public class TestSparkDataWrite { +@@ -352,7 +362,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2947,7 +3003,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -391,7 +400,7 @@ public class TestSparkDataWrite { +@@ -391,7 +401,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2956,7 +3012,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); List files = Lists.newArrayList(); -@@ -459,7 +468,7 @@ public class TestSparkDataWrite { +@@ -459,7 +469,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2965,7 +3021,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -706,7 +715,7 @@ public class TestSparkDataWrite { +@@ -706,7 +716,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2975,10 +3031,10 @@ index 4ccbf86f12..f915f22b21 100644 .hasSize(records.size() + records2.size()) .containsExactlyInAnyOrder( diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 596d05d30b..acbacf65e0 100644 +index 596d05d30b..b6ed4688ce 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -88,6 +88,15 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -88,6 +88,16 @@ public class TestSparkReadProjection extends TestReadProjection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2989,16 +3045,17 @@ index 596d05d30b..acbacf65e0 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); ImmutableMap config = ImmutableMap.of( diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 42699f4662..b8bf79886e 100644 +index 42699f4662..95ff0bbcac 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -138,6 +138,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -138,6 +138,16 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3009,12 +3066,13 @@ index 42699f4662..b8bf79886e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); -@@ -206,7 +215,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -206,7 +216,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -3025,10 +3083,10 @@ index 42699f4662..b8bf79886e 100644 @Override diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..b58321b761 100644 +index baf7fa8f88..2eaa17e8b0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,6 +182,16 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3039,16 +3097,17 @@ index baf7fa8f88..b58321b761 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 54048bbf21..2ebe9ab993 100644 +index 54048bbf21..da2b17b309 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -69,6 +69,15 @@ public class TestStructuredStreaming { +@@ -69,6 +69,16 @@ public class TestStructuredStreaming { .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .config("spark.sql.shuffle.partitions", 4) @@ -3059,16 +3118,17 @@ index 54048bbf21..2ebe9ab993 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 8b1e3fbfc7..c519a2be0f 100644 +index 8b1e3fbfc7..8f774766cc 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,20 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -3083,6 +3143,7 @@ index 8b1e3fbfc7..c519a2be0f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3090,10 +3151,10 @@ index 8b1e3fbfc7..c519a2be0f 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index c3fac70dd3..d3ffd034d2 100644 +index c3fac70dd3..5491a84de3 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -84,6 +84,15 @@ public class TestWriteMetricsConfig { +@@ -84,6 +84,16 @@ public class TestWriteMetricsConfig { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -3104,16 +3165,17 @@ index c3fac70dd3..d3ffd034d2 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 5ce56b4fec..d089cc1f69 100644 +index 5ce56b4fec..e28d4a417b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -63,6 +63,15 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -63,6 +63,16 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -3124,6 +3186,7 @@ index 5ce56b4fec..d089cc1f69 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() diff --git a/dev/diffs/iceberg/1.11.0.diff b/dev/diffs/iceberg/1.11.0.diff index 555002b51f..5fd5de39bf 100644 --- a/dev/diffs/iceberg/1.11.0.diff +++ b/dev/diffs/iceberg/1.11.0.diff @@ -39,10 +39,10 @@ index e6455fa34f..08b685f61a 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index f766fbb79a..59e4e6c285 100644 +index f766fbb79a..24c309e34b 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -71,6 +71,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -71,6 +71,16 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -53,6 +53,7 @@ index f766fbb79a..59e4e6c285 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) @@ -101,10 +102,10 @@ index 934220e5d3..92132d4b8d 100644 } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 3fd84553f0..db71a30730 100644 +index 3fd84553f0..6954ee71dd 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -180,6 +180,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -180,6 +180,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -115,16 +116,17 @@ index 3fd84553f0..db71a30730 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .master("local"); spark = builder.getOrCreate(); diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index 683f6bb46d..bd5ef5d3c0 100644 +index 683f6bb46d..21652664ba 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -395,6 +395,15 @@ public class IcebergSortCompactionBenchmark { +@@ -395,6 +395,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -135,16 +137,17 @@ index 683f6bb46d..bd5ef5d3c0 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .master("local[*]"); spark = builder.getOrCreate(); diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index 3f242ce228..ee19cc1248 100644 +index 3f242ce228..31df5b5dc4 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -240,6 +240,15 @@ public class DVReaderBenchmark { +@@ -240,6 +240,16 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -155,16 +158,17 @@ index 3f242ce228..ee19cc1248 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index db57897240..f3b47a62f6 100644 +index db57897240..4becac2607 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -224,6 +224,15 @@ public class DVWriterBenchmark { +@@ -224,6 +224,16 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -175,16 +179,17 @@ index db57897240..f3b47a62f6 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index debe37866f..f48e355a4c 100644 +index debe37866f..518e10eefe 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -95,7 +95,18 @@ public abstract class IcebergSourceBenchmark { +@@ -95,7 +95,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -199,16 +204,17 @@ index debe37866f..f48e355a4c 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { builder .config("parquet.dictionary.page.size", "1") diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index d1c724425c..da7450dd95 100644 +index d1c724425c..e84c5c7d2b 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -219,16 +225,17 @@ index d1c724425c..da7450dd95 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index a21c6a08ec..104bfcd580 100644 +index a21c6a08ec..ce4cb44dca 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -239,16 +246,17 @@ index a21c6a08ec..104bfcd580 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index 5edf482822..24bd0ee141 100644 +index 5edf482822..dfebdfd0ab 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -259,16 +267,17 @@ index 5edf482822..24bd0ee141 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index e6f3c75475..6e2a7c6fb7 100644 +index e6f3c75475..cac1cfc97b 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -279,16 +288,17 @@ index e6f3c75475..6e2a7c6fb7 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index 507d7b313b..8fbdb40228 100644 +index 507d7b313b..d22c156031 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -86,6 +86,15 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -86,6 +86,16 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -299,6 +309,7 @@ index 507d7b313b..8fbdb40228 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config("spark.ui.enabled", "false") @@ -330,10 +341,10 @@ index 8ce60f6275..75112b4354 100644 } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -index b61ecfa2f4..a3aafae4ee 100644 +index b61ecfa2f4..fc38da6d71 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -@@ -66,6 +66,15 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -66,6 +66,16 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -344,16 +355,17 @@ index b61ecfa2f4..a3aafae4ee 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index da9cd63921..c713c27456 100644 +index da9cd63921..66c7abffab 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -62,6 +62,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { +@@ -62,6 +62,16 @@ public abstract class ScanTestBase extends AvroDataTestBase { ScanTestBase.spark = SparkSession.builder() .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -364,16 +376,17 @@ index da9cd63921..c713c27456 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[2]") .config(TestBase.DISABLE_UI) .getOrCreate(); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index d381822483..f090b5bf14 100644 +index d381822483..edc5434460 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,20 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -388,6 +401,7 @@ index d381822483..f090b5bf14 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -395,10 +409,10 @@ index d381822483..f090b5bf14 100644 @BeforeEach diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index e67ec5fd62..ef19a1d349 100644 +index e67ec5fd62..d9f078a615 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -79,7 +79,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -79,7 +79,20 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -413,6 +427,7 @@ index e67ec5fd62..ef19a1d349 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -420,10 +435,10 @@ index e67ec5fd62..ef19a1d349 100644 @AfterAll diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index 24fecf4eb2..964a5bc003 100644 +index 24fecf4eb2..7c38f1d85f 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -117,6 +117,15 @@ public class TestFilteredScan { +@@ -117,6 +117,16 @@ public class TestFilteredScan { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -434,16 +449,17 @@ index 24fecf4eb2..964a5bc003 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index d0103ff46e..c634782764 100644 +index d0103ff46e..03e0b5d927 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -99,6 +99,15 @@ public class TestForwardCompatibility { +@@ -99,6 +99,16 @@ public class TestForwardCompatibility { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -454,16 +470,17 @@ index d0103ff46e..c634782764 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index a637b975fe..dc31aab58b 100644 +index a637b975fe..dffaa26c93 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -52,6 +52,15 @@ public class TestIcebergSpark { +@@ -52,6 +52,16 @@ public class TestIcebergSpark { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -474,16 +491,17 @@ index a637b975fe..dc31aab58b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 8098db81f9..a3b023a43e 100644 +index 8098db81f9..03631fb1c6 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -120,6 +120,15 @@ public class TestPartitionPruning { +@@ -120,6 +120,16 @@ public class TestPartitionPruning { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -494,16 +512,17 @@ index 8098db81f9..a3b023a43e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 9b5b22a73f..4aed7f06dc 100644 +index 9b5b22a73f..793bce01da 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -113,6 +113,15 @@ public class TestPartitionValues { +@@ -113,6 +113,16 @@ public class TestPartitionValues { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -514,16 +533,17 @@ index 9b5b22a73f..4aed7f06dc 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 3004e8fa5c..82ebbe9e81 100644 +index 3004e8fa5c..e59564d443 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -98,6 +98,15 @@ public class TestSnapshotSelection { +@@ -98,6 +98,16 @@ public class TestSnapshotSelection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -534,16 +554,17 @@ index 3004e8fa5c..82ebbe9e81 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index d719ca6751..00ec4f9cdf 100644 +index d719ca6751..17a41322c9 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -127,6 +127,15 @@ public class TestSparkDataFile { +@@ -127,6 +127,16 @@ public class TestSparkDataFile { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -554,16 +575,17 @@ index d719ca6751..00ec4f9cdf 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index c2f5afef0e..d970524cd8 100644 +index c2f5afef0e..7bb8557f0d 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -104,6 +104,15 @@ public class TestSparkDataWrite { +@@ -104,6 +104,16 @@ public class TestSparkDataWrite { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -574,12 +596,13 @@ index c2f5afef0e..d970524cd8 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } -@@ -149,7 +158,7 @@ public class TestSparkDataWrite { +@@ -149,7 +159,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -588,7 +611,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); for (ManifestFile manifest : SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { -@@ -219,7 +228,7 @@ public class TestSparkDataWrite { +@@ -219,7 +229,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -597,7 +620,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -264,7 +273,7 @@ public class TestSparkDataWrite { +@@ -264,7 +274,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -606,7 +629,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -316,7 +325,7 @@ public class TestSparkDataWrite { +@@ -316,7 +326,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -615,7 +638,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -358,7 +367,7 @@ public class TestSparkDataWrite { +@@ -358,7 +368,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -624,7 +647,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -397,7 +406,7 @@ public class TestSparkDataWrite { +@@ -397,7 +407,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -633,7 +656,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); List files = Lists.newArrayList(); -@@ -461,7 +470,7 @@ public class TestSparkDataWrite { +@@ -461,7 +471,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -642,7 +665,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -814,7 +823,7 @@ public class TestSparkDataWrite { +@@ -814,7 +824,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -652,10 +675,10 @@ index c2f5afef0e..d970524cd8 100644 .hasSize(records.size() + records2.size()) .containsExactlyInAnyOrder( diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index de6a5e5902..e98a15fe39 100644 +index de6a5e5902..5d733e6168 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -89,6 +89,15 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -89,6 +89,16 @@ public class TestSparkReadProjection extends TestReadProjection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -666,16 +689,17 @@ index de6a5e5902..e98a15fe39 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); ImmutableMap config = diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 0d61930571..0a6208cc21 100644 +index 0d61930571..dfd998ae2e 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -138,6 +138,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -138,6 +138,16 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -686,12 +710,13 @@ index 0d61930571..0a6208cc21 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .enableHiveSupport() .getOrCreate(); -@@ -210,7 +219,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -210,7 +220,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -702,10 +727,10 @@ index 0d61930571..0a6208cc21 100644 @Override diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index cb2f866fab..02c42b7299 100644 +index cb2f866fab..9ac20eaf5f 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -183,6 +183,15 @@ public class TestSparkReaderWithBloomFilter { +@@ -183,6 +183,16 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -716,16 +741,17 @@ index cb2f866fab..02c42b7299 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .enableHiveSupport() .getOrCreate(); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 5e900ea0ba..2f7d6a08a1 100644 +index 5e900ea0ba..30bd7ce86f 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -73,6 +73,15 @@ public class TestStructuredStreaming { +@@ -73,6 +73,16 @@ public class TestStructuredStreaming { .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .config("spark.sql.shuffle.partitions", 4) @@ -736,16 +762,17 @@ index 5e900ea0ba..2f7d6a08a1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 79781a8fc3..08bfa8098b 100644 +index 79781a8fc3..7c321dbc7f 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,20 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -760,6 +787,7 @@ index 79781a8fc3..08bfa8098b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -767,10 +795,10 @@ index 79781a8fc3..08bfa8098b 100644 @AfterAll diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index ab2479d610..49383ca4a6 100644 +index ab2479d610..f4d144344d 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -86,6 +86,15 @@ public class TestWriteMetricsConfig { +@@ -86,6 +86,16 @@ public class TestWriteMetricsConfig { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -781,16 +809,17 @@ index ab2479d610..49383ca4a6 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 1669301d2d..422c7e4ace 100644 +index 1669301d2d..0806a2a5a6 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -63,6 +63,15 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -63,6 +63,16 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -801,6 +830,7 @@ index 1669301d2d..422c7e4ace 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) @@ -824,10 +854,10 @@ index e5a9d63b68..220d445851 100644 assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java -index 8cdcf22e58..da3ea3560f 100644 +index 8cdcf22e58..f6cfadb6ad 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java -@@ -100,6 +100,15 @@ public class TestVariantShredding extends CatalogTestBase { +@@ -100,6 +100,16 @@ public class TestVariantShredding extends CatalogTestBase { .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config(DISABLE_UI) @@ -838,6 +868,7 @@ index 8cdcf22e58..da3ea3560f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() diff --git a/dev/diffs/iceberg/1.8.1.diff b/dev/diffs/iceberg/1.8.1.diff index 718eeccd7e..e15ef0f444 100644 --- a/dev/diffs/iceberg/1.8.1.diff +++ b/dev/diffs/iceberg/1.8.1.diff @@ -51,10 +51,10 @@ index 6eb26e8b73..38d9cf3ca3 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java -index 4f137f5b8d..f1122f0843 100644 +index 4f137f5b8d..2af2ebebf7 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java -@@ -60,7 +60,25 @@ public abstract class SparkExtensionsTestBase extends SparkCatalogTestBase { +@@ -60,7 +60,27 @@ public abstract class SparkExtensionsTestBase extends SparkCatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -65,6 +65,7 @@ index 4f137f5b8d..f1122f0843 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -75,6 +76,7 @@ index 4f137f5b8d..f1122f0843 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -95,10 +97,10 @@ index 5fa76f9da6..9184aaba83 100644 protected void createTableWithDeleteGranularity( diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index 55a413063e..61a9385ff3 100644 +index 55a413063e..ebfdc1ec79 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -61,6 +61,15 @@ public class TestCallStatementParser { +@@ -61,6 +61,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -109,6 +111,7 @@ index 55a413063e..61a9385ff3 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -143,10 +146,10 @@ index 49119d319c..04fd3d4859 100644 } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index b6ade2bff3..90a4839dad 100644 +index b6ade2bff3..af28e6bb33 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -170,6 +170,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -157,16 +160,17 @@ index b6ade2bff3..90a4839dad 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index b08c352819..63880b089d 100644 +index b08c352819..de091c0b09 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { +@@ -386,6 +386,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -177,16 +181,17 @@ index b08c352819..63880b089d 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -201,6 +206,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -1004,10 +1010,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1018,16 +1024,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 9361c63176..16ce1c614f 100644 +index 9361c63176..5c9fb10e62 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -69,6 +69,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1038,16 +1045,17 @@ index 9361c63176..16ce1c614f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1058,16 +1066,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index acd4688440..33732e6654 100644 +index acd4688440..4f7a9040a7 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting +@@ -59,6 +59,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1078,16 +1087,17 @@ index acd4688440..33732e6654 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -index 3e8953fb95..51ce392476 100644 +index 3e8953fb95..2b021f0226 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -@@ -77,6 +77,15 @@ public abstract class SparkTestBase extends SparkTestHelperBase { +@@ -77,6 +77,16 @@ public abstract class SparkTestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -1098,6 +1108,7 @@ index 3e8953fb95..51ce392476 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1129,10 +1140,10 @@ index ad969384c5..f3c2b833e3 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 3a269740b7..fc63449215 100644 +index 3a269740b7..a0f6341fbd 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -54,7 +54,19 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -54,7 +54,20 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -1147,6 +1158,7 @@ index 3a269740b7..fc63449215 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1154,10 +1166,10 @@ index 3a269740b7..fc63449215 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 724c6edde2..ff01c10e1a 100644 +index 724c6edde2..9f2e06e3bc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -103,7 +103,19 @@ public class TestCompressionSettings extends SparkCatalogTestBase { +@@ -103,7 +103,20 @@ public class TestCompressionSettings extends SparkCatalogTestBase { @BeforeClass public static void startSpark() { @@ -1172,6 +1184,7 @@ index 724c6edde2..ff01c10e1a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1179,10 +1192,10 @@ index 724c6edde2..ff01c10e1a 100644 @Parameterized.AfterParam diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index 013b8d4386..007b1af65b 100644 +index 013b8d4386..e112c78cfc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -76,7 +76,19 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { +@@ -76,7 +76,20 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { @BeforeClass public static void startSpark() { @@ -1197,6 +1210,7 @@ index 013b8d4386..007b1af65b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1204,10 +1218,10 @@ index 013b8d4386..007b1af65b 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index ba13d005bd..b9778c1986 100644 +index ba13d005bd..a426d6e79e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -118,7 +118,19 @@ public class TestFilteredScan { +@@ -118,7 +118,20 @@ public class TestFilteredScan { @BeforeClass public static void startSpark() { @@ -1222,6 +1236,7 @@ index ba13d005bd..b9778c1986 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1229,10 +1244,10 @@ index ba13d005bd..b9778c1986 100644 // define UDFs used by partition tests Function bucket4 = Transforms.bucket(4).bind(Types.LongType.get()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 9f97753094..fb5bfbad1b 100644 +index 9f97753094..39e19af0b0 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -94,7 +94,19 @@ public class TestForwardCompatibility { +@@ -94,7 +94,20 @@ public class TestForwardCompatibility { @BeforeClass public static void startSpark() { @@ -1247,6 +1262,7 @@ index 9f97753094..fb5bfbad1b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1254,10 +1270,10 @@ index 9f97753094..fb5bfbad1b 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 0154506f86..12e916cbe4 100644 +index 0154506f86..b57e0bb01b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,19 @@ public class TestIcebergSpark { +@@ -46,7 +46,20 @@ public class TestIcebergSpark { @BeforeClass public static void startSpark() { @@ -1272,6 +1288,7 @@ index 0154506f86..12e916cbe4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1279,10 +1296,10 @@ index 0154506f86..12e916cbe4 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 639d37c793..5e397909d4 100644 +index 639d37c793..6d033d2634 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -111,7 +111,19 @@ public class TestPartitionPruning { +@@ -111,7 +111,20 @@ public class TestPartitionPruning { @BeforeClass public static void startSpark() { @@ -1297,6 +1314,7 @@ index 639d37c793..5e397909d4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1304,10 +1322,10 @@ index 639d37c793..5e397909d4 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index ad0984ef42..99af33c86c 100644 +index ad0984ef42..3333372cf2 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -104,7 +104,19 @@ public class TestPartitionValues { +@@ -104,7 +104,20 @@ public class TestPartitionValues { @BeforeClass public static void startSpark() { @@ -1322,6 +1340,7 @@ index ad0984ef42..99af33c86c 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1329,10 +1348,10 @@ index ad0984ef42..99af33c86c 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 9fc576dde5..69a49f5091 100644 +index 9fc576dde5..091d2bc7b4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -83,7 +83,19 @@ public class TestSnapshotSelection { +@@ -83,7 +83,20 @@ public class TestSnapshotSelection { @BeforeClass public static void startSpark() { @@ -1347,6 +1366,7 @@ index 9fc576dde5..69a49f5091 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1354,10 +1374,10 @@ index 9fc576dde5..69a49f5091 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 16fde3c954..26e94f6dde 100644 +index 16fde3c954..312ad3485e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -121,7 +121,19 @@ public class TestSparkDataFile { +@@ -121,7 +121,20 @@ public class TestSparkDataFile { @BeforeClass public static void startSpark() { @@ -1372,6 +1392,7 @@ index 16fde3c954..26e94f6dde 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1379,10 +1400,10 @@ index 16fde3c954..26e94f6dde 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 63c18277aa..971f14b647 100644 +index 63c18277aa..4e8172f96d 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -89,7 +89,19 @@ public class TestSparkDataWrite { +@@ -89,7 +89,20 @@ public class TestSparkDataWrite { @BeforeClass public static void startSpark() { @@ -1397,13 +1418,14 @@ index 63c18277aa..971f14b647 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @Parameterized.AfterParam -@@ -138,7 +150,7 @@ public class TestSparkDataWrite { +@@ -138,7 +151,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1412,7 +1434,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); for (ManifestFile manifest : -@@ -208,7 +220,7 @@ public class TestSparkDataWrite { +@@ -208,7 +221,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1421,7 +1443,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -254,7 +266,7 @@ public class TestSparkDataWrite { +@@ -254,7 +267,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1430,7 +1452,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -307,7 +319,7 @@ public class TestSparkDataWrite { +@@ -307,7 +320,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1439,7 +1461,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -350,7 +362,7 @@ public class TestSparkDataWrite { +@@ -350,7 +363,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1448,7 +1470,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -390,7 +402,7 @@ public class TestSparkDataWrite { +@@ -390,7 +403,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1457,7 +1479,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -455,7 +467,7 @@ public class TestSparkDataWrite { +@@ -455,7 +468,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1466,7 +1488,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -619,7 +631,7 @@ public class TestSparkDataWrite { +@@ -619,7 +632,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1475,7 +1497,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -705,7 +717,7 @@ public class TestSparkDataWrite { +@@ -705,7 +718,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1485,10 +1507,10 @@ index 63c18277aa..971f14b647 100644 "Number of rows should match", records.size() + records2.size(), actual.size()); assertThat(actual) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 3a4b235c46..7633a1792d 100644 +index 3a4b235c46..994741c542 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -86,7 +86,19 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -86,7 +86,20 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeClass public static void startSpark() { @@ -1503,6 +1525,7 @@ index 3a4b235c46..7633a1792d 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1510,10 +1533,10 @@ index 3a4b235c46..7633a1792d 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index dda49b4946..69d3a0d24f 100644 +index dda49b4946..e4ee109148 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -131,7 +131,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -131,7 +131,27 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1524,6 +1547,7 @@ index dda49b4946..69d3a0d24f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1534,12 +1558,13 @@ index dda49b4946..69d3a0d24f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = -@@ -199,7 +217,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -199,7 +219,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -1550,10 +1575,10 @@ index dda49b4946..69d3a0d24f 100644 @Override diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index e5831b76e4..2433013d3e 100644 +index e5831b76e4..115635a0a5 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -176,7 +176,25 @@ public class TestSparkReaderWithBloomFilter { +@@ -176,7 +176,27 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1564,6 +1589,7 @@ index e5831b76e4..2433013d3e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1574,16 +1600,17 @@ index e5831b76e4..2433013d3e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index f420f1b955..21273826ba 100644 +index f420f1b955..21e229ebbc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -66,6 +66,15 @@ public class TestStructuredStreaming { +@@ -66,6 +66,16 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -1594,16 +1621,17 @@ index f420f1b955..21273826ba 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index ac674e2e62..c18fed7cfb 100644 +index ac674e2e62..29a936c888 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -73,7 +73,19 @@ public class TestTimestampWithoutZone extends SparkTestBase { +@@ -73,7 +73,20 @@ public class TestTimestampWithoutZone extends SparkTestBase { @BeforeClass public static void startSpark() { @@ -1618,6 +1646,7 @@ index ac674e2e62..c18fed7cfb 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1625,10 +1654,10 @@ index ac674e2e62..c18fed7cfb 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 73827b309b..ace66913d1 100644 +index 73827b309b..f8ff9a51b3 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,20 @@ public class TestWriteMetricsConfig { @BeforeClass public static void startSpark() { @@ -1643,6 +1672,7 @@ index 73827b309b..ace66913d1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1650,10 +1680,10 @@ index 73827b309b..ace66913d1 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 1a4e2f3e1c..047e0f1fb4 100644 +index 1a4e2f3e1c..6977221c3f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -65,7 +65,25 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { +@@ -65,7 +65,27 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -1664,6 +1694,7 @@ index 1a4e2f3e1c..047e0f1fb4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1674,6 +1705,7 @@ index 1a4e2f3e1c..047e0f1fb4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1738,10 +1770,10 @@ index e2d2c7a7ac..95cd3c08b5 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 578845e3da..02b6dc270f 100644 +index 578845e3da..047980e7fd 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -57,6 +57,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -57,6 +57,16 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -1752,16 +1784,17 @@ index 578845e3da..02b6dc270f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index ade19de36f..579b68e73f 100644 +index ade19de36f..6257c9efef 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,15 @@ public class TestCallStatementParser { +@@ -56,6 +56,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -1772,6 +1805,7 @@ index ade19de36f..579b68e73f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1806,10 +1840,10 @@ index 8cb92224fc..777b9bb1dd 100644 } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 64edb1002e..57767da086 100644 +index 64edb1002e..fc1d0f01ae 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -179,6 +179,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -1820,16 +1854,17 @@ index 64edb1002e..57767da086 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index a5d0456b0b..f52c387943 100644 +index a5d0456b0b..4dd831bc29 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { +@@ -392,6 +392,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -1840,16 +1875,17 @@ index a5d0456b0b..f52c387943 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index c6794e43c6..4c1a165a44 100644 +index c6794e43c6..90bde88cd5 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -239,6 +239,15 @@ public class DVReaderBenchmark { +@@ -239,6 +239,16 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1860,16 +1896,17 @@ index c6794e43c6..4c1a165a44 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index ac74fb5a10..4aa5d5b361 100644 +index ac74fb5a10..07b8c34f8a 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -223,6 +223,15 @@ public class DVWriterBenchmark { +@@ -223,6 +223,16 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1880,16 +1917,17 @@ index ac74fb5a10..4aa5d5b361 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -1904,6 +1942,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -2707,10 +2746,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2721,16 +2760,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 659507e4c5..2a3a87f079 100644 +index 659507e4c5..d44910e809 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2741,16 +2781,17 @@ index 659507e4c5..2a3a87f079 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2761,16 +2802,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index 2665d7ba8d..c24c82341a 100644 +index 2665d7ba8d..ef17afe769 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2781,16 +2823,17 @@ index 2665d7ba8d..c24c82341a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index de68351f6e..dea8226742 100644 +index de68351f6e..c7848c59ae 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -77,6 +77,15 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -77,6 +77,16 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -2801,16 +2844,17 @@ index de68351f6e..dea8226742 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -index bc4e722bc8..57770308b3 100644 +index bc4e722bc8..e24dc5f275 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -@@ -59,7 +59,19 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -59,7 +59,20 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect @BeforeAll public static void startSpark() { @@ -2825,6 +2869,7 @@ index bc4e722bc8..57770308b3 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2832,10 +2877,10 @@ index bc4e722bc8..57770308b3 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 3a269740b7..fc63449215 100644 +index 3a269740b7..a0f6341fbd 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -54,7 +54,19 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -54,7 +54,20 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -2850,6 +2895,7 @@ index 3a269740b7..fc63449215 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2857,10 +2903,10 @@ index 3a269740b7..fc63449215 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index f411920a5d..ec8eebe117 100644 +index f411920a5d..e57eb3a5b4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,20 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -2875,6 +2921,7 @@ index f411920a5d..ec8eebe117 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2882,10 +2929,10 @@ index f411920a5d..ec8eebe117 100644 @BeforeEach diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index c4ba96e634..dcf72c19b6 100644 +index c4ba96e634..db6acd8124 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -75,7 +75,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -75,7 +75,20 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -2900,6 +2947,7 @@ index c4ba96e634..dcf72c19b6 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2907,10 +2955,10 @@ index c4ba96e634..dcf72c19b6 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index 348173596e..03156f786b 100644 +index 348173596e..eb1194f709 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -110,7 +110,19 @@ public class TestFilteredScan { +@@ -110,7 +110,20 @@ public class TestFilteredScan { @BeforeAll public static void startSpark() { @@ -2925,6 +2973,7 @@ index 348173596e..03156f786b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2932,10 +2981,10 @@ index 348173596e..03156f786b 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 84c99a575c..e647ab924e 100644 +index 84c99a575c..84d44e4478 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -93,7 +93,19 @@ public class TestForwardCompatibility { +@@ -93,7 +93,20 @@ public class TestForwardCompatibility { @BeforeAll public static void startSpark() { @@ -2950,6 +2999,7 @@ index 84c99a575c..e647ab924e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2957,10 +3007,10 @@ index 84c99a575c..e647ab924e 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 7eff93d204..2fdc87a0a8 100644 +index 7eff93d204..a0a2a0845b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,19 @@ public class TestIcebergSpark { +@@ -46,7 +46,20 @@ public class TestIcebergSpark { @BeforeAll public static void startSpark() { @@ -2975,6 +3025,7 @@ index 7eff93d204..2fdc87a0a8 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2982,10 +3033,10 @@ index 7eff93d204..2fdc87a0a8 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 9464f687b0..8653560334 100644 +index 9464f687b0..88edb4b6a0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -112,7 +112,19 @@ public class TestPartitionPruning { +@@ -112,7 +112,20 @@ public class TestPartitionPruning { @BeforeAll public static void startSpark() { @@ -3000,6 +3051,7 @@ index 9464f687b0..8653560334 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3007,10 +3059,10 @@ index 9464f687b0..8653560334 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 5c218f21c4..2f17343d59 100644 +index 5c218f21c4..bc0478daa0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -107,7 +107,19 @@ public class TestPartitionValues { +@@ -107,7 +107,20 @@ public class TestPartitionValues { @BeforeAll public static void startSpark() { @@ -3025,6 +3077,7 @@ index 5c218f21c4..2f17343d59 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3032,10 +3085,10 @@ index 5c218f21c4..2f17343d59 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index a7334a580c..68210dc8e8 100644 +index a7334a580c..7eebdc2239 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -87,7 +87,19 @@ public class TestSnapshotSelection { +@@ -87,7 +87,20 @@ public class TestSnapshotSelection { @BeforeAll public static void startSpark() { @@ -3050,6 +3103,7 @@ index a7334a580c..68210dc8e8 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3057,10 +3111,10 @@ index a7334a580c..68210dc8e8 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 182b1ef8f5..76e04b1472 100644 +index 182b1ef8f5..628f14571c 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -120,7 +120,19 @@ public class TestSparkDataFile { +@@ -120,7 +120,20 @@ public class TestSparkDataFile { @BeforeAll public static void startSpark() { @@ -3075,6 +3129,7 @@ index 182b1ef8f5..76e04b1472 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3082,10 +3137,10 @@ index 182b1ef8f5..76e04b1472 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index fb2b312bed..846f50e0c9 100644 +index fb2b312bed..03fc0d2171 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -96,7 +96,19 @@ public class TestSparkDataWrite { +@@ -96,7 +96,20 @@ public class TestSparkDataWrite { @BeforeAll public static void startSpark() { @@ -3100,13 +3155,14 @@ index fb2b312bed..846f50e0c9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @AfterEach -@@ -140,7 +152,7 @@ public class TestSparkDataWrite { +@@ -140,7 +153,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3115,7 +3171,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); for (ManifestFile manifest : -@@ -210,7 +222,7 @@ public class TestSparkDataWrite { +@@ -210,7 +223,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3124,7 +3180,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -256,7 +268,7 @@ public class TestSparkDataWrite { +@@ -256,7 +269,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3133,7 +3189,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -309,7 +321,7 @@ public class TestSparkDataWrite { +@@ -309,7 +322,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3142,7 +3198,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -352,7 +364,7 @@ public class TestSparkDataWrite { +@@ -352,7 +365,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3151,7 +3207,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -392,7 +404,7 @@ public class TestSparkDataWrite { +@@ -392,7 +405,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3160,7 +3216,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -458,7 +470,7 @@ public class TestSparkDataWrite { +@@ -458,7 +471,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3169,7 +3225,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -622,7 +634,7 @@ public class TestSparkDataWrite { +@@ -622,7 +635,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3178,7 +3234,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -708,7 +720,7 @@ public class TestSparkDataWrite { +@@ -708,7 +721,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3188,10 +3244,10 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual) .describedAs("Result rows should match") diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index becf6a064d..91451c268e 100644 +index becf6a064d..f8e3729a0e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -83,7 +83,19 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -83,7 +83,20 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeAll public static void startSpark() { @@ -3206,6 +3262,7 @@ index becf6a064d..91451c268e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3213,10 +3270,10 @@ index becf6a064d..91451c268e 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 4f1cef5d37..4a1a602ed4 100644 +index 4f1cef5d37..db8738e14e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -136,6 +136,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -136,6 +136,16 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3227,12 +3284,13 @@ index 4f1cef5d37..4a1a602ed4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); -@@ -204,7 +213,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -204,7 +214,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -3243,10 +3301,10 @@ index 4f1cef5d37..4a1a602ed4 100644 @Override diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..b58321b761 100644 +index baf7fa8f88..2eaa17e8b0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,6 +182,16 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3257,16 +3315,17 @@ index baf7fa8f88..b58321b761 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 17db46b85c..d424b97d2a 100644 +index 17db46b85c..f103f6ce6b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -65,6 +65,15 @@ public class TestStructuredStreaming { +@@ -65,6 +65,16 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -3277,16 +3336,17 @@ index 17db46b85c..d424b97d2a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 306444b9f2..e4082d9cd9 100644 +index 306444b9f2..4a72089a97 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,20 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -3301,6 +3361,7 @@ index 306444b9f2..e4082d9cd9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3308,10 +3369,10 @@ index 306444b9f2..e4082d9cd9 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 841268a6be..97f7187314 100644 +index 841268a6be..1ca5f299b4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,20 @@ public class TestWriteMetricsConfig { @BeforeAll public static void startSpark() { @@ -3326,6 +3387,7 @@ index 841268a6be..97f7187314 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3333,10 +3395,10 @@ index 841268a6be..97f7187314 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 6e09252704..4b0f2e4d58 100644 +index 6e09252704..1cd8b636f3 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -60,6 +60,15 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -60,6 +60,16 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -3347,6 +3409,7 @@ index 6e09252704..4b0f2e4d58 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() diff --git a/dev/diffs/iceberg/1.9.1.diff b/dev/diffs/iceberg/1.9.1.diff index 139003cae3..6aa3633532 100644 --- a/dev/diffs/iceberg/1.9.1.diff +++ b/dev/diffs/iceberg/1.9.1.diff @@ -41,10 +41,10 @@ index 2cd8666892..4077116abb 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 578845e3da..19a200892f 100644 +index 578845e3da..5ce5db4db8 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -57,7 +57,25 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -57,7 +57,27 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -55,6 +55,7 @@ index 578845e3da..19a200892f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -65,16 +66,17 @@ index 578845e3da..19a200892f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestBase.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index 58d054bd05..ef1c1cb6d1 100644 +index 58d054bd05..096985a6b5 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,15 @@ public class TestCallStatementParser { +@@ -56,6 +56,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -85,6 +87,7 @@ index 58d054bd05..ef1c1cb6d1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -119,10 +122,10 @@ index 5a0a04edba..afe92a8b72 100644 } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index b6ade2bff3..90a4839dad 100644 +index b6ade2bff3..af28e6bb33 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -170,6 +170,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -133,16 +136,17 @@ index b6ade2bff3..90a4839dad 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index b08c352819..63880b089d 100644 +index b08c352819..de091c0b09 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { +@@ -386,6 +386,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -153,16 +157,17 @@ index b08c352819..63880b089d 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -177,6 +182,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -964,10 +970,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -978,16 +984,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 9361c63176..16ce1c614f 100644 +index 9361c63176..5c9fb10e62 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -69,6 +69,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -998,16 +1005,17 @@ index 9361c63176..16ce1c614f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1018,16 +1026,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index acd4688440..33732e6654 100644 +index acd4688440..4f7a9040a7 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting +@@ -59,6 +59,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1038,16 +1047,17 @@ index acd4688440..33732e6654 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -index 3e8953fb95..51ce392476 100644 +index 3e8953fb95..2b021f0226 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -@@ -77,6 +77,15 @@ public abstract class SparkTestBase extends SparkTestHelperBase { +@@ -77,6 +77,16 @@ public abstract class SparkTestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -1058,16 +1068,17 @@ index 3e8953fb95..51ce392476 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 06d5e0c44f..effc2bcc69 100644 +index 06d5e0c44f..b529ca3271 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -57,7 +57,19 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -57,7 +57,20 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -1082,6 +1093,7 @@ index 06d5e0c44f..effc2bcc69 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1089,10 +1101,10 @@ index 06d5e0c44f..effc2bcc69 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 26cb4dcc95..d185f1175b 100644 +index 26cb4dcc95..8fd2d28c94 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -104,7 +104,19 @@ public class TestCompressionSettings extends SparkCatalogTestBase { +@@ -104,7 +104,20 @@ public class TestCompressionSettings extends SparkCatalogTestBase { @BeforeClass public static void startSpark() { @@ -1107,6 +1119,7 @@ index 26cb4dcc95..d185f1175b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1114,10 +1127,10 @@ index 26cb4dcc95..d185f1175b 100644 @Before diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index 013b8d4386..007b1af65b 100644 +index 013b8d4386..e112c78cfc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -76,7 +76,19 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { +@@ -76,7 +76,20 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { @BeforeClass public static void startSpark() { @@ -1132,6 +1145,7 @@ index 013b8d4386..007b1af65b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1139,10 +1153,10 @@ index 013b8d4386..007b1af65b 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index ba13d005bd..b9778c1986 100644 +index ba13d005bd..a426d6e79e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -118,7 +118,19 @@ public class TestFilteredScan { +@@ -118,7 +118,20 @@ public class TestFilteredScan { @BeforeClass public static void startSpark() { @@ -1157,6 +1171,7 @@ index ba13d005bd..b9778c1986 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1164,10 +1179,10 @@ index ba13d005bd..b9778c1986 100644 // define UDFs used by partition tests Function bucket4 = Transforms.bucket(4).bind(Types.LongType.get()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 9f97753094..fb5bfbad1b 100644 +index 9f97753094..39e19af0b0 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -94,7 +94,19 @@ public class TestForwardCompatibility { +@@ -94,7 +94,20 @@ public class TestForwardCompatibility { @BeforeClass public static void startSpark() { @@ -1182,6 +1197,7 @@ index 9f97753094..fb5bfbad1b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1189,10 +1205,10 @@ index 9f97753094..fb5bfbad1b 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 0154506f86..12e916cbe4 100644 +index 0154506f86..b57e0bb01b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,19 @@ public class TestIcebergSpark { +@@ -46,7 +46,20 @@ public class TestIcebergSpark { @BeforeClass public static void startSpark() { @@ -1207,6 +1223,7 @@ index 0154506f86..12e916cbe4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1214,10 +1231,10 @@ index 0154506f86..12e916cbe4 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 639d37c793..5e397909d4 100644 +index 639d37c793..6d033d2634 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -111,7 +111,19 @@ public class TestPartitionPruning { +@@ -111,7 +111,20 @@ public class TestPartitionPruning { @BeforeClass public static void startSpark() { @@ -1232,6 +1249,7 @@ index 639d37c793..5e397909d4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1239,10 +1257,10 @@ index 639d37c793..5e397909d4 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index ad0984ef42..99af33c86c 100644 +index ad0984ef42..3333372cf2 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -104,7 +104,19 @@ public class TestPartitionValues { +@@ -104,7 +104,20 @@ public class TestPartitionValues { @BeforeClass public static void startSpark() { @@ -1257,6 +1275,7 @@ index ad0984ef42..99af33c86c 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1264,10 +1283,10 @@ index ad0984ef42..99af33c86c 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 9fc576dde5..69a49f5091 100644 +index 9fc576dde5..091d2bc7b4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -83,7 +83,19 @@ public class TestSnapshotSelection { +@@ -83,7 +83,20 @@ public class TestSnapshotSelection { @BeforeClass public static void startSpark() { @@ -1282,6 +1301,7 @@ index 9fc576dde5..69a49f5091 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1289,10 +1309,10 @@ index 9fc576dde5..69a49f5091 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 16fde3c954..26e94f6dde 100644 +index 16fde3c954..312ad3485e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -121,7 +121,19 @@ public class TestSparkDataFile { +@@ -121,7 +121,20 @@ public class TestSparkDataFile { @BeforeClass public static void startSpark() { @@ -1307,6 +1327,7 @@ index 16fde3c954..26e94f6dde 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1314,10 +1335,10 @@ index 16fde3c954..26e94f6dde 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 63c18277aa..971f14b647 100644 +index 63c18277aa..4e8172f96d 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -89,7 +89,19 @@ public class TestSparkDataWrite { +@@ -89,7 +89,20 @@ public class TestSparkDataWrite { @BeforeClass public static void startSpark() { @@ -1332,13 +1353,14 @@ index 63c18277aa..971f14b647 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @Parameterized.AfterParam -@@ -138,7 +150,7 @@ public class TestSparkDataWrite { +@@ -138,7 +151,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1347,7 +1369,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); for (ManifestFile manifest : -@@ -208,7 +220,7 @@ public class TestSparkDataWrite { +@@ -208,7 +221,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1356,7 +1378,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -254,7 +266,7 @@ public class TestSparkDataWrite { +@@ -254,7 +267,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1365,7 +1387,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -307,7 +319,7 @@ public class TestSparkDataWrite { +@@ -307,7 +320,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1374,7 +1396,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -350,7 +362,7 @@ public class TestSparkDataWrite { +@@ -350,7 +363,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1383,7 +1405,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -390,7 +402,7 @@ public class TestSparkDataWrite { +@@ -390,7 +403,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1392,7 +1414,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -455,7 +467,7 @@ public class TestSparkDataWrite { +@@ -455,7 +468,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1401,7 +1423,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -619,7 +631,7 @@ public class TestSparkDataWrite { +@@ -619,7 +632,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1410,7 +1432,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -705,7 +717,7 @@ public class TestSparkDataWrite { +@@ -705,7 +718,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1420,10 +1442,10 @@ index 63c18277aa..971f14b647 100644 "Number of rows should match", records.size() + records2.size(), actual.size()); assertThat(actual) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 584a6b1c70..b93a23d6f1 100644 +index 584a6b1c70..823f0f910f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -86,7 +86,19 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -86,7 +86,20 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeClass public static void startSpark() { @@ -1438,6 +1460,7 @@ index 584a6b1c70..b93a23d6f1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1445,10 +1468,10 @@ index 584a6b1c70..b93a23d6f1 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index dda49b4946..69d3a0d24f 100644 +index dda49b4946..e4ee109148 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -131,7 +131,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -131,7 +131,27 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1459,6 +1482,7 @@ index dda49b4946..69d3a0d24f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1469,12 +1493,13 @@ index dda49b4946..69d3a0d24f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = -@@ -199,7 +217,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -199,7 +219,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -1485,10 +1510,10 @@ index dda49b4946..69d3a0d24f 100644 @Override diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index e5831b76e4..2433013d3e 100644 +index e5831b76e4..115635a0a5 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -176,7 +176,25 @@ public class TestSparkReaderWithBloomFilter { +@@ -176,7 +176,27 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1499,6 +1524,7 @@ index e5831b76e4..2433013d3e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1509,16 +1535,17 @@ index e5831b76e4..2433013d3e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 961d69b721..f12ac3e83b 100644 +index 961d69b721..fc9a0bffb8 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -68,6 +68,15 @@ public class TestStructuredStreaming { +@@ -68,6 +68,16 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -1529,16 +1556,17 @@ index 961d69b721..f12ac3e83b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index ac674e2e62..c18fed7cfb 100644 +index ac674e2e62..29a936c888 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -73,7 +73,19 @@ public class TestTimestampWithoutZone extends SparkTestBase { +@@ -73,7 +73,20 @@ public class TestTimestampWithoutZone extends SparkTestBase { @BeforeClass public static void startSpark() { @@ -1553,6 +1581,7 @@ index ac674e2e62..c18fed7cfb 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1560,10 +1589,10 @@ index ac674e2e62..c18fed7cfb 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 73827b309b..ace66913d1 100644 +index 73827b309b..f8ff9a51b3 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,20 @@ public class TestWriteMetricsConfig { @BeforeClass public static void startSpark() { @@ -1578,6 +1607,7 @@ index 73827b309b..ace66913d1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1585,10 +1615,10 @@ index 73827b309b..ace66913d1 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 1a4e2f3e1c..047e0f1fb4 100644 +index 1a4e2f3e1c..6977221c3f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -65,7 +65,25 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { +@@ -65,7 +65,27 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -1599,6 +1629,7 @@ index 1a4e2f3e1c..047e0f1fb4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1609,6 +1640,7 @@ index 1a4e2f3e1c..047e0f1fb4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1673,10 +1705,10 @@ index 572c32f929..29013d26c0 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 578845e3da..02b6dc270f 100644 +index 578845e3da..047980e7fd 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -57,6 +57,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -57,6 +57,16 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -1687,6 +1719,7 @@ index 578845e3da..02b6dc270f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1707,10 +1740,10 @@ index b5d6415763..0763a723d3 100644 protected void createTableWithDeleteGranularity( diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index ecf9e6f8a5..beca5f8521 100644 +index ecf9e6f8a5..42591bd9b2 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,15 @@ public class TestCallStatementParser { +@@ -56,6 +56,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -1721,6 +1754,7 @@ index ecf9e6f8a5..beca5f8521 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1755,10 +1789,10 @@ index 8cb92224fc..777b9bb1dd 100644 } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 64edb1002e..57767da086 100644 +index 64edb1002e..fc1d0f01ae 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -179,6 +179,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -1769,16 +1803,17 @@ index 64edb1002e..57767da086 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index a5d0456b0b..f52c387943 100644 +index a5d0456b0b..4dd831bc29 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { +@@ -392,6 +392,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -1789,16 +1824,17 @@ index a5d0456b0b..f52c387943 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index c6794e43c6..4c1a165a44 100644 +index c6794e43c6..90bde88cd5 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -239,6 +239,15 @@ public class DVReaderBenchmark { +@@ -239,6 +239,16 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1809,16 +1845,17 @@ index c6794e43c6..4c1a165a44 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index ac74fb5a10..4aa5d5b361 100644 +index ac74fb5a10..07b8c34f8a 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -223,6 +223,15 @@ public class DVWriterBenchmark { +@@ -223,6 +223,16 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1829,16 +1866,17 @@ index ac74fb5a10..4aa5d5b361 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -1853,6 +1891,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -2640,10 +2679,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2654,16 +2693,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 659507e4c5..2a3a87f079 100644 +index 659507e4c5..d44910e809 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2674,16 +2714,17 @@ index 659507e4c5..2a3a87f079 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2694,16 +2735,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index 2665d7ba8d..c24c82341a 100644 +index 2665d7ba8d..ef17afe769 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2714,16 +2756,17 @@ index 2665d7ba8d..c24c82341a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index 3e9f3334ef..0f96c07739 100644 +index 3e9f3334ef..3f7851c263 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -77,6 +77,15 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -77,6 +77,16 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -2734,6 +2777,7 @@ index 3e9f3334ef..0f96c07739 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -2765,10 +2809,10 @@ index a31138ae01..501c8ac93e 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -index bc4e722bc8..57770308b3 100644 +index bc4e722bc8..e24dc5f275 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -@@ -59,7 +59,19 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -59,7 +59,20 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect @BeforeAll public static void startSpark() { @@ -2783,6 +2827,7 @@ index bc4e722bc8..57770308b3 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2790,10 +2835,10 @@ index bc4e722bc8..57770308b3 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 0886df957d..eca0f74533 100644 +index 0886df957d..ee1f955d55 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -57,7 +57,19 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -57,7 +57,20 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -2808,6 +2853,7 @@ index 0886df957d..eca0f74533 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2815,10 +2861,10 @@ index 0886df957d..eca0f74533 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 6b7d861364..23419aa9b1 100644 +index 6b7d861364..82563ec088 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,20 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -2833,6 +2879,7 @@ index 6b7d861364..23419aa9b1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2840,10 +2887,10 @@ index 6b7d861364..23419aa9b1 100644 @BeforeEach diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index c4ba96e634..dcf72c19b6 100644 +index c4ba96e634..db6acd8124 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -75,7 +75,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -75,7 +75,20 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -2858,6 +2905,7 @@ index c4ba96e634..dcf72c19b6 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2865,10 +2913,10 @@ index c4ba96e634..dcf72c19b6 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index 348173596e..03156f786b 100644 +index 348173596e..eb1194f709 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -110,7 +110,19 @@ public class TestFilteredScan { +@@ -110,7 +110,20 @@ public class TestFilteredScan { @BeforeAll public static void startSpark() { @@ -2883,6 +2931,7 @@ index 348173596e..03156f786b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2890,10 +2939,10 @@ index 348173596e..03156f786b 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 84c99a575c..e647ab924e 100644 +index 84c99a575c..84d44e4478 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -93,7 +93,19 @@ public class TestForwardCompatibility { +@@ -93,7 +93,20 @@ public class TestForwardCompatibility { @BeforeAll public static void startSpark() { @@ -2908,6 +2957,7 @@ index 84c99a575c..e647ab924e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2915,10 +2965,10 @@ index 84c99a575c..e647ab924e 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 7eff93d204..2fdc87a0a8 100644 +index 7eff93d204..a0a2a0845b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,19 @@ public class TestIcebergSpark { +@@ -46,7 +46,20 @@ public class TestIcebergSpark { @BeforeAll public static void startSpark() { @@ -2933,6 +2983,7 @@ index 7eff93d204..2fdc87a0a8 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2940,10 +2991,10 @@ index 7eff93d204..2fdc87a0a8 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 9464f687b0..8653560334 100644 +index 9464f687b0..88edb4b6a0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -112,7 +112,19 @@ public class TestPartitionPruning { +@@ -112,7 +112,20 @@ public class TestPartitionPruning { @BeforeAll public static void startSpark() { @@ -2958,6 +3009,7 @@ index 9464f687b0..8653560334 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2965,10 +3017,10 @@ index 9464f687b0..8653560334 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 5c218f21c4..2f17343d59 100644 +index 5c218f21c4..bc0478daa0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -107,7 +107,19 @@ public class TestPartitionValues { +@@ -107,7 +107,20 @@ public class TestPartitionValues { @BeforeAll public static void startSpark() { @@ -2983,6 +3035,7 @@ index 5c218f21c4..2f17343d59 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2990,10 +3043,10 @@ index 5c218f21c4..2f17343d59 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index a7334a580c..68210dc8e8 100644 +index a7334a580c..7eebdc2239 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -87,7 +87,19 @@ public class TestSnapshotSelection { +@@ -87,7 +87,20 @@ public class TestSnapshotSelection { @BeforeAll public static void startSpark() { @@ -3008,6 +3061,7 @@ index a7334a580c..68210dc8e8 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3015,10 +3069,10 @@ index a7334a580c..68210dc8e8 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 182b1ef8f5..76e04b1472 100644 +index 182b1ef8f5..628f14571c 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -120,7 +120,19 @@ public class TestSparkDataFile { +@@ -120,7 +120,20 @@ public class TestSparkDataFile { @BeforeAll public static void startSpark() { @@ -3033,6 +3087,7 @@ index 182b1ef8f5..76e04b1472 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3040,10 +3095,10 @@ index 182b1ef8f5..76e04b1472 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index fb2b312bed..846f50e0c9 100644 +index fb2b312bed..03fc0d2171 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -96,7 +96,19 @@ public class TestSparkDataWrite { +@@ -96,7 +96,20 @@ public class TestSparkDataWrite { @BeforeAll public static void startSpark() { @@ -3058,13 +3113,14 @@ index fb2b312bed..846f50e0c9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @AfterEach -@@ -140,7 +152,7 @@ public class TestSparkDataWrite { +@@ -140,7 +153,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3073,7 +3129,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); for (ManifestFile manifest : -@@ -210,7 +222,7 @@ public class TestSparkDataWrite { +@@ -210,7 +223,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3082,7 +3138,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -256,7 +268,7 @@ public class TestSparkDataWrite { +@@ -256,7 +269,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3091,7 +3147,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -309,7 +321,7 @@ public class TestSparkDataWrite { +@@ -309,7 +322,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3100,7 +3156,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -352,7 +364,7 @@ public class TestSparkDataWrite { +@@ -352,7 +365,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3109,7 +3165,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -392,7 +404,7 @@ public class TestSparkDataWrite { +@@ -392,7 +405,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3118,7 +3174,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -458,7 +470,7 @@ public class TestSparkDataWrite { +@@ -458,7 +471,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3127,7 +3183,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -622,7 +634,7 @@ public class TestSparkDataWrite { +@@ -622,7 +635,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3136,7 +3192,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -708,7 +720,7 @@ public class TestSparkDataWrite { +@@ -708,7 +721,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3146,10 +3202,10 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual) .describedAs("Result rows should match") diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index becf6a064d..91451c268e 100644 +index becf6a064d..f8e3729a0e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -83,7 +83,19 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -83,7 +83,20 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeAll public static void startSpark() { @@ -3164,6 +3220,7 @@ index becf6a064d..91451c268e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3171,10 +3228,10 @@ index becf6a064d..91451c268e 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 4f1cef5d37..4a1a602ed4 100644 +index 4f1cef5d37..db8738e14e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -136,6 +136,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -136,6 +136,16 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3185,12 +3242,13 @@ index 4f1cef5d37..4a1a602ed4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); -@@ -204,7 +213,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -204,7 +214,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -3201,10 +3259,10 @@ index 4f1cef5d37..4a1a602ed4 100644 @Override diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..b58321b761 100644 +index baf7fa8f88..2eaa17e8b0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,6 +182,16 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3215,16 +3273,17 @@ index baf7fa8f88..b58321b761 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index c84a65cbe9..e17ba73ee4 100644 +index c84a65cbe9..9bb31efc35 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -67,6 +67,15 @@ public class TestStructuredStreaming { +@@ -67,6 +67,16 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -3235,16 +3294,17 @@ index c84a65cbe9..e17ba73ee4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 306444b9f2..e4082d9cd9 100644 +index 306444b9f2..4a72089a97 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,20 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -3259,6 +3319,7 @@ index 306444b9f2..e4082d9cd9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3266,10 +3327,10 @@ index 306444b9f2..e4082d9cd9 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 841268a6be..97f7187314 100644 +index 841268a6be..1ca5f299b4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,20 @@ public class TestWriteMetricsConfig { @BeforeAll public static void startSpark() { @@ -3284,6 +3345,7 @@ index 841268a6be..97f7187314 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3291,10 +3353,10 @@ index 841268a6be..97f7187314 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 6e09252704..4b0f2e4d58 100644 +index 6e09252704..1cd8b636f3 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -60,6 +60,15 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -60,6 +60,16 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -3305,6 +3367,7 @@ index 6e09252704..4b0f2e4d58 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() diff --git a/docs/source/contributor-guide/iceberg-spark-tests.md b/docs/source/contributor-guide/iceberg-spark-tests.md index df6c168af6..e91261695f 100644 --- a/docs/source/contributor-guide/iceberg-spark-tests.md +++ b/docs/source/contributor-guide/iceberg-spark-tests.md @@ -32,9 +32,14 @@ Here is an overview of the changes that the diffs make to Iceberg: - Configure test base classes (`TestBase`, `ExtensionsTestBase`, `ScanTestBase`, etc.) to load the Comet Spark plugin and shuffle manager - Enable the Iceberg write split-operator plan (`spark.comet.write.iceberg.splitOperator.enabled`) alongside the - native scan in every Comet-configured session. The flag is off by default for users, so Iceberg's own suites - are the only place the split plan (`IcebergCommit -> IcebergWrite`) is exercised against Iceberg's write, - commit, and row-level-operation tests. See [#5259] + native scan in every Comet-configured session, so that Iceberg's own write, commit, and row-level-operation + tests exercise the split plan (`IcebergCommit -> IcebergWrite`). See [#5259] +- Enable `spark.comet.exec.localTableScan.enabled` in the same sessions. `CometIcebergNativeWrite` sets + `requiresNativeChildren`, so without this flag a write fed by an inline `VALUES` list keeps Spark's row-based + `LocalTableScanExec`, the conversion is declined, and the write silently runs on the JVM writer. Many Iceberg + suites seed their data that way, so leaving it off hides the native writer from most of the write surface. +- Enable fallback logging (`spark.comet.explainFallback.enabled`) so that every operator Comet declines is + reported in the test output together with the reason it was declined. [#3739]: https://github.com/apache/datafusion-comet/pull/3739 [#5259]: https://github.com/apache/datafusion-comet/issues/5259