diff --git a/dev/diffs/iceberg/1.10.0.diff b/dev/diffs/iceberg/1.10.0.diff index 7bbf584c31..4d9bf0c3a5 100644 --- a/dev/diffs/iceberg/1.10.0.diff +++ b/dev/diffs/iceberg/1.10.0.diff @@ -24,10 +24,10 @@ index 714be0831d..dede65f213 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 4c1a509591..186a264a88 100644 +index 4c1a509591..51f9244f10 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -59,7 +59,25 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -59,7 +59,27 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -38,6 +38,7 @@ index 4c1a509591..186a264a88 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -48,16 +49,17 @@ index 4c1a509591..186a264a88 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestBase.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index 58d054bd05..ef1c1cb6d1 100644 +index 58d054bd05..096985a6b5 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,15 @@ public class TestCallStatementParser { +@@ -56,6 +56,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -68,6 +70,7 @@ index 58d054bd05..ef1c1cb6d1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -102,10 +105,10 @@ index 5a0a04edba..afe92a8b72 100644 } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index b6ade2bff3..90a4839dad 100644 +index b6ade2bff3..af28e6bb33 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -170,6 +170,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -116,16 +119,17 @@ index b6ade2bff3..90a4839dad 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index 7cd41bd7e7..0b6d490529 100644 +index 7cd41bd7e7..b3fa94f225 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { +@@ -386,6 +386,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -136,16 +140,17 @@ index 7cd41bd7e7..0b6d490529 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -160,6 +165,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -947,10 +953,10 @@ index a0f45e7610..78e7845911 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -961,16 +967,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 9361c63176..16ce1c614f 100644 +index 9361c63176..5c9fb10e62 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -69,6 +69,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -981,16 +988,17 @@ index 9361c63176..16ce1c614f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1001,16 +1009,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index acd4688440..33732e6654 100644 +index acd4688440..4f7a9040a7 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting +@@ -59,6 +59,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1021,16 +1030,17 @@ index acd4688440..33732e6654 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index 3c32b46936..2a39c4605c 100644 +index 3c32b46936..1b14b79784 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -79,6 +79,15 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -79,6 +79,16 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -1041,16 +1051,17 @@ index 3c32b46936..2a39c4605c 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 6647a1b483..aeb16f85ac 100644 +index 6647a1b483..92ece0d94f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -62,6 +62,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { +@@ -62,6 +62,16 @@ public abstract class ScanTestBase extends AvroDataTestBase { SparkSession.builder() .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .master("local[2]") @@ -1061,16 +1072,17 @@ index 6647a1b483..aeb16f85ac 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 24a14bb64d..9329cd99b3 100644 +index 24a14bb64d..4cab0ae7e1 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,20 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -1085,6 +1097,7 @@ index 24a14bb64d..9329cd99b3 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1092,10 +1105,10 @@ index 24a14bb64d..9329cd99b3 100644 @BeforeEach diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index 99d3f38ee7..f1a74280dd 100644 +index 99d3f38ee7..db87f1904b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -74,7 +74,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -74,7 +74,20 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -1110,6 +1123,7 @@ index 99d3f38ee7..f1a74280dd 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1117,10 +1131,10 @@ index 99d3f38ee7..f1a74280dd 100644 @AfterAll diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index a70aeb1ba6..fae2359d01 100644 +index a70aeb1ba6..ed82c3a8b2 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -123,6 +123,15 @@ public class TestFilteredScan { +@@ -123,6 +123,16 @@ public class TestFilteredScan { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1131,16 +1145,17 @@ index a70aeb1ba6..fae2359d01 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); // define UDFs used by partition tests diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index c03f7b94ec..30032fa107 100644 +index c03f7b94ec..52c3728a77 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -99,6 +99,15 @@ public class TestForwardCompatibility { +@@ -99,6 +99,16 @@ public class TestForwardCompatibility { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1151,16 +1166,17 @@ index c03f7b94ec..30032fa107 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index f4f57157e4..418364e3da 100644 +index f4f57157e4..d3fb378fec 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -51,6 +51,15 @@ public class TestIcebergSpark { +@@ -51,6 +51,16 @@ public class TestIcebergSpark { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1171,16 +1187,17 @@ index f4f57157e4..418364e3da 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index e1402396fa..3092902873 100644 +index e1402396fa..e007e8e35b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -118,6 +118,15 @@ public class TestPartitionPruning { +@@ -118,6 +118,16 @@ public class TestPartitionPruning { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1191,16 +1208,17 @@ index e1402396fa..3092902873 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index b0ad930487..6764bf645a 100644 +index b0ad930487..886726d908 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -112,6 +112,15 @@ public class TestPartitionValues { +@@ -112,6 +112,16 @@ public class TestPartitionValues { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1211,16 +1229,17 @@ index b0ad930487..6764bf645a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 11865db7fc..c7c1e6a02f 100644 +index 11865db7fc..e09c0fc74e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -91,6 +91,15 @@ public class TestSnapshotSelection { +@@ -91,6 +91,16 @@ public class TestSnapshotSelection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1231,16 +1250,17 @@ index 11865db7fc..c7c1e6a02f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 833a0b6b5e..eb29e23aac 100644 +index 833a0b6b5e..6dbcc8d985 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -125,6 +125,15 @@ public class TestSparkDataFile { +@@ -125,6 +125,16 @@ public class TestSparkDataFile { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1251,16 +1271,17 @@ index 833a0b6b5e..eb29e23aac 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 8884654fe0..fde607663e 100644 +index 8884654fe0..fe01afe653 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -100,6 +100,15 @@ public class TestSparkDataWrite { +@@ -100,6 +100,16 @@ public class TestSparkDataWrite { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1271,12 +1292,13 @@ index 8884654fe0..fde607663e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } -@@ -144,7 +153,7 @@ public class TestSparkDataWrite { +@@ -144,7 +154,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1285,7 +1307,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); for (ManifestFile manifest : SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { -@@ -213,7 +222,7 @@ public class TestSparkDataWrite { +@@ -213,7 +223,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1294,7 +1316,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -258,7 +267,7 @@ public class TestSparkDataWrite { +@@ -258,7 +268,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1303,7 +1325,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -310,7 +319,7 @@ public class TestSparkDataWrite { +@@ -310,7 +320,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1312,7 +1334,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -352,7 +361,7 @@ public class TestSparkDataWrite { +@@ -352,7 +362,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1321,7 +1343,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -391,7 +400,7 @@ public class TestSparkDataWrite { +@@ -391,7 +401,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1330,7 +1352,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); List files = Lists.newArrayList(); -@@ -459,7 +468,7 @@ public class TestSparkDataWrite { +@@ -459,7 +469,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1339,7 +1361,7 @@ index 8884654fe0..fde607663e 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -707,7 +716,7 @@ public class TestSparkDataWrite { +@@ -707,7 +717,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1349,10 +1371,10 @@ index 8884654fe0..fde607663e 100644 .hasSize(records.size() + records2.size()) .containsExactlyInAnyOrder( diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 596d05d30b..acbacf65e0 100644 +index 596d05d30b..b6ed4688ce 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -88,6 +88,15 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -88,6 +88,16 @@ public class TestSparkReadProjection extends TestReadProjection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1363,16 +1385,17 @@ index 596d05d30b..acbacf65e0 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); ImmutableMap config = ImmutableMap.of( diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 16fa726032..ce27c5cd4d 100644 +index 16fa726032..be4813379e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -132,7 +132,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -132,7 +132,27 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1383,6 +1406,7 @@ index 16fa726032..ce27c5cd4d 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1393,12 +1417,13 @@ index 16fa726032..ce27c5cd4d 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = -@@ -200,7 +218,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -200,7 +220,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -1409,10 +1434,10 @@ index 16fa726032..ce27c5cd4d 100644 @Override diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..fb677d7114 100644 +index baf7fa8f88..69181efc94 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,7 +182,25 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,7 +182,27 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1423,6 +1448,7 @@ index baf7fa8f88..fb677d7114 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1433,16 +1459,17 @@ index baf7fa8f88..fb677d7114 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index dc4fc7e187..2ab6dd12c2 100644 +index dc4fc7e187..b56dc6db7b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -69,6 +69,15 @@ public class TestStructuredStreaming { +@@ -69,6 +69,16 @@ public class TestStructuredStreaming { .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .config("spark.sql.shuffle.partitions", 4) @@ -1453,16 +1480,17 @@ index dc4fc7e187..2ab6dd12c2 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 8b1e3fbfc7..c519a2be0f 100644 +index 8b1e3fbfc7..8f774766cc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,20 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -1477,6 +1505,7 @@ index 8b1e3fbfc7..c519a2be0f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1484,10 +1513,10 @@ index 8b1e3fbfc7..c519a2be0f 100644 @AfterAll diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index c3fac70dd3..d3ffd034d2 100644 +index c3fac70dd3..5491a84de3 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -84,6 +84,15 @@ public class TestWriteMetricsConfig { +@@ -84,6 +84,16 @@ public class TestWriteMetricsConfig { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -1498,16 +1527,17 @@ index c3fac70dd3..d3ffd034d2 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 5ce56b4fec..3b614e3fc9 100644 +index 5ce56b4fec..ee6ce0cbb1 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -63,7 +63,25 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -63,7 +63,27 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -1518,6 +1548,7 @@ index 5ce56b4fec..3b614e3fc9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1528,6 +1559,7 @@ index 5ce56b4fec..3b614e3fc9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1561,10 +1593,10 @@ index 69700d8436..49ea338a45 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 4c1a509591..3b4e44cc13 100644 +index 4c1a509591..2477b29cba 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -59,6 +59,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -59,6 +59,16 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -1575,6 +1607,7 @@ index 4c1a509591..3b4e44cc13 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1595,10 +1628,10 @@ index 893f9931cf..17e5fef217 100644 protected void createTableWithDeleteGranularity( diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index ecf9e6f8a5..beca5f8521 100644 +index ecf9e6f8a5..42591bd9b2 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,15 @@ public class TestCallStatementParser { +@@ -56,6 +56,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -1609,6 +1642,7 @@ index ecf9e6f8a5..beca5f8521 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1643,10 +1677,10 @@ index 8cb92224fc..777b9bb1dd 100644 } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 64edb1002e..57767da086 100644 +index 64edb1002e..fc1d0f01ae 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -179,6 +179,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -1657,16 +1691,17 @@ index 64edb1002e..57767da086 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index 77b79384a6..8708ffa72a 100644 +index 77b79384a6..0ac69d5487 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { +@@ -392,6 +392,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -1677,16 +1712,17 @@ index 77b79384a6..8708ffa72a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index c6794e43c6..4c1a165a44 100644 +index c6794e43c6..90bde88cd5 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -239,6 +239,15 @@ public class DVReaderBenchmark { +@@ -239,6 +239,16 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1697,16 +1733,17 @@ index c6794e43c6..4c1a165a44 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index ac74fb5a10..4aa5d5b361 100644 +index ac74fb5a10..07b8c34f8a 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -223,6 +223,15 @@ public class DVWriterBenchmark { +@@ -223,6 +223,16 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1717,16 +1754,17 @@ index ac74fb5a10..4aa5d5b361 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -1741,6 +1779,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -2528,10 +2567,10 @@ index a0f45e7610..78e7845911 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2542,16 +2581,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 659507e4c5..2a3a87f079 100644 +index 659507e4c5..d44910e809 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2562,16 +2602,17 @@ index 659507e4c5..2a3a87f079 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2582,16 +2623,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index 2665d7ba8d..c24c82341a 100644 +index 2665d7ba8d..ef17afe769 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2602,16 +2644,17 @@ index 2665d7ba8d..c24c82341a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index daf4e29ac0..235cf60557 100644 +index daf4e29ac0..7f21607315 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -79,6 +79,15 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -79,6 +79,16 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -2622,6 +2665,7 @@ index daf4e29ac0..235cf60557 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -2653,10 +2697,10 @@ index 0db6a65fd3..9ee8b9e084 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -index 973a17c9a3..4e4d3c02f1 100644 +index 973a17c9a3..9d3852665f 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -@@ -65,6 +65,15 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -65,6 +65,16 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2667,16 +2711,17 @@ index 973a17c9a3..4e4d3c02f1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 1c5905744a..b9cf548290 100644 +index 1c5905744a..08b0e0a361 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -61,6 +61,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { +@@ -61,6 +61,16 @@ public abstract class ScanTestBase extends AvroDataTestBase { ScanTestBase.spark = SparkSession.builder() .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2687,16 +2732,17 @@ index 1c5905744a..b9cf548290 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[2]") .getOrCreate(); ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 19ec6d13dd..c5f73bf666 100644 +index 19ec6d13dd..125e3b2ccb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,20 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -2711,6 +2757,7 @@ index 19ec6d13dd..c5f73bf666 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2718,10 +2765,10 @@ index 19ec6d13dd..c5f73bf666 100644 @BeforeEach diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index a7702b169a..67fabf3cdc 100644 +index a7702b169a..61359553cb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -74,7 +74,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -74,7 +74,20 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -2736,6 +2783,7 @@ index a7702b169a..67fabf3cdc 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2743,10 +2791,10 @@ index a7702b169a..67fabf3cdc 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index fd7d52178f..36831cfb07 100644 +index fd7d52178f..fb806b7023 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -114,6 +114,15 @@ public class TestFilteredScan { +@@ -114,6 +114,16 @@ public class TestFilteredScan { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2757,16 +2805,17 @@ index fd7d52178f..36831cfb07 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 153564f7d1..973eac1878 100644 +index 153564f7d1..b9eb74e443 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -98,6 +98,15 @@ public class TestForwardCompatibility { +@@ -98,6 +98,16 @@ public class TestForwardCompatibility { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2777,16 +2826,17 @@ index 153564f7d1..973eac1878 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index f4f57157e4..418364e3da 100644 +index f4f57157e4..d3fb378fec 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -51,6 +51,15 @@ public class TestIcebergSpark { +@@ -51,6 +51,16 @@ public class TestIcebergSpark { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2797,16 +2847,17 @@ index f4f57157e4..418364e3da 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index e1402396fa..3092902873 100644 +index e1402396fa..e007e8e35b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -118,6 +118,15 @@ public class TestPartitionPruning { +@@ -118,6 +118,16 @@ public class TestPartitionPruning { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2817,16 +2868,17 @@ index e1402396fa..3092902873 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 0b6ab2052b..8682b4a7da 100644 +index 0b6ab2052b..5ff42b3182 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -112,6 +112,15 @@ public class TestPartitionValues { +@@ -112,6 +112,16 @@ public class TestPartitionValues { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2837,16 +2889,17 @@ index 0b6ab2052b..8682b4a7da 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 11865db7fc..c7c1e6a02f 100644 +index 11865db7fc..e09c0fc74e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -91,6 +91,15 @@ public class TestSnapshotSelection { +@@ -91,6 +91,16 @@ public class TestSnapshotSelection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2857,16 +2910,17 @@ index 11865db7fc..c7c1e6a02f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 3051e27d72..cb3011b222 100644 +index 3051e27d72..7f8b806351 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -125,6 +125,15 @@ public class TestSparkDataFile { +@@ -125,6 +125,16 @@ public class TestSparkDataFile { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2877,16 +2931,17 @@ index 3051e27d72..cb3011b222 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 4ccbf86f12..f915f22b21 100644 +index 4ccbf86f12..24f7ea1979 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -100,6 +100,15 @@ public class TestSparkDataWrite { +@@ -100,6 +100,16 @@ public class TestSparkDataWrite { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2897,12 +2952,13 @@ index 4ccbf86f12..f915f22b21 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } -@@ -144,7 +153,7 @@ public class TestSparkDataWrite { +@@ -144,7 +154,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2911,7 +2967,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); for (ManifestFile manifest : SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { -@@ -213,7 +222,7 @@ public class TestSparkDataWrite { +@@ -213,7 +223,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2920,7 +2976,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -258,7 +267,7 @@ public class TestSparkDataWrite { +@@ -258,7 +268,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2929,7 +2985,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -310,7 +319,7 @@ public class TestSparkDataWrite { +@@ -310,7 +320,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2938,7 +2994,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -352,7 +361,7 @@ public class TestSparkDataWrite { +@@ -352,7 +362,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2947,7 +3003,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -391,7 +400,7 @@ public class TestSparkDataWrite { +@@ -391,7 +401,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2956,7 +3012,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); List files = Lists.newArrayList(); -@@ -459,7 +468,7 @@ public class TestSparkDataWrite { +@@ -459,7 +469,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2965,7 +3021,7 @@ index 4ccbf86f12..f915f22b21 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -706,7 +715,7 @@ public class TestSparkDataWrite { +@@ -706,7 +716,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -2975,10 +3031,10 @@ index 4ccbf86f12..f915f22b21 100644 .hasSize(records.size() + records2.size()) .containsExactlyInAnyOrder( diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 596d05d30b..acbacf65e0 100644 +index 596d05d30b..b6ed4688ce 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -88,6 +88,15 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -88,6 +88,16 @@ public class TestSparkReadProjection extends TestReadProjection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -2989,16 +3045,17 @@ index 596d05d30b..acbacf65e0 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); ImmutableMap config = ImmutableMap.of( diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 42699f4662..b8bf79886e 100644 +index 42699f4662..95ff0bbcac 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -138,6 +138,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -138,6 +138,16 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3009,12 +3066,13 @@ index 42699f4662..b8bf79886e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); -@@ -206,7 +215,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -206,7 +216,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -3025,10 +3083,10 @@ index 42699f4662..b8bf79886e 100644 @Override diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..b58321b761 100644 +index baf7fa8f88..2eaa17e8b0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,6 +182,16 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3039,16 +3097,17 @@ index baf7fa8f88..b58321b761 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 54048bbf21..2ebe9ab993 100644 +index 54048bbf21..da2b17b309 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -69,6 +69,15 @@ public class TestStructuredStreaming { +@@ -69,6 +69,16 @@ public class TestStructuredStreaming { .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .config("spark.sql.shuffle.partitions", 4) @@ -3059,16 +3118,17 @@ index 54048bbf21..2ebe9ab993 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 8b1e3fbfc7..c519a2be0f 100644 +index 8b1e3fbfc7..8f774766cc 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,20 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -3083,6 +3143,7 @@ index 8b1e3fbfc7..c519a2be0f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3090,10 +3151,10 @@ index 8b1e3fbfc7..c519a2be0f 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index c3fac70dd3..d3ffd034d2 100644 +index c3fac70dd3..5491a84de3 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -84,6 +84,15 @@ public class TestWriteMetricsConfig { +@@ -84,6 +84,16 @@ public class TestWriteMetricsConfig { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -3104,16 +3165,17 @@ index c3fac70dd3..d3ffd034d2 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 5ce56b4fec..d089cc1f69 100644 +index 5ce56b4fec..e28d4a417b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -63,6 +63,15 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -63,6 +63,16 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -3124,6 +3186,7 @@ index 5ce56b4fec..d089cc1f69 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() diff --git a/dev/diffs/iceberg/1.11.0.diff b/dev/diffs/iceberg/1.11.0.diff index 555002b51f..5fd5de39bf 100644 --- a/dev/diffs/iceberg/1.11.0.diff +++ b/dev/diffs/iceberg/1.11.0.diff @@ -39,10 +39,10 @@ index e6455fa34f..08b685f61a 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index f766fbb79a..59e4e6c285 100644 +index f766fbb79a..24c309e34b 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -71,6 +71,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -71,6 +71,16 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -53,6 +53,7 @@ index f766fbb79a..59e4e6c285 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) @@ -101,10 +102,10 @@ index 934220e5d3..92132d4b8d 100644 } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 3fd84553f0..db71a30730 100644 +index 3fd84553f0..6954ee71dd 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -180,6 +180,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -180,6 +180,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -115,16 +116,17 @@ index 3fd84553f0..db71a30730 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .master("local"); spark = builder.getOrCreate(); diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index 683f6bb46d..bd5ef5d3c0 100644 +index 683f6bb46d..21652664ba 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -395,6 +395,15 @@ public class IcebergSortCompactionBenchmark { +@@ -395,6 +395,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -135,16 +137,17 @@ index 683f6bb46d..bd5ef5d3c0 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .master("local[*]"); spark = builder.getOrCreate(); diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index 3f242ce228..ee19cc1248 100644 +index 3f242ce228..31df5b5dc4 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -240,6 +240,15 @@ public class DVReaderBenchmark { +@@ -240,6 +240,16 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -155,16 +158,17 @@ index 3f242ce228..ee19cc1248 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index db57897240..f3b47a62f6 100644 +index db57897240..4becac2607 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -224,6 +224,15 @@ public class DVWriterBenchmark { +@@ -224,6 +224,16 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -175,16 +179,17 @@ index db57897240..f3b47a62f6 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index debe37866f..f48e355a4c 100644 +index debe37866f..518e10eefe 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -95,7 +95,18 @@ public abstract class IcebergSourceBenchmark { +@@ -95,7 +95,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -199,16 +204,17 @@ index debe37866f..f48e355a4c 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { builder .config("parquet.dictionary.page.size", "1") diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index d1c724425c..da7450dd95 100644 +index d1c724425c..e84c5c7d2b 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -219,16 +225,17 @@ index d1c724425c..da7450dd95 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index a21c6a08ec..104bfcd580 100644 +index a21c6a08ec..ce4cb44dca 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -239,16 +246,17 @@ index a21c6a08ec..104bfcd580 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index 5edf482822..24bd0ee141 100644 +index 5edf482822..dfebdfd0ab 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -259,16 +267,17 @@ index 5edf482822..24bd0ee141 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index e6f3c75475..6e2a7c6fb7 100644 +index e6f3c75475..cac1cfc97b 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -279,16 +288,17 @@ index e6f3c75475..6e2a7c6fb7 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index 507d7b313b..8fbdb40228 100644 +index 507d7b313b..d22c156031 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -86,6 +86,15 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -86,6 +86,16 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -299,6 +309,7 @@ index 507d7b313b..8fbdb40228 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config("spark.ui.enabled", "false") @@ -330,10 +341,10 @@ index 8ce60f6275..75112b4354 100644 } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -index b61ecfa2f4..a3aafae4ee 100644 +index b61ecfa2f4..fc38da6d71 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java -@@ -66,6 +66,15 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -66,6 +66,16 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -344,16 +355,17 @@ index b61ecfa2f4..a3aafae4ee 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index da9cd63921..c713c27456 100644 +index da9cd63921..66c7abffab 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -62,6 +62,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { +@@ -62,6 +62,16 @@ public abstract class ScanTestBase extends AvroDataTestBase { ScanTestBase.spark = SparkSession.builder() .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -364,16 +376,17 @@ index da9cd63921..c713c27456 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[2]") .config(TestBase.DISABLE_UI) .getOrCreate(); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index d381822483..f090b5bf14 100644 +index d381822483..edc5434460 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,20 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -388,6 +401,7 @@ index d381822483..f090b5bf14 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -395,10 +409,10 @@ index d381822483..f090b5bf14 100644 @BeforeEach diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index e67ec5fd62..ef19a1d349 100644 +index e67ec5fd62..d9f078a615 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -79,7 +79,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -79,7 +79,20 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -413,6 +427,7 @@ index e67ec5fd62..ef19a1d349 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -420,10 +435,10 @@ index e67ec5fd62..ef19a1d349 100644 @AfterAll diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index 24fecf4eb2..964a5bc003 100644 +index 24fecf4eb2..7c38f1d85f 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -117,6 +117,15 @@ public class TestFilteredScan { +@@ -117,6 +117,16 @@ public class TestFilteredScan { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -434,16 +449,17 @@ index 24fecf4eb2..964a5bc003 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index d0103ff46e..c634782764 100644 +index d0103ff46e..03e0b5d927 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -99,6 +99,15 @@ public class TestForwardCompatibility { +@@ -99,6 +99,16 @@ public class TestForwardCompatibility { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -454,16 +470,17 @@ index d0103ff46e..c634782764 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index a637b975fe..dc31aab58b 100644 +index a637b975fe..dffaa26c93 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -52,6 +52,15 @@ public class TestIcebergSpark { +@@ -52,6 +52,16 @@ public class TestIcebergSpark { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -474,16 +491,17 @@ index a637b975fe..dc31aab58b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 8098db81f9..a3b023a43e 100644 +index 8098db81f9..03631fb1c6 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -120,6 +120,15 @@ public class TestPartitionPruning { +@@ -120,6 +120,16 @@ public class TestPartitionPruning { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -494,16 +512,17 @@ index 8098db81f9..a3b023a43e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 9b5b22a73f..4aed7f06dc 100644 +index 9b5b22a73f..793bce01da 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -113,6 +113,15 @@ public class TestPartitionValues { +@@ -113,6 +113,16 @@ public class TestPartitionValues { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -514,16 +533,17 @@ index 9b5b22a73f..4aed7f06dc 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 3004e8fa5c..82ebbe9e81 100644 +index 3004e8fa5c..e59564d443 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -98,6 +98,15 @@ public class TestSnapshotSelection { +@@ -98,6 +98,16 @@ public class TestSnapshotSelection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -534,16 +554,17 @@ index 3004e8fa5c..82ebbe9e81 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index d719ca6751..00ec4f9cdf 100644 +index d719ca6751..17a41322c9 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -127,6 +127,15 @@ public class TestSparkDataFile { +@@ -127,6 +127,16 @@ public class TestSparkDataFile { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -554,16 +575,17 @@ index d719ca6751..00ec4f9cdf 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index c2f5afef0e..d970524cd8 100644 +index c2f5afef0e..7bb8557f0d 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -104,6 +104,15 @@ public class TestSparkDataWrite { +@@ -104,6 +104,16 @@ public class TestSparkDataWrite { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -574,12 +596,13 @@ index c2f5afef0e..d970524cd8 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } -@@ -149,7 +158,7 @@ public class TestSparkDataWrite { +@@ -149,7 +159,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -588,7 +611,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); for (ManifestFile manifest : SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { -@@ -219,7 +228,7 @@ public class TestSparkDataWrite { +@@ -219,7 +229,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -597,7 +620,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -264,7 +273,7 @@ public class TestSparkDataWrite { +@@ -264,7 +274,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -606,7 +629,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -316,7 +325,7 @@ public class TestSparkDataWrite { +@@ -316,7 +326,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -615,7 +638,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -358,7 +367,7 @@ public class TestSparkDataWrite { +@@ -358,7 +368,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -624,7 +647,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -397,7 +406,7 @@ public class TestSparkDataWrite { +@@ -397,7 +407,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -633,7 +656,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); List files = Lists.newArrayList(); -@@ -461,7 +470,7 @@ public class TestSparkDataWrite { +@@ -461,7 +471,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -642,7 +665,7 @@ index c2f5afef0e..d970524cd8 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -814,7 +823,7 @@ public class TestSparkDataWrite { +@@ -814,7 +824,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -652,10 +675,10 @@ index c2f5afef0e..d970524cd8 100644 .hasSize(records.size() + records2.size()) .containsExactlyInAnyOrder( diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index de6a5e5902..e98a15fe39 100644 +index de6a5e5902..5d733e6168 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -89,6 +89,15 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -89,6 +89,16 @@ public class TestSparkReadProjection extends TestReadProjection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -666,16 +689,17 @@ index de6a5e5902..e98a15fe39 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); ImmutableMap config = diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 0d61930571..0a6208cc21 100644 +index 0d61930571..dfd998ae2e 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -138,6 +138,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -138,6 +138,16 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -686,12 +710,13 @@ index 0d61930571..0a6208cc21 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .enableHiveSupport() .getOrCreate(); -@@ -210,7 +219,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -210,7 +220,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -702,10 +727,10 @@ index 0d61930571..0a6208cc21 100644 @Override diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index cb2f866fab..02c42b7299 100644 +index cb2f866fab..9ac20eaf5f 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -183,6 +183,15 @@ public class TestSparkReaderWithBloomFilter { +@@ -183,6 +183,16 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -716,16 +741,17 @@ index cb2f866fab..02c42b7299 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .enableHiveSupport() .getOrCreate(); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 5e900ea0ba..2f7d6a08a1 100644 +index 5e900ea0ba..30bd7ce86f 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -73,6 +73,15 @@ public class TestStructuredStreaming { +@@ -73,6 +73,16 @@ public class TestStructuredStreaming { .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) .config("spark.sql.shuffle.partitions", 4) @@ -736,16 +762,17 @@ index 5e900ea0ba..2f7d6a08a1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 79781a8fc3..08bfa8098b 100644 +index 79781a8fc3..7c321dbc7f 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,20 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -760,6 +787,7 @@ index 79781a8fc3..08bfa8098b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -767,10 +795,10 @@ index 79781a8fc3..08bfa8098b 100644 @AfterAll diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index ab2479d610..49383ca4a6 100644 +index ab2479d610..f4d144344d 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -86,6 +86,15 @@ public class TestWriteMetricsConfig { +@@ -86,6 +86,16 @@ public class TestWriteMetricsConfig { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -781,16 +809,17 @@ index ab2479d610..49383ca4a6 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) .getOrCreate(); TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 1669301d2d..422c7e4ace 100644 +index 1669301d2d..0806a2a5a6 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -63,6 +63,15 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -63,6 +63,16 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -801,6 +830,7 @@ index 1669301d2d..422c7e4ace 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .config(TestBase.DISABLE_UI) @@ -824,10 +854,10 @@ index e5a9d63b68..220d445851 100644 assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java -index 8cdcf22e58..da3ea3560f 100644 +index 8cdcf22e58..f6cfadb6ad 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java -@@ -100,6 +100,15 @@ public class TestVariantShredding extends CatalogTestBase { +@@ -100,6 +100,16 @@ public class TestVariantShredding extends CatalogTestBase { .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config(DISABLE_UI) @@ -838,6 +868,7 @@ index 8cdcf22e58..da3ea3560f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() diff --git a/dev/diffs/iceberg/1.8.1.diff b/dev/diffs/iceberg/1.8.1.diff index 718eeccd7e..e15ef0f444 100644 --- a/dev/diffs/iceberg/1.8.1.diff +++ b/dev/diffs/iceberg/1.8.1.diff @@ -51,10 +51,10 @@ index 6eb26e8b73..38d9cf3ca3 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java -index 4f137f5b8d..f1122f0843 100644 +index 4f137f5b8d..2af2ebebf7 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java -@@ -60,7 +60,25 @@ public abstract class SparkExtensionsTestBase extends SparkCatalogTestBase { +@@ -60,7 +60,27 @@ public abstract class SparkExtensionsTestBase extends SparkCatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -65,6 +65,7 @@ index 4f137f5b8d..f1122f0843 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -75,6 +76,7 @@ index 4f137f5b8d..f1122f0843 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -95,10 +97,10 @@ index 5fa76f9da6..9184aaba83 100644 protected void createTableWithDeleteGranularity( diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index 55a413063e..61a9385ff3 100644 +index 55a413063e..ebfdc1ec79 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -61,6 +61,15 @@ public class TestCallStatementParser { +@@ -61,6 +61,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -109,6 +111,7 @@ index 55a413063e..61a9385ff3 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -143,10 +146,10 @@ index 49119d319c..04fd3d4859 100644 } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index b6ade2bff3..90a4839dad 100644 +index b6ade2bff3..af28e6bb33 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -170,6 +170,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -157,16 +160,17 @@ index b6ade2bff3..90a4839dad 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index b08c352819..63880b089d 100644 +index b08c352819..de091c0b09 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { +@@ -386,6 +386,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -177,16 +181,17 @@ index b08c352819..63880b089d 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -201,6 +206,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -1004,10 +1010,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1018,16 +1024,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 9361c63176..16ce1c614f 100644 +index 9361c63176..5c9fb10e62 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -69,6 +69,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1038,16 +1045,17 @@ index 9361c63176..16ce1c614f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1058,16 +1066,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index acd4688440..33732e6654 100644 +index acd4688440..4f7a9040a7 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting +@@ -59,6 +59,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1078,16 +1087,17 @@ index acd4688440..33732e6654 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -index 3e8953fb95..51ce392476 100644 +index 3e8953fb95..2b021f0226 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -@@ -77,6 +77,15 @@ public abstract class SparkTestBase extends SparkTestHelperBase { +@@ -77,6 +77,16 @@ public abstract class SparkTestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -1098,6 +1108,7 @@ index 3e8953fb95..51ce392476 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1129,10 +1140,10 @@ index ad969384c5..f3c2b833e3 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 3a269740b7..fc63449215 100644 +index 3a269740b7..a0f6341fbd 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -54,7 +54,19 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -54,7 +54,20 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -1147,6 +1158,7 @@ index 3a269740b7..fc63449215 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1154,10 +1166,10 @@ index 3a269740b7..fc63449215 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 724c6edde2..ff01c10e1a 100644 +index 724c6edde2..9f2e06e3bc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -103,7 +103,19 @@ public class TestCompressionSettings extends SparkCatalogTestBase { +@@ -103,7 +103,20 @@ public class TestCompressionSettings extends SparkCatalogTestBase { @BeforeClass public static void startSpark() { @@ -1172,6 +1184,7 @@ index 724c6edde2..ff01c10e1a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1179,10 +1192,10 @@ index 724c6edde2..ff01c10e1a 100644 @Parameterized.AfterParam diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index 013b8d4386..007b1af65b 100644 +index 013b8d4386..e112c78cfc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -76,7 +76,19 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { +@@ -76,7 +76,20 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { @BeforeClass public static void startSpark() { @@ -1197,6 +1210,7 @@ index 013b8d4386..007b1af65b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1204,10 +1218,10 @@ index 013b8d4386..007b1af65b 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index ba13d005bd..b9778c1986 100644 +index ba13d005bd..a426d6e79e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -118,7 +118,19 @@ public class TestFilteredScan { +@@ -118,7 +118,20 @@ public class TestFilteredScan { @BeforeClass public static void startSpark() { @@ -1222,6 +1236,7 @@ index ba13d005bd..b9778c1986 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1229,10 +1244,10 @@ index ba13d005bd..b9778c1986 100644 // define UDFs used by partition tests Function bucket4 = Transforms.bucket(4).bind(Types.LongType.get()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 9f97753094..fb5bfbad1b 100644 +index 9f97753094..39e19af0b0 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -94,7 +94,19 @@ public class TestForwardCompatibility { +@@ -94,7 +94,20 @@ public class TestForwardCompatibility { @BeforeClass public static void startSpark() { @@ -1247,6 +1262,7 @@ index 9f97753094..fb5bfbad1b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1254,10 +1270,10 @@ index 9f97753094..fb5bfbad1b 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 0154506f86..12e916cbe4 100644 +index 0154506f86..b57e0bb01b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,19 @@ public class TestIcebergSpark { +@@ -46,7 +46,20 @@ public class TestIcebergSpark { @BeforeClass public static void startSpark() { @@ -1272,6 +1288,7 @@ index 0154506f86..12e916cbe4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1279,10 +1296,10 @@ index 0154506f86..12e916cbe4 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 639d37c793..5e397909d4 100644 +index 639d37c793..6d033d2634 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -111,7 +111,19 @@ public class TestPartitionPruning { +@@ -111,7 +111,20 @@ public class TestPartitionPruning { @BeforeClass public static void startSpark() { @@ -1297,6 +1314,7 @@ index 639d37c793..5e397909d4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1304,10 +1322,10 @@ index 639d37c793..5e397909d4 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index ad0984ef42..99af33c86c 100644 +index ad0984ef42..3333372cf2 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -104,7 +104,19 @@ public class TestPartitionValues { +@@ -104,7 +104,20 @@ public class TestPartitionValues { @BeforeClass public static void startSpark() { @@ -1322,6 +1340,7 @@ index ad0984ef42..99af33c86c 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1329,10 +1348,10 @@ index ad0984ef42..99af33c86c 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 9fc576dde5..69a49f5091 100644 +index 9fc576dde5..091d2bc7b4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -83,7 +83,19 @@ public class TestSnapshotSelection { +@@ -83,7 +83,20 @@ public class TestSnapshotSelection { @BeforeClass public static void startSpark() { @@ -1347,6 +1366,7 @@ index 9fc576dde5..69a49f5091 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1354,10 +1374,10 @@ index 9fc576dde5..69a49f5091 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 16fde3c954..26e94f6dde 100644 +index 16fde3c954..312ad3485e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -121,7 +121,19 @@ public class TestSparkDataFile { +@@ -121,7 +121,20 @@ public class TestSparkDataFile { @BeforeClass public static void startSpark() { @@ -1372,6 +1392,7 @@ index 16fde3c954..26e94f6dde 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1379,10 +1400,10 @@ index 16fde3c954..26e94f6dde 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 63c18277aa..971f14b647 100644 +index 63c18277aa..4e8172f96d 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -89,7 +89,19 @@ public class TestSparkDataWrite { +@@ -89,7 +89,20 @@ public class TestSparkDataWrite { @BeforeClass public static void startSpark() { @@ -1397,13 +1418,14 @@ index 63c18277aa..971f14b647 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @Parameterized.AfterParam -@@ -138,7 +150,7 @@ public class TestSparkDataWrite { +@@ -138,7 +151,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1412,7 +1434,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); for (ManifestFile manifest : -@@ -208,7 +220,7 @@ public class TestSparkDataWrite { +@@ -208,7 +221,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1421,7 +1443,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -254,7 +266,7 @@ public class TestSparkDataWrite { +@@ -254,7 +267,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1430,7 +1452,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -307,7 +319,7 @@ public class TestSparkDataWrite { +@@ -307,7 +320,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1439,7 +1461,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -350,7 +362,7 @@ public class TestSparkDataWrite { +@@ -350,7 +363,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1448,7 +1470,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -390,7 +402,7 @@ public class TestSparkDataWrite { +@@ -390,7 +403,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1457,7 +1479,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -455,7 +467,7 @@ public class TestSparkDataWrite { +@@ -455,7 +468,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1466,7 +1488,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -619,7 +631,7 @@ public class TestSparkDataWrite { +@@ -619,7 +632,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1475,7 +1497,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -705,7 +717,7 @@ public class TestSparkDataWrite { +@@ -705,7 +718,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1485,10 +1507,10 @@ index 63c18277aa..971f14b647 100644 "Number of rows should match", records.size() + records2.size(), actual.size()); assertThat(actual) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 3a4b235c46..7633a1792d 100644 +index 3a4b235c46..994741c542 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -86,7 +86,19 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -86,7 +86,20 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeClass public static void startSpark() { @@ -1503,6 +1525,7 @@ index 3a4b235c46..7633a1792d 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1510,10 +1533,10 @@ index 3a4b235c46..7633a1792d 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index dda49b4946..69d3a0d24f 100644 +index dda49b4946..e4ee109148 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -131,7 +131,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -131,7 +131,27 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1524,6 +1547,7 @@ index dda49b4946..69d3a0d24f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1534,12 +1558,13 @@ index dda49b4946..69d3a0d24f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = -@@ -199,7 +217,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -199,7 +219,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -1550,10 +1575,10 @@ index dda49b4946..69d3a0d24f 100644 @Override diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index e5831b76e4..2433013d3e 100644 +index e5831b76e4..115635a0a5 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -176,7 +176,25 @@ public class TestSparkReaderWithBloomFilter { +@@ -176,7 +176,27 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1564,6 +1589,7 @@ index e5831b76e4..2433013d3e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1574,16 +1600,17 @@ index e5831b76e4..2433013d3e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index f420f1b955..21273826ba 100644 +index f420f1b955..21e229ebbc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -66,6 +66,15 @@ public class TestStructuredStreaming { +@@ -66,6 +66,16 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -1594,16 +1621,17 @@ index f420f1b955..21273826ba 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index ac674e2e62..c18fed7cfb 100644 +index ac674e2e62..29a936c888 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -73,7 +73,19 @@ public class TestTimestampWithoutZone extends SparkTestBase { +@@ -73,7 +73,20 @@ public class TestTimestampWithoutZone extends SparkTestBase { @BeforeClass public static void startSpark() { @@ -1618,6 +1646,7 @@ index ac674e2e62..c18fed7cfb 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1625,10 +1654,10 @@ index ac674e2e62..c18fed7cfb 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 73827b309b..ace66913d1 100644 +index 73827b309b..f8ff9a51b3 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,20 @@ public class TestWriteMetricsConfig { @BeforeClass public static void startSpark() { @@ -1643,6 +1672,7 @@ index 73827b309b..ace66913d1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1650,10 +1680,10 @@ index 73827b309b..ace66913d1 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 1a4e2f3e1c..047e0f1fb4 100644 +index 1a4e2f3e1c..6977221c3f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -65,7 +65,25 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { +@@ -65,7 +65,27 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -1664,6 +1694,7 @@ index 1a4e2f3e1c..047e0f1fb4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1674,6 +1705,7 @@ index 1a4e2f3e1c..047e0f1fb4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1738,10 +1770,10 @@ index e2d2c7a7ac..95cd3c08b5 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 578845e3da..02b6dc270f 100644 +index 578845e3da..047980e7fd 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -57,6 +57,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -57,6 +57,16 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -1752,16 +1784,17 @@ index 578845e3da..02b6dc270f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index ade19de36f..579b68e73f 100644 +index ade19de36f..6257c9efef 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,15 @@ public class TestCallStatementParser { +@@ -56,6 +56,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -1772,6 +1805,7 @@ index ade19de36f..579b68e73f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1806,10 +1840,10 @@ index 8cb92224fc..777b9bb1dd 100644 } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 64edb1002e..57767da086 100644 +index 64edb1002e..fc1d0f01ae 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -179,6 +179,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -1820,16 +1854,17 @@ index 64edb1002e..57767da086 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index a5d0456b0b..f52c387943 100644 +index a5d0456b0b..4dd831bc29 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { +@@ -392,6 +392,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -1840,16 +1875,17 @@ index a5d0456b0b..f52c387943 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index c6794e43c6..4c1a165a44 100644 +index c6794e43c6..90bde88cd5 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -239,6 +239,15 @@ public class DVReaderBenchmark { +@@ -239,6 +239,16 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1860,16 +1896,17 @@ index c6794e43c6..4c1a165a44 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index ac74fb5a10..4aa5d5b361 100644 +index ac74fb5a10..07b8c34f8a 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -223,6 +223,15 @@ public class DVWriterBenchmark { +@@ -223,6 +223,16 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1880,16 +1917,17 @@ index ac74fb5a10..4aa5d5b361 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -1904,6 +1942,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -2707,10 +2746,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2721,16 +2760,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 659507e4c5..2a3a87f079 100644 +index 659507e4c5..d44910e809 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2741,16 +2781,17 @@ index 659507e4c5..2a3a87f079 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2761,16 +2802,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index 2665d7ba8d..c24c82341a 100644 +index 2665d7ba8d..ef17afe769 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2781,16 +2823,17 @@ index 2665d7ba8d..c24c82341a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index de68351f6e..dea8226742 100644 +index de68351f6e..c7848c59ae 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -77,6 +77,15 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -77,6 +77,16 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -2801,16 +2844,17 @@ index de68351f6e..dea8226742 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -index bc4e722bc8..57770308b3 100644 +index bc4e722bc8..e24dc5f275 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -@@ -59,7 +59,19 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -59,7 +59,20 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect @BeforeAll public static void startSpark() { @@ -2825,6 +2869,7 @@ index bc4e722bc8..57770308b3 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2832,10 +2877,10 @@ index bc4e722bc8..57770308b3 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 3a269740b7..fc63449215 100644 +index 3a269740b7..a0f6341fbd 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -54,7 +54,19 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -54,7 +54,20 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -2850,6 +2895,7 @@ index 3a269740b7..fc63449215 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2857,10 +2903,10 @@ index 3a269740b7..fc63449215 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index f411920a5d..ec8eebe117 100644 +index f411920a5d..e57eb3a5b4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,20 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -2875,6 +2921,7 @@ index f411920a5d..ec8eebe117 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2882,10 +2929,10 @@ index f411920a5d..ec8eebe117 100644 @BeforeEach diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index c4ba96e634..dcf72c19b6 100644 +index c4ba96e634..db6acd8124 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -75,7 +75,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -75,7 +75,20 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -2900,6 +2947,7 @@ index c4ba96e634..dcf72c19b6 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2907,10 +2955,10 @@ index c4ba96e634..dcf72c19b6 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index 348173596e..03156f786b 100644 +index 348173596e..eb1194f709 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -110,7 +110,19 @@ public class TestFilteredScan { +@@ -110,7 +110,20 @@ public class TestFilteredScan { @BeforeAll public static void startSpark() { @@ -2925,6 +2973,7 @@ index 348173596e..03156f786b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2932,10 +2981,10 @@ index 348173596e..03156f786b 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 84c99a575c..e647ab924e 100644 +index 84c99a575c..84d44e4478 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -93,7 +93,19 @@ public class TestForwardCompatibility { +@@ -93,7 +93,20 @@ public class TestForwardCompatibility { @BeforeAll public static void startSpark() { @@ -2950,6 +2999,7 @@ index 84c99a575c..e647ab924e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2957,10 +3007,10 @@ index 84c99a575c..e647ab924e 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 7eff93d204..2fdc87a0a8 100644 +index 7eff93d204..a0a2a0845b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,19 @@ public class TestIcebergSpark { +@@ -46,7 +46,20 @@ public class TestIcebergSpark { @BeforeAll public static void startSpark() { @@ -2975,6 +3025,7 @@ index 7eff93d204..2fdc87a0a8 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2982,10 +3033,10 @@ index 7eff93d204..2fdc87a0a8 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 9464f687b0..8653560334 100644 +index 9464f687b0..88edb4b6a0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -112,7 +112,19 @@ public class TestPartitionPruning { +@@ -112,7 +112,20 @@ public class TestPartitionPruning { @BeforeAll public static void startSpark() { @@ -3000,6 +3051,7 @@ index 9464f687b0..8653560334 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3007,10 +3059,10 @@ index 9464f687b0..8653560334 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 5c218f21c4..2f17343d59 100644 +index 5c218f21c4..bc0478daa0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -107,7 +107,19 @@ public class TestPartitionValues { +@@ -107,7 +107,20 @@ public class TestPartitionValues { @BeforeAll public static void startSpark() { @@ -3025,6 +3077,7 @@ index 5c218f21c4..2f17343d59 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3032,10 +3085,10 @@ index 5c218f21c4..2f17343d59 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index a7334a580c..68210dc8e8 100644 +index a7334a580c..7eebdc2239 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -87,7 +87,19 @@ public class TestSnapshotSelection { +@@ -87,7 +87,20 @@ public class TestSnapshotSelection { @BeforeAll public static void startSpark() { @@ -3050,6 +3103,7 @@ index a7334a580c..68210dc8e8 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3057,10 +3111,10 @@ index a7334a580c..68210dc8e8 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 182b1ef8f5..76e04b1472 100644 +index 182b1ef8f5..628f14571c 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -120,7 +120,19 @@ public class TestSparkDataFile { +@@ -120,7 +120,20 @@ public class TestSparkDataFile { @BeforeAll public static void startSpark() { @@ -3075,6 +3129,7 @@ index 182b1ef8f5..76e04b1472 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3082,10 +3137,10 @@ index 182b1ef8f5..76e04b1472 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index fb2b312bed..846f50e0c9 100644 +index fb2b312bed..03fc0d2171 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -96,7 +96,19 @@ public class TestSparkDataWrite { +@@ -96,7 +96,20 @@ public class TestSparkDataWrite { @BeforeAll public static void startSpark() { @@ -3100,13 +3155,14 @@ index fb2b312bed..846f50e0c9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @AfterEach -@@ -140,7 +152,7 @@ public class TestSparkDataWrite { +@@ -140,7 +153,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3115,7 +3171,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); for (ManifestFile manifest : -@@ -210,7 +222,7 @@ public class TestSparkDataWrite { +@@ -210,7 +223,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3124,7 +3180,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -256,7 +268,7 @@ public class TestSparkDataWrite { +@@ -256,7 +269,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3133,7 +3189,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -309,7 +321,7 @@ public class TestSparkDataWrite { +@@ -309,7 +322,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3142,7 +3198,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -352,7 +364,7 @@ public class TestSparkDataWrite { +@@ -352,7 +365,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3151,7 +3207,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -392,7 +404,7 @@ public class TestSparkDataWrite { +@@ -392,7 +405,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3160,7 +3216,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -458,7 +470,7 @@ public class TestSparkDataWrite { +@@ -458,7 +471,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3169,7 +3225,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -622,7 +634,7 @@ public class TestSparkDataWrite { +@@ -622,7 +635,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3178,7 +3234,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -708,7 +720,7 @@ public class TestSparkDataWrite { +@@ -708,7 +721,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3188,10 +3244,10 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual) .describedAs("Result rows should match") diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index becf6a064d..91451c268e 100644 +index becf6a064d..f8e3729a0e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -83,7 +83,19 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -83,7 +83,20 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeAll public static void startSpark() { @@ -3206,6 +3262,7 @@ index becf6a064d..91451c268e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3213,10 +3270,10 @@ index becf6a064d..91451c268e 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 4f1cef5d37..4a1a602ed4 100644 +index 4f1cef5d37..db8738e14e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -136,6 +136,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -136,6 +136,16 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3227,12 +3284,13 @@ index 4f1cef5d37..4a1a602ed4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); -@@ -204,7 +213,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -204,7 +214,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -3243,10 +3301,10 @@ index 4f1cef5d37..4a1a602ed4 100644 @Override diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..b58321b761 100644 +index baf7fa8f88..2eaa17e8b0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,6 +182,16 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3257,16 +3315,17 @@ index baf7fa8f88..b58321b761 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 17db46b85c..d424b97d2a 100644 +index 17db46b85c..f103f6ce6b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -65,6 +65,15 @@ public class TestStructuredStreaming { +@@ -65,6 +65,16 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -3277,16 +3336,17 @@ index 17db46b85c..d424b97d2a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 306444b9f2..e4082d9cd9 100644 +index 306444b9f2..4a72089a97 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,20 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -3301,6 +3361,7 @@ index 306444b9f2..e4082d9cd9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3308,10 +3369,10 @@ index 306444b9f2..e4082d9cd9 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 841268a6be..97f7187314 100644 +index 841268a6be..1ca5f299b4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,20 @@ public class TestWriteMetricsConfig { @BeforeAll public static void startSpark() { @@ -3326,6 +3387,7 @@ index 841268a6be..97f7187314 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3333,10 +3395,10 @@ index 841268a6be..97f7187314 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 6e09252704..4b0f2e4d58 100644 +index 6e09252704..1cd8b636f3 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -60,6 +60,15 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -60,6 +60,16 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -3347,6 +3409,7 @@ index 6e09252704..4b0f2e4d58 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() diff --git a/dev/diffs/iceberg/1.9.1.diff b/dev/diffs/iceberg/1.9.1.diff index 139003cae3..6aa3633532 100644 --- a/dev/diffs/iceberg/1.9.1.diff +++ b/dev/diffs/iceberg/1.9.1.diff @@ -41,10 +41,10 @@ index 2cd8666892..4077116abb 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 578845e3da..19a200892f 100644 +index 578845e3da..5ce5db4db8 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -57,7 +57,25 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -57,7 +57,27 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -55,6 +55,7 @@ index 578845e3da..19a200892f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -65,16 +66,17 @@ index 578845e3da..19a200892f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); TestBase.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index 58d054bd05..ef1c1cb6d1 100644 +index 58d054bd05..096985a6b5 100644 --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,15 @@ public class TestCallStatementParser { +@@ -56,6 +56,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -85,6 +87,7 @@ index 58d054bd05..ef1c1cb6d1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -119,10 +122,10 @@ index 5a0a04edba..afe92a8b72 100644 } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index b6ade2bff3..90a4839dad 100644 +index b6ade2bff3..af28e6bb33 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -170,6 +170,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -133,16 +136,17 @@ index b6ade2bff3..90a4839dad 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index b08c352819..63880b089d 100644 +index b08c352819..de091c0b09 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { +@@ -386,6 +386,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -153,16 +157,17 @@ index b08c352819..63880b089d 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -177,6 +182,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -964,10 +970,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -978,16 +984,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 9361c63176..16ce1c614f 100644 +index 9361c63176..5c9fb10e62 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -69,6 +69,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -998,16 +1005,17 @@ index 9361c63176..16ce1c614f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1018,16 +1026,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index acd4688440..33732e6654 100644 +index acd4688440..4f7a9040a7 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting +@@ -59,6 +59,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -1038,16 +1047,17 @@ index acd4688440..33732e6654 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -index 3e8953fb95..51ce392476 100644 +index 3e8953fb95..2b021f0226 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java -@@ -77,6 +77,15 @@ public abstract class SparkTestBase extends SparkTestHelperBase { +@@ -77,6 +77,16 @@ public abstract class SparkTestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -1058,16 +1068,17 @@ index 3e8953fb95..51ce392476 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 06d5e0c44f..effc2bcc69 100644 +index 06d5e0c44f..b529ca3271 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -57,7 +57,19 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -57,7 +57,20 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -1082,6 +1093,7 @@ index 06d5e0c44f..effc2bcc69 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1089,10 +1101,10 @@ index 06d5e0c44f..effc2bcc69 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 26cb4dcc95..d185f1175b 100644 +index 26cb4dcc95..8fd2d28c94 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -104,7 +104,19 @@ public class TestCompressionSettings extends SparkCatalogTestBase { +@@ -104,7 +104,20 @@ public class TestCompressionSettings extends SparkCatalogTestBase { @BeforeClass public static void startSpark() { @@ -1107,6 +1119,7 @@ index 26cb4dcc95..d185f1175b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1114,10 +1127,10 @@ index 26cb4dcc95..d185f1175b 100644 @Before diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index 013b8d4386..007b1af65b 100644 +index 013b8d4386..e112c78cfc 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -76,7 +76,19 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { +@@ -76,7 +76,20 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { @BeforeClass public static void startSpark() { @@ -1132,6 +1145,7 @@ index 013b8d4386..007b1af65b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1139,10 +1153,10 @@ index 013b8d4386..007b1af65b 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index ba13d005bd..b9778c1986 100644 +index ba13d005bd..a426d6e79e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -118,7 +118,19 @@ public class TestFilteredScan { +@@ -118,7 +118,20 @@ public class TestFilteredScan { @BeforeClass public static void startSpark() { @@ -1157,6 +1171,7 @@ index ba13d005bd..b9778c1986 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1164,10 +1179,10 @@ index ba13d005bd..b9778c1986 100644 // define UDFs used by partition tests Function bucket4 = Transforms.bucket(4).bind(Types.LongType.get()); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 9f97753094..fb5bfbad1b 100644 +index 9f97753094..39e19af0b0 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -94,7 +94,19 @@ public class TestForwardCompatibility { +@@ -94,7 +94,20 @@ public class TestForwardCompatibility { @BeforeClass public static void startSpark() { @@ -1182,6 +1197,7 @@ index 9f97753094..fb5bfbad1b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1189,10 +1205,10 @@ index 9f97753094..fb5bfbad1b 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 0154506f86..12e916cbe4 100644 +index 0154506f86..b57e0bb01b 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,19 @@ public class TestIcebergSpark { +@@ -46,7 +46,20 @@ public class TestIcebergSpark { @BeforeClass public static void startSpark() { @@ -1207,6 +1223,7 @@ index 0154506f86..12e916cbe4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1214,10 +1231,10 @@ index 0154506f86..12e916cbe4 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 639d37c793..5e397909d4 100644 +index 639d37c793..6d033d2634 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -111,7 +111,19 @@ public class TestPartitionPruning { +@@ -111,7 +111,20 @@ public class TestPartitionPruning { @BeforeClass public static void startSpark() { @@ -1232,6 +1249,7 @@ index 639d37c793..5e397909d4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1239,10 +1257,10 @@ index 639d37c793..5e397909d4 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index ad0984ef42..99af33c86c 100644 +index ad0984ef42..3333372cf2 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -104,7 +104,19 @@ public class TestPartitionValues { +@@ -104,7 +104,20 @@ public class TestPartitionValues { @BeforeClass public static void startSpark() { @@ -1257,6 +1275,7 @@ index ad0984ef42..99af33c86c 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1264,10 +1283,10 @@ index ad0984ef42..99af33c86c 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index 9fc576dde5..69a49f5091 100644 +index 9fc576dde5..091d2bc7b4 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -83,7 +83,19 @@ public class TestSnapshotSelection { +@@ -83,7 +83,20 @@ public class TestSnapshotSelection { @BeforeClass public static void startSpark() { @@ -1282,6 +1301,7 @@ index 9fc576dde5..69a49f5091 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1289,10 +1309,10 @@ index 9fc576dde5..69a49f5091 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 16fde3c954..26e94f6dde 100644 +index 16fde3c954..312ad3485e 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -121,7 +121,19 @@ public class TestSparkDataFile { +@@ -121,7 +121,20 @@ public class TestSparkDataFile { @BeforeClass public static void startSpark() { @@ -1307,6 +1327,7 @@ index 16fde3c954..26e94f6dde 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1314,10 +1335,10 @@ index 16fde3c954..26e94f6dde 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index 63c18277aa..971f14b647 100644 +index 63c18277aa..4e8172f96d 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -89,7 +89,19 @@ public class TestSparkDataWrite { +@@ -89,7 +89,20 @@ public class TestSparkDataWrite { @BeforeClass public static void startSpark() { @@ -1332,13 +1353,14 @@ index 63c18277aa..971f14b647 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @Parameterized.AfterParam -@@ -138,7 +150,7 @@ public class TestSparkDataWrite { +@@ -138,7 +151,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1347,7 +1369,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); for (ManifestFile manifest : -@@ -208,7 +220,7 @@ public class TestSparkDataWrite { +@@ -208,7 +221,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1356,7 +1378,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -254,7 +266,7 @@ public class TestSparkDataWrite { +@@ -254,7 +267,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1365,7 +1387,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -307,7 +319,7 @@ public class TestSparkDataWrite { +@@ -307,7 +320,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1374,7 +1396,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -350,7 +362,7 @@ public class TestSparkDataWrite { +@@ -350,7 +363,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1383,7 +1405,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -390,7 +402,7 @@ public class TestSparkDataWrite { +@@ -390,7 +403,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1392,7 +1414,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -455,7 +467,7 @@ public class TestSparkDataWrite { +@@ -455,7 +468,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1401,7 +1423,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); } -@@ -619,7 +631,7 @@ public class TestSparkDataWrite { +@@ -619,7 +632,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1410,7 +1432,7 @@ index 63c18277aa..971f14b647 100644 Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); Assert.assertEquals("Result rows should match", expected, actual); -@@ -705,7 +717,7 @@ public class TestSparkDataWrite { +@@ -705,7 +718,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -1420,10 +1442,10 @@ index 63c18277aa..971f14b647 100644 "Number of rows should match", records.size() + records2.size(), actual.size()); assertThat(actual) diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index 584a6b1c70..b93a23d6f1 100644 +index 584a6b1c70..823f0f910f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -86,7 +86,19 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -86,7 +86,20 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeClass public static void startSpark() { @@ -1438,6 +1460,7 @@ index 584a6b1c70..b93a23d6f1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1445,10 +1468,10 @@ index 584a6b1c70..b93a23d6f1 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index dda49b4946..69d3a0d24f 100644 +index dda49b4946..e4ee109148 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -131,7 +131,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -131,7 +131,27 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1459,6 +1482,7 @@ index dda49b4946..69d3a0d24f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1469,12 +1493,13 @@ index dda49b4946..69d3a0d24f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = -@@ -199,7 +217,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -199,7 +219,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -1485,10 +1510,10 @@ index dda49b4946..69d3a0d24f 100644 @Override diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index e5831b76e4..2433013d3e 100644 +index e5831b76e4..115635a0a5 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -176,7 +176,25 @@ public class TestSparkReaderWithBloomFilter { +@@ -176,7 +176,27 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -1499,6 +1524,7 @@ index e5831b76e4..2433013d3e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1509,16 +1535,17 @@ index e5831b76e4..2433013d3e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); catalog = diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index 961d69b721..f12ac3e83b 100644 +index 961d69b721..fc9a0bffb8 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -68,6 +68,15 @@ public class TestStructuredStreaming { +@@ -68,6 +68,16 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -1529,16 +1556,17 @@ index 961d69b721..f12ac3e83b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index ac674e2e62..c18fed7cfb 100644 +index ac674e2e62..29a936c888 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -73,7 +73,19 @@ public class TestTimestampWithoutZone extends SparkTestBase { +@@ -73,7 +73,20 @@ public class TestTimestampWithoutZone extends SparkTestBase { @BeforeClass public static void startSpark() { @@ -1553,6 +1581,7 @@ index ac674e2e62..c18fed7cfb 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1560,10 +1589,10 @@ index ac674e2e62..c18fed7cfb 100644 @AfterClass diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 73827b309b..ace66913d1 100644 +index 73827b309b..f8ff9a51b3 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,20 @@ public class TestWriteMetricsConfig { @BeforeClass public static void startSpark() { @@ -1578,6 +1607,7 @@ index 73827b309b..ace66913d1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -1585,10 +1615,10 @@ index 73827b309b..ace66913d1 100644 } diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 1a4e2f3e1c..047e0f1fb4 100644 +index 1a4e2f3e1c..6977221c3f 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -65,7 +65,25 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { +@@ -65,7 +65,27 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -1599,6 +1629,7 @@ index 1a4e2f3e1c..047e0f1fb4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1609,6 +1640,7 @@ index 1a4e2f3e1c..047e0f1fb4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1673,10 +1705,10 @@ index 572c32f929..29013d26c0 100644 // runtime dependencies for running Hive Catalog based integration test integrationRuntimeOnly project(':iceberg-hive-metastore') diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -index 578845e3da..02b6dc270f 100644 +index 578845e3da..047980e7fd 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java -@@ -57,6 +57,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { +@@ -57,6 +57,16 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") .config( SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) @@ -1687,6 +1719,7 @@ index 578845e3da..02b6dc270f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -1707,10 +1740,10 @@ index b5d6415763..0763a723d3 100644 protected void createTableWithDeleteGranularity( diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -index ecf9e6f8a5..beca5f8521 100644 +index ecf9e6f8a5..42591bd9b2 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java -@@ -56,6 +56,15 @@ public class TestCallStatementParser { +@@ -56,6 +56,16 @@ public class TestCallStatementParser { .master("local[2]") .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) .config("spark.extra.prop", "value") @@ -1721,6 +1754,7 @@ index ecf9e6f8a5..beca5f8521 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); @@ -1755,10 +1789,10 @@ index 8cb92224fc..777b9bb1dd 100644 } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -index 64edb1002e..57767da086 100644 +index 64edb1002e..fc1d0f01ae 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java -@@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { +@@ -179,6 +179,16 @@ public class DeleteOrphanFilesBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) @@ -1769,16 +1803,17 @@ index 64edb1002e..57767da086 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local"); spark = builder.getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -index a5d0456b0b..f52c387943 100644 +index a5d0456b0b..4dd831bc29 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java -@@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { +@@ -392,6 +392,16 @@ public class IcebergSortCompactionBenchmark { "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) @@ -1789,16 +1824,17 @@ index a5d0456b0b..f52c387943 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]"); spark = builder.getOrCreate(); Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -index c6794e43c6..4c1a165a44 100644 +index c6794e43c6..90bde88cd5 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java -@@ -239,6 +239,15 @@ public class DVReaderBenchmark { +@@ -239,6 +239,16 @@ public class DVReaderBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1809,16 +1845,17 @@ index c6794e43c6..4c1a165a44 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -index ac74fb5a10..4aa5d5b361 100644 +index ac74fb5a10..07b8c34f8a 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java -@@ -223,6 +223,15 @@ public class DVWriterBenchmark { +@@ -223,6 +223,16 @@ public class DVWriterBenchmark { .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) .config("spark.sql.catalog.spark_catalog.type", "hadoop") .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) @@ -1829,16 +1866,17 @@ index ac74fb5a10..4aa5d5b361 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .master("local[*]") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -index 68c537e34a..2e421c7706 100644 +index 68c537e34a..cf8ef0db89 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java -@@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { +@@ -94,7 +94,19 @@ public abstract class IcebergSourceBenchmark { } protected void setupSpark(boolean enableDictionaryEncoding) { @@ -1853,6 +1891,7 @@ index 68c537e34a..2e421c7706 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g"); if (!enableDictionaryEncoding) { @@ -2640,10 +2679,10 @@ index 780e1750a5..25f253eede 100644 .filter(residual) .caseSensitive(caseSensitive()) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -index 404ba72846..b375458f05 100644 +index 404ba72846..77c4340dc4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java -@@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase +@@ -90,6 +90,16 @@ public abstract class SparkDistributedDataScanTestBase .master("local[2]") .config("spark.serializer", serializer) .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2654,16 +2693,17 @@ index 404ba72846..b375458f05 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -index 659507e4c5..2a3a87f079 100644 +index 659507e4c5..d44910e809 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java -@@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes +@@ -73,6 +73,16 @@ public class TestSparkDistributedDataScanDeletes .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2674,16 +2714,17 @@ index 659507e4c5..2a3a87f079 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -index a218f965ea..e30a8d6c6f 100644 +index a218f965ea..cfb96d03fb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java -@@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles +@@ -62,6 +62,16 @@ public class TestSparkDistributedDataScanFilterFiles .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2694,16 +2735,17 @@ index a218f965ea..e30a8d6c6f 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -index 2665d7ba8d..c24c82341a 100644 +index 2665d7ba8d..ef17afe769 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java -@@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting +@@ -63,6 +63,16 @@ public class TestSparkDistributedDataScanReporting .master("local[2]") .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") @@ -2714,16 +2756,17 @@ index 2665d7ba8d..c24c82341a 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -index 3e9f3334ef..0f96c07739 100644 +index 3e9f3334ef..3f7851c263 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java -@@ -77,6 +77,15 @@ public abstract class TestBase extends SparkTestHelperBase { +@@ -77,6 +77,16 @@ public abstract class TestBase extends SparkTestHelperBase { .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") @@ -2734,6 +2777,7 @@ index 3e9f3334ef..0f96c07739 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() @@ -2765,10 +2809,10 @@ index a31138ae01..501c8ac93e 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -index bc4e722bc8..57770308b3 100644 +index bc4e722bc8..e24dc5f275 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java -@@ -59,7 +59,19 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect +@@ -59,7 +59,20 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect @BeforeAll public static void startSpark() { @@ -2783,6 +2827,7 @@ index bc4e722bc8..57770308b3 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2790,10 +2835,10 @@ index bc4e722bc8..57770308b3 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -index 0886df957d..eca0f74533 100644 +index 0886df957d..ee1f955d55 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java -@@ -57,7 +57,19 @@ public abstract class ScanTestBase extends AvroDataTest { +@@ -57,7 +57,20 @@ public abstract class ScanTestBase extends AvroDataTest { @BeforeAll public static void startSpark() { @@ -2808,6 +2853,7 @@ index 0886df957d..eca0f74533 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2815,10 +2861,10 @@ index 0886df957d..eca0f74533 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index 6b7d861364..23419aa9b1 100644 +index 6b7d861364..82563ec088 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -144,7 +144,20 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -2833,6 +2879,7 @@ index 6b7d861364..23419aa9b1 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2840,10 +2887,10 @@ index 6b7d861364..23419aa9b1 100644 @BeforeEach diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -index c4ba96e634..dcf72c19b6 100644 +index c4ba96e634..db6acd8124 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java -@@ -75,7 +75,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { +@@ -75,7 +75,20 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { @BeforeAll public static void startSpark() { @@ -2858,6 +2905,7 @@ index c4ba96e634..dcf72c19b6 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2865,10 +2913,10 @@ index c4ba96e634..dcf72c19b6 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index 348173596e..03156f786b 100644 +index 348173596e..eb1194f709 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -@@ -110,7 +110,19 @@ public class TestFilteredScan { +@@ -110,7 +110,20 @@ public class TestFilteredScan { @BeforeAll public static void startSpark() { @@ -2883,6 +2931,7 @@ index 348173596e..03156f786b 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2890,10 +2939,10 @@ index 348173596e..03156f786b 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -index 84c99a575c..e647ab924e 100644 +index 84c99a575c..84d44e4478 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java -@@ -93,7 +93,19 @@ public class TestForwardCompatibility { +@@ -93,7 +93,20 @@ public class TestForwardCompatibility { @BeforeAll public static void startSpark() { @@ -2908,6 +2957,7 @@ index 84c99a575c..e647ab924e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2915,10 +2965,10 @@ index 84c99a575c..e647ab924e 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -index 7eff93d204..2fdc87a0a8 100644 +index 7eff93d204..a0a2a0845b 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java -@@ -46,7 +46,19 @@ public class TestIcebergSpark { +@@ -46,7 +46,20 @@ public class TestIcebergSpark { @BeforeAll public static void startSpark() { @@ -2933,6 +2983,7 @@ index 7eff93d204..2fdc87a0a8 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2940,10 +2991,10 @@ index 7eff93d204..2fdc87a0a8 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 9464f687b0..8653560334 100644 +index 9464f687b0..88edb4b6a0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -112,7 +112,19 @@ public class TestPartitionPruning { +@@ -112,7 +112,20 @@ public class TestPartitionPruning { @BeforeAll public static void startSpark() { @@ -2958,6 +3009,7 @@ index 9464f687b0..8653560334 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2965,10 +3017,10 @@ index 9464f687b0..8653560334 100644 String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 5c218f21c4..2f17343d59 100644 +index 5c218f21c4..bc0478daa0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -107,7 +107,19 @@ public class TestPartitionValues { +@@ -107,7 +107,20 @@ public class TestPartitionValues { @BeforeAll public static void startSpark() { @@ -2983,6 +3035,7 @@ index 5c218f21c4..2f17343d59 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -2990,10 +3043,10 @@ index 5c218f21c4..2f17343d59 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -index a7334a580c..68210dc8e8 100644 +index a7334a580c..7eebdc2239 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java -@@ -87,7 +87,19 @@ public class TestSnapshotSelection { +@@ -87,7 +87,20 @@ public class TestSnapshotSelection { @BeforeAll public static void startSpark() { @@ -3008,6 +3061,7 @@ index a7334a580c..68210dc8e8 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3015,10 +3069,10 @@ index a7334a580c..68210dc8e8 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -index 182b1ef8f5..76e04b1472 100644 +index 182b1ef8f5..628f14571c 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java -@@ -120,7 +120,19 @@ public class TestSparkDataFile { +@@ -120,7 +120,20 @@ public class TestSparkDataFile { @BeforeAll public static void startSpark() { @@ -3033,6 +3087,7 @@ index 182b1ef8f5..76e04b1472 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3040,10 +3095,10 @@ index 182b1ef8f5..76e04b1472 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index fb2b312bed..846f50e0c9 100644 +index fb2b312bed..03fc0d2171 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -96,7 +96,19 @@ public class TestSparkDataWrite { +@@ -96,7 +96,20 @@ public class TestSparkDataWrite { @BeforeAll public static void startSpark() { @@ -3058,13 +3113,14 @@ index fb2b312bed..846f50e0c9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); } @AfterEach -@@ -140,7 +152,7 @@ public class TestSparkDataWrite { +@@ -140,7 +153,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3073,7 +3129,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); for (ManifestFile manifest : -@@ -210,7 +222,7 @@ public class TestSparkDataWrite { +@@ -210,7 +223,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3082,7 +3138,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -256,7 +268,7 @@ public class TestSparkDataWrite { +@@ -256,7 +269,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3091,7 +3147,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -309,7 +321,7 @@ public class TestSparkDataWrite { +@@ -309,7 +322,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3100,7 +3156,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -352,7 +364,7 @@ public class TestSparkDataWrite { +@@ -352,7 +365,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3109,7 +3165,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -392,7 +404,7 @@ public class TestSparkDataWrite { +@@ -392,7 +405,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3118,7 +3174,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -458,7 +470,7 @@ public class TestSparkDataWrite { +@@ -458,7 +471,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3127,7 +3183,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); } -@@ -622,7 +634,7 @@ public class TestSparkDataWrite { +@@ -622,7 +635,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3136,7 +3192,7 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); assertThat(actual).as("Result rows should match").isEqualTo(expected); -@@ -708,7 +720,7 @@ public class TestSparkDataWrite { +@@ -708,7 +721,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -3146,10 +3202,10 @@ index fb2b312bed..846f50e0c9 100644 assertThat(actual) .describedAs("Result rows should match") diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index becf6a064d..91451c268e 100644 +index becf6a064d..f8e3729a0e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -83,7 +83,19 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -83,7 +83,20 @@ public class TestSparkReadProjection extends TestReadProjection { @BeforeAll public static void startSpark() { @@ -3164,6 +3220,7 @@ index becf6a064d..91451c268e 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3171,10 +3228,10 @@ index becf6a064d..91451c268e 100644 ImmutableMap.of( "type", "hive", diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 4f1cef5d37..4a1a602ed4 100644 +index 4f1cef5d37..db8738e14e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -136,6 +136,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -136,6 +136,16 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3185,12 +3242,13 @@ index 4f1cef5d37..4a1a602ed4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); -@@ -204,7 +213,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -204,7 +214,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -3201,10 +3259,10 @@ index 4f1cef5d37..4a1a602ed4 100644 @Override diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -index baf7fa8f88..b58321b761 100644 +index baf7fa8f88..2eaa17e8b0 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java -@@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { +@@ -182,6 +182,16 @@ public class TestSparkReaderWithBloomFilter { SparkSession.builder() .master("local[2]") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -3215,16 +3273,17 @@ index baf7fa8f88..b58321b761 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() .getOrCreate(); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -index c84a65cbe9..e17ba73ee4 100644 +index c84a65cbe9..9bb31efc35 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java -@@ -67,6 +67,15 @@ public class TestStructuredStreaming { +@@ -67,6 +67,16 @@ public class TestStructuredStreaming { SparkSession.builder() .master("local[2]") .config("spark.sql.shuffle.partitions", 4) @@ -3235,16 +3294,17 @@ index c84a65cbe9..e17ba73ee4 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .getOrCreate(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -index 306444b9f2..e4082d9cd9 100644 +index 306444b9f2..4a72089a97 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java -@@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { +@@ -75,7 +75,20 @@ public class TestTimestampWithoutZone extends TestBase { @BeforeAll public static void startSpark() { @@ -3259,6 +3319,7 @@ index 306444b9f2..e4082d9cd9 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3266,10 +3327,10 @@ index 306444b9f2..e4082d9cd9 100644 @AfterAll diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -index 841268a6be..97f7187314 100644 +index 841268a6be..1ca5f299b4 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java -@@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { +@@ -80,7 +80,20 @@ public class TestWriteMetricsConfig { @BeforeAll public static void startSpark() { @@ -3284,6 +3345,7 @@ index 841268a6be..97f7187314 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") + .getOrCreate(); @@ -3291,10 +3353,10 @@ index 841268a6be..97f7187314 100644 } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -index 6e09252704..4b0f2e4d58 100644 +index 6e09252704..1cd8b636f3 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java -@@ -60,6 +60,15 @@ public class TestAggregatePushDown extends CatalogTestBase { +@@ -60,6 +60,16 @@ public class TestAggregatePushDown extends CatalogTestBase { SparkSession.builder() .master("local[2]") .config("spark.sql.iceberg.aggregate_pushdown", "true") @@ -3305,6 +3367,7 @@ index 6e09252704..4b0f2e4d58 100644 + .config("spark.comet.explainFallback.enabled", "true") + .config("spark.comet.scan.icebergNative.enabled", "true") + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") + .config("spark.memory.offHeap.enabled", "true") + .config("spark.memory.offHeap.size", "10g") .enableHiveSupport() diff --git a/docs/source/contributor-guide/iceberg-spark-tests.md b/docs/source/contributor-guide/iceberg-spark-tests.md index df6c168af6..e91261695f 100644 --- a/docs/source/contributor-guide/iceberg-spark-tests.md +++ b/docs/source/contributor-guide/iceberg-spark-tests.md @@ -32,9 +32,14 @@ Here is an overview of the changes that the diffs make to Iceberg: - Configure test base classes (`TestBase`, `ExtensionsTestBase`, `ScanTestBase`, etc.) to load the Comet Spark plugin and shuffle manager - Enable the Iceberg write split-operator plan (`spark.comet.write.iceberg.splitOperator.enabled`) alongside the - native scan in every Comet-configured session. The flag is off by default for users, so Iceberg's own suites - are the only place the split plan (`IcebergCommit -> IcebergWrite`) is exercised against Iceberg's write, - commit, and row-level-operation tests. See [#5259] + native scan in every Comet-configured session, so that Iceberg's own write, commit, and row-level-operation + tests exercise the split plan (`IcebergCommit -> IcebergWrite`). See [#5259] +- Enable `spark.comet.exec.localTableScan.enabled` in the same sessions. `CometIcebergNativeWrite` sets + `requiresNativeChildren`, so without this flag a write fed by an inline `VALUES` list keeps Spark's row-based + `LocalTableScanExec`, the conversion is declined, and the write silently runs on the JVM writer. Many Iceberg + suites seed their data that way, so leaving it off hides the native writer from most of the write surface. +- Enable fallback logging (`spark.comet.explainFallback.enabled`) so that every operator Comet declines is + reported in the test output together with the reason it was declined. [#3739]: https://github.com/apache/datafusion-comet/pull/3739 [#5259]: https://github.com/apache/datafusion-comet/issues/5259 diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index f4aeacc847..2fc049521e 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -129,7 +129,7 @@ object CometConf extends ShimCometConf { "into Comet's two-operator shape: a file writer exec (inside AQE) and a committer " + "(outside AQE).") .booleanConf - .createWithDefault(false) + .createWithDefault(true) val COMET_ICEBERG_NATIVE_WRITE_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.iceberg.write.enabled") @@ -137,9 +137,9 @@ object CometConf extends ShimCometConf { .doc( "Whether to delegate the executor-side Parquet write to Comet's native (iceberg-rust) " + "writer when the table's properties allow it. Requires " + - "`spark.comet.write.iceberg.splitOperator.enabled = true`. Off by default.") + "`spark.comet.write.iceberg.splitOperator.enabled = true`.") .booleanConf - .createWithDefault(false) + .createWithDefault(true) val COMET_ICEBERG_DATA_FILE_CONCURRENCY_LIMIT: ConfigEntry[Int] = conf("spark.comet.scan.icebergNative.dataFileConcurrencyLimit")