From 445ef772bf4c40d95719ee4bf019237150d7b954 Mon Sep 17 00:00:00 2001
From: Andy Grove <agrove@apache.org>
Date: Wed, 28 Oct 2026 13:47:21 -0600
Subject: [PATCH] deps: bump arrow-java to 19.0.0

Arrow Java 19 no longer closes the source ArrowArray struct once
ArrayImporter moves it into its own copy (GH-765), so ArrowImporter now
closes it itself. Without that, every imported column leaked its
ArrowArray struct in CometArrowImportAllocator.

BitVectorHelper.getValidityBufferSize is replaced by
getValidityBufferSizeFromCount, and setValueCount(0) now allocates an
offset buffer on large variable-width vectors, so the empty-vector test
uses an unallocated vector instead.
---
 pom.xml                                               |  2 +-
 .../main/java/org/apache/arrow/c/ArrowImporter.java   | 11 ++++++++---
 .../scala/org/apache/spark/sql/comet/util/Utils.scala |  2 +-
 .../org/apache/spark/sql/comet/util/UtilsSuite.scala  |  8 ++++----
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/pom.xml b/pom.xml
index a215e2a818f..0d74b56a455 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,7 +88,7 @@ under the License.
     <parquet.version>1.16.0</parquet.version>
     <parquet.maven.scope>provided</parquet.maven.scope>
     <hadoop.version>3.3.4</hadoop.version>
-    <arrow.version>18.3.0</arrow.version>
+    <arrow.version>19.0.0</arrow.version>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
     <spotless.version>2.43.0</spotless.version>
     <!-- semanticdb-scalac is published per Scala patch version, so the 2.13 profiles need a
diff --git a/spark/src/main/java/org/apache/arrow/c/ArrowImporter.java b/spark/src/main/java/org/apache/arrow/c/ArrowImporter.java
index 94c4916f9f1..1485faf0ba2 100644
--- a/spark/src/main/java/org/apache/arrow/c/ArrowImporter.java
+++ b/spark/src/main/java/org/apache/arrow/c/ArrowImporter.java
@@ -60,6 +60,11 @@ public FieldVector importVector(
       vector = field.createVector(allocator);
       ArrayImporter importer = new ArrayImporter(allocator, vector, provider);
       importer.importArray(array);
+      // Arrow Java 19 stopped closing the source struct once its contents move into the importer's
+      // copy (GH-765), so free it here; earlier versions closed it already.
+      if (!array.isClosed()) {
+        array.close();
+      }
       return vector;
     } catch (RuntimeException | Error failure) {
       if (vector != null) {
@@ -68,9 +73,9 @@ public FieldVector importVector(
       if (field != null) {
         closeDictionaries(field, provider, failure);
       }
-      // ArrayImporter.importArray moves the array into a copy it owns and closes the source
-      // before doImport can fail, so a closed array means the importer already took it and
-      // releasing it again would be a double release.
+      // ArrayImporter.importArray moves the array into a copy it owns before doImport can fail, so
+      // the source's release callback is already cleared and release() is then a no-op. Earlier
+      // Arrow versions also closed the source, and a closed array must not be released again.
       if (!array.isClosed()) {
         AutoCloseables.close(failure, array::release, array);
       }
diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
index 975d4b05823..8bfb46c94cc 100644
--- a/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
+++ b/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
@@ -789,7 +789,7 @@ object Utils extends CometTypeShim with Logging {
       if (numValues > 0) {
         // An Arrow Java vector has no array offset, so validity bit i belongs to value i even when
         // the offsets start past zero, and the bitmap is copied from bit 0.
-        val validityBytes = BitVectorHelper.getValidityBufferSize(numValues).toLong
+        val validityBytes = BitVectorHelper.getValidityBufferSizeFromCount(numValues).toLong
         narrow.getValidityBuffer.setBytes(0, large.getValidityBuffer, 0, validityBytes)
         val offsets = narrow.getOffsetBuffer
         (0 to numValues).foreach { i =>
diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala
index 29c4547ccd3..64e8a651386 100644
--- a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala
@@ -301,12 +301,11 @@ class UtilsSuite extends CometTestBase {
   }
 
   test("serializeBatches writes empty large-offset vectors with 32-bit offsets") {
-    // A vector that holds no values need not have an offset buffer, so none is read.
+    // A vector that holds no values need not have an offset buffer, so none is read. Arrow 19's
+    // setValueCount(0) allocates one, so the test leaves the vectors unallocated.
     val strings = new LargeVarCharVector("s", CometArrowAllocator)
     val binaries = new LargeVarBinaryVector("b", CometArrowAllocator)
     try {
-      strings.setValueCount(0)
-      binaries.setValueCount(0)
       assert(strings.getOffsetBuffer.capacity == 0)
       val batch = new ColumnarBatch(Array(cometVector(strings), cometVector(binaries)), 0)
       assert(
@@ -487,7 +486,8 @@ class UtilsSuite extends CometTestBase {
     val width = BaseLargeVariableWidthVector.OFFSET_WIDTH.toLong
 
     val validity =
-      CometArrowAllocator.buffer(BitVectorHelper.getValidityBufferSize(values.length).toLong)
+      CometArrowAllocator.buffer(
+        BitVectorHelper.getValidityBufferSizeFromCount(values.length).toLong)
     val offsetBuffer = CometArrowAllocator.buffer(offsets.length * width)
     val data = CometArrowAllocator.buffer(offsets.last)
     val buffers = Seq(validity, offsetBuffer, data)