Implement GetBytes and GetChars

DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs

@@ -1,3 +1,4 @@
+using System.Buffers;
 using System.IO;
 using System.Text;
 
@@ -99,6 +100,79 @@ private unsafe string GetString(ulong offset)
         return Encoding.UTF8.GetString((byte*)data->Data, data->Length);
     }
 
+    internal unsafe long GetChars(ulong offset, long dataOffset, char[]? buffer, int bufferOffset, int length)
+    {
+        if (DuckDBType != DuckDBType.Varchar)
+        {
+            throw new NotSupportedException($"GetChars is not supported for {DuckDBType} columns");
+        }
+
+        if (!IsValid(offset))
+        {
+            throw new InvalidCastException($"Column '{ColumnName}' value is null");
+        }
+
+        var data = (DuckDBString*)DataPointer + offset;
+        var bytes = new ReadOnlySpan<byte>((byte*)data->Data, data->Length);
+        var byteOffset = 0;
+        var characterOffset = 0L;
+        var copied = 0;
+
+        while (byteOffset < bytes.Length)
+        {
+            var status = Rune.DecodeFromUtf8(bytes[byteOffset..], out var rune, out var bytesConsumed);
+            if (status != OperationStatus.Done)
+            {
+                rune = Rune.ReplacementChar;
+                bytesConsumed = 1;
+            }
+
+            var runeLength = rune.Utf16SequenceLength;
+            var charactersToSkip = dataOffset - characterOffset;
+            if (charactersToSkip > 0)
+            {
+                if (charactersToSkip < runeLength && buffer != null && copied < length)
+                {
+                    buffer[bufferOffset + copied++] = GetSurrogate(rune, charactersToSkip);
+                }
+            }
+            else if (buffer != null && copied < length)
+            {
+                var charactersToCopy = Math.Min(runeLength, length - copied);
+                WriteRune(rune, buffer, bufferOffset + copied, charactersToCopy);
+                copied += charactersToCopy;
+            }
+
+            byteOffset += bytesConsumed;
+            characterOffset += runeLength;
+
+            if (buffer != null && copied == length)
+            {
+                return copied;
+            }
+        }
+
+        return buffer == null ? characterOffset : copied;
+    }
+
+    private static char GetSurrogate(Rune rune, long offset)
+    {
+        var value = rune.Value - 0x10000;
+        return (char)(offset == 0 ? (value >> 10) + 0xD800 : (value & 0x3FF) + 0xDC00);
+    }
+
+    private static void WriteRune(Rune rune, char[] buffer, int offset, int length)
+    {
+        buffer[offset] = rune.Utf16SequenceLength == 1
+            ? (char)rune.Value
+            : GetSurrogate(rune, 0);
+
+        if (length > 1)
+        {
+            buffer[offset + 1] = GetSurrogate(rune, 1);
+        }
+    }
+
     private unsafe Stream GetStream(ulong offset)
     {
         var data = (DuckDBString*)DataPointer + offset;

DuckDB.NET.Data/DuckDBDataReader.cs

@@ -168,19 +168,84 @@ public override byte GetByte(int ordinal)
         return GetFieldValue<byte>(ordinal);
     }
 
+    /// <summary>
+    /// IDataRecord.GetBytes: reads "from the specified column offset into the buffer as an array, starting at the given
+    /// buffer offset", and "If you pass a buffer that is null, GetBytes returns the length of the row in bytes."
+    /// </summary>
     public override long GetBytes(int ordinal, long dataOffset, byte[]? buffer, int bufferOffset, int length)
     {
-        throw new NotImplementedException();
+        using var stream = GetStream(ordinal);
+        if (buffer is null)
+        {
+            return stream.Length;
+        }
+
+        ValidateBufferArguments(dataOffset, buffer.Length, bufferOffset, length);
+        if (dataOffset >= stream.Length)
+        {
+            return 0;
+        }
+
+        stream.Position = dataOffset;
+        var count = (int)Math.Min(length, stream.Length - dataOffset);
+        var read = 0;
+        while (read < count)
+        {
+            var chunk = stream.Read(buffer, bufferOffset + read, count - read);
+            if (chunk == 0)
+            {
+                break;
+            }
+
+            read += chunk;
+        }
+
+        return read;
     }
 
     public override char GetChar(int ordinal)
     {
         throw new NotSupportedException();
     }
 
+    /// <summary>
+    /// IDataRecord.GetChars: reads characters "from the specified column offset into the buffer as an array, starting at
+    /// the given buffer offset", and "If you pass a buffer that is null, GetChars returns the length of the field in
+    /// characters."
+    /// </summary>
     public override long GetChars(int ordinal, long dataOffset, char[]? buffer, int bufferOffset, int length)
     {
-        throw new NotSupportedException();
+        CheckRowRead();
+
+        if (buffer != null)
+        {
+            ValidateBufferArguments(dataOffset, buffer.Length, bufferOffset, length);
+        }
+
+        if (vectorReaders[ordinal] is not StringVectorDataReader reader)
+        {
+            throw new NotSupportedException($"GetChars is not supported for {vectorReaders[ordinal].DuckDBType} columns");
+        }
+
+        return reader.GetChars(rowsReadFromCurrentChunk - 1, dataOffset, buffer, bufferOffset, length);
+    }
+
+    private static void ValidateBufferArguments(long dataOffset, int bufferLength, int bufferOffset, int length)
+    {
+        if (dataOffset < 0)
+        {
+            throw new ArgumentOutOfRangeException(nameof(dataOffset), dataOffset, "The data offset cannot be negative.");
+        }
+
+        if (bufferOffset < 0 || bufferOffset > bufferLength)
+        {
+            throw new ArgumentOutOfRangeException(nameof(bufferOffset), bufferOffset, "The buffer offset must be within the buffer.");
+        }
+
+        if (length < 0 || length > bufferLength - bufferOffset)
+        {
+            throw new ArgumentOutOfRangeException(nameof(length), length, "The length must fit in the buffer after the buffer offset.");
+        }
     }
 
     public override string GetDataTypeName(int ordinal)
@@ -460,4 +525,4 @@ private static void ThrowNoRowRead()
     {
         throw new InvalidOperationException("No row has been read");
     }
-}
+}

DuckDB.NET.Test/DuckDBDataReaderTests.cs

@@ -752,6 +752,37 @@ public void ReadStringsOfDifferentLengths()
         reader.IsDBNull(6).Should().BeTrue();
     }
 
+    [Fact]
+    public void GetCharsReadsLargeUtf8ValuesInUtf16Chunks()
+    {
+        var expected = string.Concat(Enumerable.Repeat("aéქართული😀𐐷", 2_000));
+        Command.CommandText = "SELECT ?::VARCHAR";
+        Command.Parameters.Add(new DuckDBParameter(expected));
+
+        using var reader = Command.ExecuteReader();
+        reader.Read();
+
+        reader.GetChars(0, 0, null, 0, 0).Should().Be(expected.Length);
+
+        var buffer = new char[1_024];
+        var result = new StringBuilder(expected.Length);
+        for (var dataOffset = 0L; dataOffset < expected.Length;)
+        {
+            var read = reader.GetChars(0, dataOffset, buffer, 0, buffer.Length);
+            result.Append(buffer, 0, (int)read);
+            dataOffset += read;
+        }
+
+        result.ToString().Should().Be(expected);
+
+        var surrogateOffset = expected.IndexOf("😀", StringComparison.Ordinal) + 1;
+        var splitBuffer = new char[1];
+        reader.GetChars(0, surrogateOffset, splitBuffer, 0, 1).Should().Be(1);
+        splitBuffer[0].Should().Be(expected[surrogateOffset]);
+        reader.GetChars(0, surrogateOffset + 1, splitBuffer, 0, 1).Should().Be(1);
+        splitBuffer[0].Should().Be(expected[surrogateOffset + 1]);
+    }
+
     [Fact]
     public void ReadHugeIntAsDuckDBHugeInt()
     {
@@ -844,6 +875,65 @@ public void GetOrdinalThrowsIndexOutOfRangeForAnUnknownName()
         reader.Invoking(r => r.GetOrdinal("missing")).Should().Throw<IndexOutOfRangeException>();
     }
 
+    // https://learn.microsoft.com/dotnet/api/system.data.idatarecord.getbytes: reads "from the specified column offset
+    // into the buffer as an array, starting at the given buffer offset"; "If you pass a buffer that is null, GetBytes
+    // returns the length of the row in bytes."
+    [Fact]
+    public void GetBytesReadsABlobWholeOrInParts()
+    {
+        Command.CommandText = "SELECT '\\x01\\x02\\x03\\x04'::BLOB";
+        using var reader = Command.ExecuteReader();
+        reader.Read().Should().BeTrue();
+
+        reader.GetBytes(0, 0, null, 0, 0).Should().Be(4);
+
+        var whole = new byte[6];
+        reader.GetBytes(0, 0, whole, 0, 6).Should().Be(4);
+        whole.Should().Equal(1, 2, 3, 4, 0, 0);
+
+        var part = new byte[5];
+        reader.GetBytes(0, 1, part, 2, 2).Should().Be(2);
+        part.Should().Equal(0, 0, 2, 3, 0);
+
+        reader.GetBytes(0, 4, part, 0, 1).Should().Be(0, "there is nothing past the end of the value");
+    }
+
+    // https://learn.microsoft.com/dotnet/api/system.data.idatarecord.getchars: "If you pass a buffer that is null,
+    // GetChars returns the length of the field in characters."
+    [Fact]
+    public void GetCharsReadsAStringWholeOrInParts()
+    {
+        Command.CommandText = "SELECT 'ab¢d'";
+        using var reader = Command.ExecuteReader();
+        reader.Read().Should().BeTrue();
+
+        reader.GetChars(0, 0, null, 0, 0).Should().Be(4, "the length is in characters, not UTF-8 bytes");
+
+        var whole = new char[6];
+        reader.GetChars(0, 0, whole, 0, 6).Should().Be(4);
+        whole.Should().Equal('a', 'b', '¢', 'd', '\0', '\0');
+
+        var part = new char[5];
+        reader.GetChars(0, 1, part, 2, 2).Should().Be(2);
+        part.Should().Equal('\0', '\0', 'b', '¢', '\0');
+
+        reader.GetChars(0, 4, part, 0, 1).Should().Be(0);
+    }
+
+    [Fact]
+    public void GetBytesAndGetCharsRejectOffsetsOutsideTheBuffer()
+    {
+        Command.CommandText = "SELECT '\\x01\\x02'::BLOB, 'ab'";
+        using var reader = Command.ExecuteReader();
+        reader.Read().Should().BeTrue();
+
+        reader.Invoking(r => r.GetBytes(0, -1, new byte[2], 0, 2)).Should().Throw<ArgumentOutOfRangeException>();
+        reader.Invoking(r => r.GetBytes(0, 0, new byte[2], 3, 1)).Should().Throw<ArgumentOutOfRangeException>();
+        reader.Invoking(r => r.GetBytes(0, 0, new byte[2], 1, 2)).Should().Throw<ArgumentOutOfRangeException>();
+        reader.Invoking(r => r.GetChars(1, -1, new char[2], 0, 2)).Should().Throw<ArgumentOutOfRangeException>();
+        reader.Invoking(r => r.GetChars(1, 0, new char[2], 1, 2)).Should().Throw<ArgumentOutOfRangeException>();
+    }
+
     // https://learn.microsoft.com/dotnet/api/system.data.idatarecord.getvalues: "You can pass an Object array that
     // contains fewer than the number of columns that are contained in the resulting row. Only the amount of data the
     // Object array holds is copied to the array. You can also pass an Object array whose length is more than the