From 2c045e1808931f59a60c8d7243351fa317569078 Mon Sep 17 00:00:00 2001 From: "Dailey, Alaric" Date: Tue, 20 Oct 2026 06:43:53 -0500 Subject: [PATCH 1/2] Implement GetBytes and GetChars GetBytes threw NotImplementedException and GetChars NotSupportedException. IDataRecord.GetBytes and GetChars read part of a value into a buffer, and "If you pass a buffer that is null" return the value's length. GetBytes reads the blob through GetStream; GetChars reads the string. Co-Authored-By: Claude Opus 5.5 --- DuckDB.NET.Data/DuckDBDataReader.cs | 71 +++++++++++++++++++++++- DuckDB.NET.Test/DuckDBDataReaderTests.cs | 60 ++++++++++++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBDataReader.cs b/DuckDB.NET.Data/DuckDBDataReader.cs index f9edb0be..20e63d24 100644 --- a/DuckDB.NET.Data/DuckDBDataReader.cs +++ b/DuckDB.NET.Data/DuckDBDataReader.cs @@ -167,9 +167,39 @@ public override byte GetByte(int ordinal) return GetFieldValue(ordinal); } + /// + /// IDataRecord.GetBytes: reads "from the specified column offset into the buffer as an array, starting at the given + /// buffer offset", and "If you pass a buffer that is null, GetBytes returns the length of the row in bytes." + /// public override long GetBytes(int ordinal, long dataOffset, byte[]? buffer, int bufferOffset, int length) { - throw new NotImplementedException(); + using var stream = GetStream(ordinal); + if (buffer is null) + { + return stream.Length; + } + + ValidateBufferArguments(dataOffset, buffer.Length, bufferOffset, length); + if (dataOffset >= stream.Length) + { + return 0; + } + + stream.Position = dataOffset; + var count = (int)Math.Min(length, stream.Length - dataOffset); + var read = 0; + while (read < count) + { + var chunk = stream.Read(buffer, bufferOffset + read, count - read); + if (chunk == 0) + { + break; + } + + read += chunk; + } + + return read; } public override char GetChar(int ordinal) @@ -177,9 +207,46 @@ public override char GetChar(int ordinal) throw new NotSupportedException(); } + /// + /// IDataRecord.GetChars: reads characters "from the specified column offset into the buffer as an array, starting at + /// the given buffer offset", and "If you pass a buffer that is null, GetChars returns the length of the field in + /// characters." + /// public override long GetChars(int ordinal, long dataOffset, char[]? buffer, int bufferOffset, int length) { - throw new NotSupportedException(); + var value = GetString(ordinal); + if (buffer is null) + { + return value.Length; + } + + ValidateBufferArguments(dataOffset, buffer.Length, bufferOffset, length); + if (dataOffset >= value.Length) + { + return 0; + } + + var count = (int)Math.Min(length, value.Length - dataOffset); + value.CopyTo((int)dataOffset, buffer, bufferOffset, count); + return count; + } + + private static void ValidateBufferArguments(long dataOffset, int bufferLength, int bufferOffset, int length) + { + if (dataOffset < 0) + { + throw new ArgumentOutOfRangeException(nameof(dataOffset), dataOffset, "The data offset cannot be negative."); + } + + if (bufferOffset < 0 || bufferOffset > bufferLength) + { + throw new ArgumentOutOfRangeException(nameof(bufferOffset), bufferOffset, "The buffer offset must be within the buffer."); + } + + if (length < 0 || length > bufferLength - bufferOffset) + { + throw new ArgumentOutOfRangeException(nameof(length), length, "The length must fit in the buffer after the buffer offset."); + } } public override string GetDataTypeName(int ordinal) diff --git a/DuckDB.NET.Test/DuckDBDataReaderTests.cs b/DuckDB.NET.Test/DuckDBDataReaderTests.cs index 133ff737..ef1a1484 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderTests.cs @@ -819,4 +819,64 @@ public void ReadIntegerAsAnotherIntegerType() reader.Invoking(r => r.GetFieldValue(1)).Should().Throw().WithMessage("Cannot cast from Int32 to UInt64 in column negative"); reader.Invoking(r => r.GetFieldValue(2)).Should().Throw().WithMessage("Cannot cast from Int16 to Byte in column small"); } + + // https://learn.microsoft.com/dotnet/api/system.data.idatarecord.getbytes: reads "from the specified column offset + // into the buffer as an array, starting at the given buffer offset"; "If you pass a buffer that is null, GetBytes + // returns the length of the row in bytes." + [Fact] + public void GetBytesReadsABlobWholeOrInParts() + { + Command.CommandText = "SELECT '\\x01\\x02\\x03\\x04'::BLOB"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + + reader.GetBytes(0, 0, null, 0, 0).Should().Be(4); + + var whole = new byte[6]; + reader.GetBytes(0, 0, whole, 0, 6).Should().Be(4); + whole.Should().Equal(1, 2, 3, 4, 0, 0); + + var part = new byte[5]; + reader.GetBytes(0, 1, part, 2, 2).Should().Be(2); + part.Should().Equal(0, 0, 2, 3, 0); + + reader.GetBytes(0, 4, part, 0, 1).Should().Be(0, "there is nothing past the end of the value"); + } + + // https://learn.microsoft.com/dotnet/api/system.data.idatarecord.getchars: "If you pass a buffer that is null, + // GetChars returns the length of the field in characters." + [Fact] + public void GetCharsReadsAStringWholeOrInParts() + { + Command.CommandText = "SELECT 'ab¢d'"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + + reader.GetChars(0, 0, null, 0, 0).Should().Be(4, "the length is in characters, not UTF-8 bytes"); + + var whole = new char[6]; + reader.GetChars(0, 0, whole, 0, 6).Should().Be(4); + whole.Should().Equal('a', 'b', '¢', 'd', '\0', '\0'); + + var part = new char[5]; + reader.GetChars(0, 1, part, 2, 2).Should().Be(2); + part.Should().Equal('\0', '\0', 'b', '¢', '\0'); + + reader.GetChars(0, 4, part, 0, 1).Should().Be(0); + } + + [Fact] + public void GetBytesAndGetCharsRejectOffsetsOutsideTheBuffer() + { + Command.CommandText = "SELECT '\\x01\\x02'::BLOB, 'ab'"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + + reader.Invoking(r => r.GetBytes(0, -1, new byte[2], 0, 2)).Should().Throw(); + reader.Invoking(r => r.GetBytes(0, 0, new byte[2], 3, 1)).Should().Throw(); + reader.Invoking(r => r.GetBytes(0, 0, new byte[2], 1, 2)).Should().Throw(); + reader.Invoking(r => r.GetChars(1, -1, new char[2], 0, 2)).Should().Throw(); + reader.Invoking(r => r.GetChars(1, 0, new char[2], 1, 2)).Should().Throw(); + } } + From 087521542674d8cc2c4c4c64805c38ba8e414ddf Mon Sep 17 00:00:00 2001 From: "Dailey, Alaric" Date: Tue, 20 Oct 2026 08:07:26 -0500 Subject: [PATCH 2/2] Implement chunked UTF-8 GetChars reads --- .../Reader/StringVectorDataReader.cs | 74 +++++++++++++++++++ DuckDB.NET.Data/DuckDBDataReader.cs | 18 ++--- DuckDB.NET.Test/DuckDBDataReaderTests.cs | 31 ++++++++ 3 files changed, 113 insertions(+), 10 deletions(-) diff --git a/DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs b/DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs index 794d4f8a..548aa42a 100644 --- a/DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs +++ b/DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.IO; using System.Text; @@ -99,6 +100,79 @@ private unsafe string GetString(ulong offset) return Encoding.UTF8.GetString((byte*)data->Data, data->Length); } + internal unsafe long GetChars(ulong offset, long dataOffset, char[]? buffer, int bufferOffset, int length) + { + if (DuckDBType != DuckDBType.Varchar) + { + throw new NotSupportedException($"GetChars is not supported for {DuckDBType} columns"); + } + + if (!IsValid(offset)) + { + throw new InvalidCastException($"Column '{ColumnName}' value is null"); + } + + var data = (DuckDBString*)DataPointer + offset; + var bytes = new ReadOnlySpan((byte*)data->Data, data->Length); + var byteOffset = 0; + var characterOffset = 0L; + var copied = 0; + + while (byteOffset < bytes.Length) + { + var status = Rune.DecodeFromUtf8(bytes[byteOffset..], out var rune, out var bytesConsumed); + if (status != OperationStatus.Done) + { + rune = Rune.ReplacementChar; + bytesConsumed = 1; + } + + var runeLength = rune.Utf16SequenceLength; + var charactersToSkip = dataOffset - characterOffset; + if (charactersToSkip > 0) + { + if (charactersToSkip < runeLength && buffer != null && copied < length) + { + buffer[bufferOffset + copied++] = GetSurrogate(rune, charactersToSkip); + } + } + else if (buffer != null && copied < length) + { + var charactersToCopy = Math.Min(runeLength, length - copied); + WriteRune(rune, buffer, bufferOffset + copied, charactersToCopy); + copied += charactersToCopy; + } + + byteOffset += bytesConsumed; + characterOffset += runeLength; + + if (buffer != null && copied == length) + { + return copied; + } + } + + return buffer == null ? characterOffset : copied; + } + + private static char GetSurrogate(Rune rune, long offset) + { + var value = rune.Value - 0x10000; + return (char)(offset == 0 ? (value >> 10) + 0xD800 : (value & 0x3FF) + 0xDC00); + } + + private static void WriteRune(Rune rune, char[] buffer, int offset, int length) + { + buffer[offset] = rune.Utf16SequenceLength == 1 + ? (char)rune.Value + : GetSurrogate(rune, 0); + + if (length > 1) + { + buffer[offset + 1] = GetSurrogate(rune, 1); + } + } + private unsafe Stream GetStream(ulong offset) { var data = (DuckDBString*)DataPointer + offset; diff --git a/DuckDB.NET.Data/DuckDBDataReader.cs b/DuckDB.NET.Data/DuckDBDataReader.cs index 5fe11a4a..4bfdf295 100644 --- a/DuckDB.NET.Data/DuckDBDataReader.cs +++ b/DuckDB.NET.Data/DuckDBDataReader.cs @@ -215,21 +215,19 @@ public override char GetChar(int ordinal) /// public override long GetChars(int ordinal, long dataOffset, char[]? buffer, int bufferOffset, int length) { - var value = GetString(ordinal); - if (buffer is null) + CheckRowRead(); + + if (buffer != null) { - return value.Length; + ValidateBufferArguments(dataOffset, buffer.Length, bufferOffset, length); } - ValidateBufferArguments(dataOffset, buffer.Length, bufferOffset, length); - if (dataOffset >= value.Length) + if (vectorReaders[ordinal] is not StringVectorDataReader reader) { - return 0; + throw new NotSupportedException($"GetChars is not supported for {vectorReaders[ordinal].DuckDBType} columns"); } - var count = (int)Math.Min(length, value.Length - dataOffset); - value.CopyTo((int)dataOffset, buffer, bufferOffset, count); - return count; + return reader.GetChars(rowsReadFromCurrentChunk - 1, dataOffset, buffer, bufferOffset, length); } private static void ValidateBufferArguments(long dataOffset, int bufferLength, int bufferOffset, int length) @@ -522,4 +520,4 @@ private static void ThrowNoRowRead() { throw new InvalidOperationException("No row has been read"); } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Test/DuckDBDataReaderTests.cs b/DuckDB.NET.Test/DuckDBDataReaderTests.cs index 87052e10..abf5ca94 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderTests.cs @@ -752,6 +752,37 @@ public void ReadStringsOfDifferentLengths() reader.IsDBNull(6).Should().BeTrue(); } + [Fact] + public void GetCharsReadsLargeUtf8ValuesInUtf16Chunks() + { + var expected = string.Concat(Enumerable.Repeat("aéქართული😀𐐷", 2_000)); + Command.CommandText = "SELECT ?::VARCHAR"; + Command.Parameters.Add(new DuckDBParameter(expected)); + + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.GetChars(0, 0, null, 0, 0).Should().Be(expected.Length); + + var buffer = new char[1_024]; + var result = new StringBuilder(expected.Length); + for (var dataOffset = 0L; dataOffset < expected.Length;) + { + var read = reader.GetChars(0, dataOffset, buffer, 0, buffer.Length); + result.Append(buffer, 0, (int)read); + dataOffset += read; + } + + result.ToString().Should().Be(expected); + + var surrogateOffset = expected.IndexOf("😀", StringComparison.Ordinal) + 1; + var splitBuffer = new char[1]; + reader.GetChars(0, surrogateOffset, splitBuffer, 0, 1).Should().Be(1); + splitBuffer[0].Should().Be(expected[surrogateOffset]); + reader.GetChars(0, surrogateOffset + 1, splitBuffer, 0, 1).Should().Be(1); + splitBuffer[0].Should().Be(expected[surrogateOffset + 1]); + } + [Fact] public void ReadHugeIntAsDuckDBHugeInt() {