diff --git a/DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs b/DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs index 794d4f8a..548aa42a 100644 --- a/DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs +++ b/DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.IO; using System.Text; @@ -99,6 +100,79 @@ private unsafe string GetString(ulong offset) return Encoding.UTF8.GetString((byte*)data->Data, data->Length); } + internal unsafe long GetChars(ulong offset, long dataOffset, char[]? buffer, int bufferOffset, int length) + { + if (DuckDBType != DuckDBType.Varchar) + { + throw new NotSupportedException($"GetChars is not supported for {DuckDBType} columns"); + } + + if (!IsValid(offset)) + { + throw new InvalidCastException($"Column '{ColumnName}' value is null"); + } + + var data = (DuckDBString*)DataPointer + offset; + var bytes = new ReadOnlySpan((byte*)data->Data, data->Length); + var byteOffset = 0; + var characterOffset = 0L; + var copied = 0; + + while (byteOffset < bytes.Length) + { + var status = Rune.DecodeFromUtf8(bytes[byteOffset..], out var rune, out var bytesConsumed); + if (status != OperationStatus.Done) + { + rune = Rune.ReplacementChar; + bytesConsumed = 1; + } + + var runeLength = rune.Utf16SequenceLength; + var charactersToSkip = dataOffset - characterOffset; + if (charactersToSkip > 0) + { + if (charactersToSkip < runeLength && buffer != null && copied < length) + { + buffer[bufferOffset + copied++] = GetSurrogate(rune, charactersToSkip); + } + } + else if (buffer != null && copied < length) + { + var charactersToCopy = Math.Min(runeLength, length - copied); + WriteRune(rune, buffer, bufferOffset + copied, charactersToCopy); + copied += charactersToCopy; + } + + byteOffset += bytesConsumed; + characterOffset += runeLength; + + if (buffer != null && copied == length) + { + return copied; + } + } + + return buffer == null ? characterOffset : copied; + } + + private static char GetSurrogate(Rune rune, long offset) + { + var value = rune.Value - 0x10000; + return (char)(offset == 0 ? (value >> 10) + 0xD800 : (value & 0x3FF) + 0xDC00); + } + + private static void WriteRune(Rune rune, char[] buffer, int offset, int length) + { + buffer[offset] = rune.Utf16SequenceLength == 1 + ? (char)rune.Value + : GetSurrogate(rune, 0); + + if (length > 1) + { + buffer[offset + 1] = GetSurrogate(rune, 1); + } + } + private unsafe Stream GetStream(ulong offset) { var data = (DuckDBString*)DataPointer + offset; diff --git a/DuckDB.NET.Data/DuckDBDataReader.cs b/DuckDB.NET.Data/DuckDBDataReader.cs index 2e750c89..750db6b5 100644 --- a/DuckDB.NET.Data/DuckDBDataReader.cs +++ b/DuckDB.NET.Data/DuckDBDataReader.cs @@ -168,9 +168,39 @@ public override byte GetByte(int ordinal) return GetFieldValue(ordinal); } + /// + /// IDataRecord.GetBytes: reads "from the specified column offset into the buffer as an array, starting at the given + /// buffer offset", and "If you pass a buffer that is null, GetBytes returns the length of the row in bytes." + /// public override long GetBytes(int ordinal, long dataOffset, byte[]? buffer, int bufferOffset, int length) { - throw new NotImplementedException(); + using var stream = GetStream(ordinal); + if (buffer is null) + { + return stream.Length; + } + + ValidateBufferArguments(dataOffset, buffer.Length, bufferOffset, length); + if (dataOffset >= stream.Length) + { + return 0; + } + + stream.Position = dataOffset; + var count = (int)Math.Min(length, stream.Length - dataOffset); + var read = 0; + while (read < count) + { + var chunk = stream.Read(buffer, bufferOffset + read, count - read); + if (chunk == 0) + { + break; + } + + read += chunk; + } + + return read; } public override char GetChar(int ordinal) @@ -178,9 +208,44 @@ public override char GetChar(int ordinal) throw new NotSupportedException(); } + /// + /// IDataRecord.GetChars: reads characters "from the specified column offset into the buffer as an array, starting at + /// the given buffer offset", and "If you pass a buffer that is null, GetChars returns the length of the field in + /// characters." + /// public override long GetChars(int ordinal, long dataOffset, char[]? buffer, int bufferOffset, int length) { - throw new NotSupportedException(); + CheckRowRead(); + + if (buffer != null) + { + ValidateBufferArguments(dataOffset, buffer.Length, bufferOffset, length); + } + + if (vectorReaders[ordinal] is not StringVectorDataReader reader) + { + throw new NotSupportedException($"GetChars is not supported for {vectorReaders[ordinal].DuckDBType} columns"); + } + + return reader.GetChars(rowsReadFromCurrentChunk - 1, dataOffset, buffer, bufferOffset, length); + } + + private static void ValidateBufferArguments(long dataOffset, int bufferLength, int bufferOffset, int length) + { + if (dataOffset < 0) + { + throw new ArgumentOutOfRangeException(nameof(dataOffset), dataOffset, "The data offset cannot be negative."); + } + + if (bufferOffset < 0 || bufferOffset > bufferLength) + { + throw new ArgumentOutOfRangeException(nameof(bufferOffset), bufferOffset, "The buffer offset must be within the buffer."); + } + + if (length < 0 || length > bufferLength - bufferOffset) + { + throw new ArgumentOutOfRangeException(nameof(length), length, "The length must fit in the buffer after the buffer offset."); + } } public override string GetDataTypeName(int ordinal) @@ -460,4 +525,4 @@ private static void ThrowNoRowRead() { throw new InvalidOperationException("No row has been read"); } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Test/DuckDBDataReaderTests.cs b/DuckDB.NET.Test/DuckDBDataReaderTests.cs index 1ff6eb36..18355ed3 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderTests.cs @@ -752,6 +752,37 @@ public void ReadStringsOfDifferentLengths() reader.IsDBNull(6).Should().BeTrue(); } + [Fact] + public void GetCharsReadsLargeUtf8ValuesInUtf16Chunks() + { + var expected = string.Concat(Enumerable.Repeat("aéქართული😀𐐷", 2_000)); + Command.CommandText = "SELECT ?::VARCHAR"; + Command.Parameters.Add(new DuckDBParameter(expected)); + + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.GetChars(0, 0, null, 0, 0).Should().Be(expected.Length); + + var buffer = new char[1_024]; + var result = new StringBuilder(expected.Length); + for (var dataOffset = 0L; dataOffset < expected.Length;) + { + var read = reader.GetChars(0, dataOffset, buffer, 0, buffer.Length); + result.Append(buffer, 0, (int)read); + dataOffset += read; + } + + result.ToString().Should().Be(expected); + + var surrogateOffset = expected.IndexOf("😀", StringComparison.Ordinal) + 1; + var splitBuffer = new char[1]; + reader.GetChars(0, surrogateOffset, splitBuffer, 0, 1).Should().Be(1); + splitBuffer[0].Should().Be(expected[surrogateOffset]); + reader.GetChars(0, surrogateOffset + 1, splitBuffer, 0, 1).Should().Be(1); + splitBuffer[0].Should().Be(expected[surrogateOffset + 1]); + } + [Fact] public void ReadHugeIntAsDuckDBHugeInt() { @@ -844,6 +875,65 @@ public void GetOrdinalThrowsIndexOutOfRangeForAnUnknownName() reader.Invoking(r => r.GetOrdinal("missing")).Should().Throw(); } + // https://learn.microsoft.com/dotnet/api/system.data.idatarecord.getbytes: reads "from the specified column offset + // into the buffer as an array, starting at the given buffer offset"; "If you pass a buffer that is null, GetBytes + // returns the length of the row in bytes." + [Fact] + public void GetBytesReadsABlobWholeOrInParts() + { + Command.CommandText = "SELECT '\\x01\\x02\\x03\\x04'::BLOB"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + + reader.GetBytes(0, 0, null, 0, 0).Should().Be(4); + + var whole = new byte[6]; + reader.GetBytes(0, 0, whole, 0, 6).Should().Be(4); + whole.Should().Equal(1, 2, 3, 4, 0, 0); + + var part = new byte[5]; + reader.GetBytes(0, 1, part, 2, 2).Should().Be(2); + part.Should().Equal(0, 0, 2, 3, 0); + + reader.GetBytes(0, 4, part, 0, 1).Should().Be(0, "there is nothing past the end of the value"); + } + + // https://learn.microsoft.com/dotnet/api/system.data.idatarecord.getchars: "If you pass a buffer that is null, + // GetChars returns the length of the field in characters." + [Fact] + public void GetCharsReadsAStringWholeOrInParts() + { + Command.CommandText = "SELECT 'ab¢d'"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + + reader.GetChars(0, 0, null, 0, 0).Should().Be(4, "the length is in characters, not UTF-8 bytes"); + + var whole = new char[6]; + reader.GetChars(0, 0, whole, 0, 6).Should().Be(4); + whole.Should().Equal('a', 'b', '¢', 'd', '\0', '\0'); + + var part = new char[5]; + reader.GetChars(0, 1, part, 2, 2).Should().Be(2); + part.Should().Equal('\0', '\0', 'b', '¢', '\0'); + + reader.GetChars(0, 4, part, 0, 1).Should().Be(0); + } + + [Fact] + public void GetBytesAndGetCharsRejectOffsetsOutsideTheBuffer() + { + Command.CommandText = "SELECT '\\x01\\x02'::BLOB, 'ab'"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + + reader.Invoking(r => r.GetBytes(0, -1, new byte[2], 0, 2)).Should().Throw(); + reader.Invoking(r => r.GetBytes(0, 0, new byte[2], 3, 1)).Should().Throw(); + reader.Invoking(r => r.GetBytes(0, 0, new byte[2], 1, 2)).Should().Throw(); + reader.Invoking(r => r.GetChars(1, -1, new char[2], 0, 2)).Should().Throw(); + reader.Invoking(r => r.GetChars(1, 0, new char[2], 1, 2)).Should().Throw(); + } + // https://learn.microsoft.com/dotnet/api/system.data.idatarecord.getvalues: "You can pass an Object array that // contains fewer than the number of columns that are contained in the resulting row. Only the amount of data the // Object array holds is copied to the array. You can also pass an Object array whose length is more than the