Support ADO.NET schema data types and metadata probing

DuckDB.NET.Data/DuckDBSchema.cs

@@ -1,5 +1,7 @@
 using System.Linq;
+using System.IO;
 using System.Text;
+using DuckDB.NET.Native;
 
 namespace DuckDB.NET.Data;
 
@@ -13,10 +15,14 @@ internal static class DuckDBSchema
 
     private static readonly string[] IndexesRestrictions = ["index_catalog", "index_schema", "table_name", "index_name"];
 
-    public static DataTable GetSchema(DuckDBConnection connection, string collectionName, string?[]? restrictionValues) =>
-        collectionName.ToUpperInvariant() switch
+    public static DataTable GetSchema(DuckDBConnection connection, string collectionName, string?[]? restrictionValues)
+    {
+        ValidateRestrictions(collectionName, restrictionValues);
+        return collectionName.ToUpperInvariant() switch
         {
             "METADATACOLLECTIONS" => GetMetaDataCollections(),
+            "DATASOURCEINFORMATION" => GetDataSourceInformation(connection),
+            "DATATYPES" => GetDataTypes(connection),
             "RESTRICTIONS" => GetRestrictions(),
             "RESERVEDWORDS" => GetReservedWords(connection),
             "TABLES" => GetTables(connection, restrictionValues),
@@ -25,6 +31,16 @@ public static DataTable GetSchema(DuckDBConnection connection, string collection
             "INDEXES" => GetIndexes(connection, restrictionValues),
             _ => throw new ArgumentOutOfRangeException(nameof(collectionName), collectionName, "Invalid collection name.")
         };
+    }
+
+    private static void ValidateRestrictions(string collectionName, string?[]? restrictionValues)
+    {
+        if (restrictionValues is { Length: > 0 }
+            && collectionName.ToUpperInvariant() is "METADATACOLLECTIONS" or "DATASOURCEINFORMATION" or "DATATYPES" or "RESTRICTIONS" or "RESERVEDWORDS")
+        {
+            throw new ArgumentException("This collection does not support restrictions.", nameof(restrictionValues));
+        }
+    }
 
     private static DataTable GetMetaDataCollections() =>
         new(DbMetaDataCollectionNames.MetaDataCollections)
@@ -37,16 +53,227 @@ private static DataTable GetMetaDataCollections() =>
             },
             Rows =
             {
+                { DbMetaDataCollectionNames.DataSourceInformation, 0, 0 },
+                { DbMetaDataCollectionNames.DataTypes, 0, 0 },
                 { DbMetaDataCollectionNames.MetaDataCollections, 0, 0 },
-                { DbMetaDataCollectionNames.Restrictions, 0, 0 },
                 { DbMetaDataCollectionNames.ReservedWords, 0, 0 },
+                { DbMetaDataCollectionNames.Restrictions, 0, 0 },
                 { DuckDbMetaDataCollectionNames.Tables, TableRestrictions.Length, 3 },
                 { DuckDbMetaDataCollectionNames.Columns, ColumnRestrictions.Length, 4 },
                 { DuckDbMetaDataCollectionNames.ForeignKeys, ForeignKeyRestrictions.Length, 3 },
                 { DuckDbMetaDataCollectionNames.Indexes, IndexesRestrictions.Length, 3 },
             }
         };
 
+    private static DataTable GetDataSourceInformation(DuckDBConnection connection) =>
+        new(DbMetaDataCollectionNames.DataSourceInformation)
+        {
+            Columns =
+            {
+                { DbMetaDataColumnNames.CompositeIdentifierSeparatorPattern, typeof(string) },
+                { DbMetaDataColumnNames.DataSourceProductName, typeof(string) },
+                { DbMetaDataColumnNames.DataSourceProductVersion, typeof(string) },
+                { DbMetaDataColumnNames.DataSourceProductVersionNormalized, typeof(string) },
+                { DbMetaDataColumnNames.GroupByBehavior, typeof(int) },
+                { DbMetaDataColumnNames.IdentifierCase, typeof(int) },
+                { DbMetaDataColumnNames.IdentifierPattern, typeof(string) },
+                { DbMetaDataColumnNames.OrderByColumnsInSelect, typeof(bool) },
+                { DbMetaDataColumnNames.ParameterMarkerFormat, typeof(string) },
+                { DbMetaDataColumnNames.ParameterMarkerPattern, typeof(string) },
+                { DbMetaDataColumnNames.ParameterNameMaxLength, typeof(int) },
+                { DbMetaDataColumnNames.ParameterNamePattern, typeof(string) },
+                { DbMetaDataColumnNames.QuotedIdentifierCase, typeof(int) },
+                { DbMetaDataColumnNames.QuotedIdentifierPattern, typeof(string) },
+                { DbMetaDataColumnNames.StatementSeparatorPattern, typeof(string) },
+                { DbMetaDataColumnNames.StringLiteralPattern, typeof(string) },
+                { DbMetaDataColumnNames.SupportedJoinOperators, typeof(int) }
+            },
+            Rows =
+            {
+                {
+                    @"\.", "DuckDB", connection.ServerVersion, NormalizeVersion(connection.ServerVersion),
+                    (int)GroupByBehavior.MustContainAll,
+                    (int)IdentifierCase.Insensitive,
+                    @"^[\p{L}_][\p{L}\p{N}_$]*$", false,
+                    "${0}", @"(\$[\p{L}_][\p{L}\p{N}_]*)", DBNull.Value,
+                    @"^[\p{L}_][\p{L}\p{N}_$]*$", (int)IdentifierCase.Insensitive,
+                    "^(([^\\\"]|\\\"\\\")*)$", ";", @"'(([^']|'')*)'", (int)(SupportedJoinOperators.Inner | SupportedJoinOperators.LeftOuter
+                        | SupportedJoinOperators.RightOuter | SupportedJoinOperators.FullOuter)
+                }
+            }
+        };
+
+    private static DataTable GetDataTypes(DuckDBConnection connection)
+    {
+        var table = new DataTable(DbMetaDataCollectionNames.DataTypes);
+        foreach (var column in new (string Name, Type Type)[]
+        {
+            (DbMetaDataColumnNames.TypeName, typeof(string)), (DbMetaDataColumnNames.ProviderDbType, typeof(int)),
+            (DbMetaDataColumnNames.DataType, typeof(string)), (DbMetaDataColumnNames.ColumnSize, typeof(long)),
+            (DbMetaDataColumnNames.CreateFormat, typeof(string)), (DbMetaDataColumnNames.CreateParameters, typeof(string)),
+            (DbMetaDataColumnNames.IsAutoIncrementable, typeof(bool)), (DbMetaDataColumnNames.IsBestMatch, typeof(bool)),
+            (DbMetaDataColumnNames.IsCaseSensitive, typeof(bool)), (DbMetaDataColumnNames.IsConcurrencyType, typeof(bool)),
+            (DbMetaDataColumnNames.IsFixedLength, typeof(bool)), (DbMetaDataColumnNames.IsFixedPrecisionScale, typeof(bool)),
+            (DbMetaDataColumnNames.IsLiteralSupported, typeof(bool)), (DbMetaDataColumnNames.IsLong, typeof(bool)),
+            (DbMetaDataColumnNames.IsNullable, typeof(bool)), (DbMetaDataColumnNames.IsSearchable, typeof(bool)),
+            (DbMetaDataColumnNames.IsSearchableWithLike, typeof(bool)), (DbMetaDataColumnNames.IsUnsigned, typeof(bool)),
+            (DbMetaDataColumnNames.LiteralPrefix, typeof(string)), (DbMetaDataColumnNames.LiteralSuffix, typeof(string)),
+            (DbMetaDataColumnNames.MaximumScale, typeof(short)), (DbMetaDataColumnNames.MinimumScale, typeof(short)),
+            ("NativeDataType", typeof(string))
+        })
+        {
+            table.Columns.Add(column.Name, column.Type);
+        }
+
+        using var command = connection.CreateCommand();
+        command.CommandText = "SELECT DISTINCT type_name, logical_type FROM duckdb_types() ORDER BY type_name";
+        using var reader = command.ExecuteReader();
+        var types = new Dictionary<string, (string TypeName, string LogicalType)>(StringComparer.OrdinalIgnoreCase);
+        while (reader.Read())
+        {
+            var typeName = reader.GetString(0);
+            var logicalTypeName = reader.GetString(1);
+
+            if (!types.TryGetValue(typeName, out var existing)
+                || typeName.Equals(logicalTypeName, StringComparison.Ordinal) && !existing.TypeName.Equals(existing.LogicalType, StringComparison.Ordinal))
+            {
+                types[typeName] = (typeName, logicalTypeName);
+            }
+        }
+
+        foreach (var (typeName, logicalTypeName) in types.Values.OrderBy(type => type.TypeName, StringComparer.OrdinalIgnoreCase))
+        {
+            var providerType = ParseDuckDBType(logicalTypeName);
+            var displayTypeName = typeName.Equals(logicalTypeName, StringComparison.OrdinalIgnoreCase) ? logicalTypeName : typeName;
+
+            AddType(
+                table, displayTypeName, providerType, GetClrType(providerType),
+                createFormat: GetCreateFormat(providerType, displayTypeName),
+                createParameters: providerType == DuckDBType.Decimal ? "precision,scale" : null,
+                isUnsigned: providerType is DuckDBType.UnsignedTinyInt or DuckDBType.UnsignedSmallInt
+                    or DuckDBType.UnsignedInteger or DuckDBType.UnsignedBigInt or DuckDBType.UnsignedHugeInt,
+                isCaseSensitive: providerType == DuckDBType.Varchar,
+                isLong: providerType == DuckDBType.Blob,
+                isBestMatch: displayTypeName.Equals(logicalTypeName, StringComparison.Ordinal),
+                nativeTypeName: logicalTypeName);
+        }
+
+        return table;
+    }
+
+    private static DuckDBType ParseDuckDBType(string typeName) => typeName.ToUpperInvariant().Replace(" WITH TIME ZONE", "_TZ") switch
+    {
+        "BOOLEAN" => DuckDBType.Boolean,
+        "TINYINT" => DuckDBType.TinyInt,
+        "SMALLINT" => DuckDBType.SmallInt,
+        "INTEGER" => DuckDBType.Integer,
+        "BIGINT" => DuckDBType.BigInt,
+        "UTINYINT" => DuckDBType.UnsignedTinyInt,
+        "USMALLINT" => DuckDBType.UnsignedSmallInt,
+        "UINTEGER" => DuckDBType.UnsignedInteger,
+        "UBIGINT" => DuckDBType.UnsignedBigInt,
+        "FLOAT" => DuckDBType.Float,
+        "DOUBLE" => DuckDBType.Double,
+        "DECIMAL" => DuckDBType.Decimal,
+        "VARCHAR" => DuckDBType.Varchar,
+        "BLOB" => DuckDBType.Blob,
+        "UUID" => DuckDBType.Uuid,
+        "DATE" => DuckDBType.Date,
+        "TIME" => DuckDBType.Time,
+        "TIME_TZ" => DuckDBType.TimeTz,
+        "TIMESTAMP" => DuckDBType.Timestamp,
+        "TIMESTAMP_S" => DuckDBType.TimestampS,
+        "TIMESTAMP_MS" => DuckDBType.TimestampMs,
+        "TIMESTAMP_NS" => DuckDBType.TimestampNs,
+        "TIMESTAMP_TZ" => DuckDBType.TimestampTz,
+        "INTERVAL" => DuckDBType.Interval,
+        "HUGEINT" => DuckDBType.HugeInt,
+        "UHUGEINT" => DuckDBType.UnsignedHugeInt,
+        "VARINT" or "BIGNUM" => DuckDBType.VarInt,
+        "LIST" => DuckDBType.List,
+        "ARRAY" => DuckDBType.Array,
+        "STRUCT" => DuckDBType.Struct,
+        "MAP" => DuckDBType.Map,
+        "UNION" => DuckDBType.Union,
+        "ENUM" => DuckDBType.Enum,
+        "BIT" => DuckDBType.Bit,
+        "ANY" => DuckDBType.Any,
+        "SQLNULL" or "NULL" => DuckDBType.SqlNull,
+        "STRING_LITERAL" => DuckDBType.StringLiteral,
+        "INTEGER_LITERAL" => DuckDBType.IntegerLiteral,
+        "TIME_NS" => DuckDBType.TimeNs,
+        "GEOMETRY" => DuckDBType.Geometry,
+        "VARIANT" => DuckDBType.Variant,
+        _ => DuckDBType.Invalid
+    };
+
+    private static string? GetCreateFormat(DuckDBType type, string typeName) => type switch
+    {
+        DuckDBType.Decimal => "DECIMAL({0},{1})",
+        DuckDBType.List or DuckDBType.Array or DuckDBType.Struct or DuckDBType.Map or DuckDBType.Union
+            or DuckDBType.Enum or DuckDBType.SqlNull or DuckDBType.Invalid or DuckDBType.Any => null,
+        _ => typeName
+    };
+
+    private static Type GetClrType(DuckDBType type) => type switch
+    {
+        DuckDBType.Boolean => typeof(bool),
+        DuckDBType.TinyInt => typeof(sbyte),
+        DuckDBType.SmallInt => typeof(short),
+        DuckDBType.Integer => typeof(int),
+        DuckDBType.BigInt => typeof(long),
+        DuckDBType.UnsignedTinyInt => typeof(byte),
+        DuckDBType.UnsignedSmallInt => typeof(ushort),
+        DuckDBType.UnsignedInteger => typeof(uint),
+        DuckDBType.UnsignedBigInt => typeof(ulong),
+        DuckDBType.Float => typeof(float),
+        DuckDBType.Double => typeof(double),
+        DuckDBType.Decimal => typeof(decimal),
+        DuckDBType.Varchar => typeof(string),
+        DuckDBType.Blob => typeof(Stream),
+        DuckDBType.Uuid => typeof(Guid),
+        DuckDBType.Date => typeof(DateOnly),
+        DuckDBType.Time => typeof(TimeOnly),
+        DuckDBType.TimeTz => typeof(DateTimeOffset),
+        DuckDBType.TimestampTz => typeof(DateTime),
+        DuckDBType.Timestamp or DuckDBType.TimestampS or DuckDBType.TimestampMs or DuckDBType.TimestampNs => typeof(DateTime),
+        DuckDBType.Interval => typeof(TimeSpan),
+        DuckDBType.HugeInt or DuckDBType.UnsignedHugeInt or DuckDBType.VarInt => typeof(System.Numerics.BigInteger),
+        DuckDBType.Enum or DuckDBType.Bit => typeof(string),
+        DuckDBType.SqlNull => typeof(DBNull),
+        _ => typeof(object)
+    };
+
+    private static void AddType(
+        DataTable table,
+        string typeName,
+        DuckDBType providerType,
+        Type dataType,
+        long columnSize = -1,
+        string? createFormat = null,
+        string? createParameters = null,
+        bool isUnsigned = false,
+        bool isCaseSensitive = false,
+        bool isLong = false,
+        bool isBestMatch = false,
+        string? nativeTypeName = null)
+    {
+        table.Rows.Add(
+            typeName, (int)providerType, dataType.FullName!, columnSize,
+            createFormat ?? (object)DBNull.Value, createParameters ?? (object)DBNull.Value,
+            false, isBestMatch, isCaseSensitive, false, false, false, true, isLong,
+            true, true, isCaseSensitive, isUnsigned ? (object)true : DBNull.Value,
+            DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, nativeTypeName ?? typeName);
+    }
+
+    private static string NormalizeVersion(string serverVersion)
+    {
+        var value = serverVersion.TrimStart('v', 'V');
+        return Version.TryParse(value, out var version)
+            ? $"{version.Major:00}.{version.Minor:000}.{(version.Build < 0 ? 0 : version.Build):0000}.{(version.Revision < 0 ? 0 : version.Revision):0000}"
+            : serverVersion;
+    }
+
     private static DataTable GetRestrictions() =>
         new(DbMetaDataCollectionNames.Restrictions)
         {
@@ -74,10 +301,10 @@ private static DataTable GetRestrictions() =>
                 { DuckDbMetaDataCollectionNames.ForeignKeys, "Table", "table_name", 3 },
                 { DuckDbMetaDataCollectionNames.ForeignKeys, "Constraint", "constraint_name", 4 },
 
-                { DuckDbMetaDataCollectionNames.Indexes, "Catalog", "constraint_catalog", 1 },
-                { DuckDbMetaDataCollectionNames.Indexes, "Schema", "constraint_schema", 2 },
+                { DuckDbMetaDataCollectionNames.Indexes, "Catalog", "index_catalog", 1 },
+                { DuckDbMetaDataCollectionNames.Indexes, "Schema", "index_schema", 2 },
                 { DuckDbMetaDataCollectionNames.Indexes, "Table", "table_name", 3 },
-                { DuckDbMetaDataCollectionNames.Indexes, "Constraint", "constraint_name", 4 },
+                { DuckDbMetaDataCollectionNames.Indexes, "Index", "index_name", 4 },
             },
         };
 
@@ -218,4 +445,4 @@ private static DataTable GetDataTable(string tableName, DuckDBCommand command)
 
         return table;
     }
-}
+}

DuckDB.NET.Test/SchemaTests.cs

@@ -20,6 +20,17 @@ public void InvalidCollection()
     {
         Assert.Throws<ArgumentOutOfRangeException>(() => Connection.GetSchema("Invalid"));
     }
+
+    [Theory]
+    [InlineData("MetaDataCollections")]
+    [InlineData("DataSourceInformation")]
+    [InlineData("DataTypes")]
+    [InlineData("Restrictions")]
+    [InlineData("ReservedWords")]
+    public void CollectionsWithoutRestrictionsRejectRestrictionValues(string collectionName)
+    {
+        Assert.Throws<ArgumentException>(() => Connection.GetSchema(collectionName, ["unexpected"]));
+    }
 
     [Fact]
     public void MetaDataCollections()
@@ -119,6 +130,79 @@ public void ColumnsWithRestrictions()
         Assert.Equal("foo_id", schema.Rows[0]["column_name"]);
     }
 
+    [Fact]
+    public void ColumnsExposeDataType()
+    {
+        var schema = Connection.GetSchema("Columns", [null, null, "foo", "foo_id"]);
+        Assert.Equal("INTEGER", schema.Rows[0]["data_type"]);
+    }
+
+    [Fact]
+    public void DataSourceInformationSupportsProbing()
+    {
+        var schema = Connection.GetSchema(DbMetaDataCollectionNames.DataSourceInformation);
+        var row = Assert.IsType<DataRow>(Assert.Single(schema.Rows));
+        Assert.Equal("DuckDB", row["DataSourceProductName"]);
+        Assert.Equal(Connection.ServerVersion, row["DataSourceProductVersion"]);
+        Assert.Equal("${0}", row["ParameterMarkerFormat"]);
+    }
+
+    [Fact]
+    public void DataTypesExposeStandardAdoNetMetadata()
+    {
+        var schema = Connection.GetSchema("DataTypes");
+        var integer = schema.Rows.Cast<DataRow>().Single(row => (string)row["TypeName"] == "INTEGER");
+
+        Assert.Equal(typeof(int).FullName, integer["DataType"]);
+        Assert.Equal((int)DuckDBType.Integer, integer["ProviderDbType"]);
+        Assert.Equal("INTEGER", integer["CreateFormat"]);
+        Assert.True((bool)integer["IsBestMatch"]);
+        Assert.True((bool)integer["IsSearchable"]);
+
+        var blob = schema.Rows.Cast<DataRow>().First(row => (int)row["ProviderDbType"] == (int)DuckDBType.Blob);
+        Assert.Equal(typeof(Stream).FullName, blob["DataType"]);
+
+        var timestampTz = schema.Rows.Cast<DataRow>().First(row => (int)row["ProviderDbType"] == (int)DuckDBType.TimestampTz);
+        Assert.Equal(typeof(DateTime).FullName, timestampTz["DataType"]);
+
+        var bignum = schema.Rows.Cast<DataRow>().First(row => (int)row["ProviderDbType"] == (int)DuckDBType.VarInt);
+        Assert.Equal(typeof(System.Numerics.BigInteger).FullName, bignum["DataType"]);
+
+        Assert.DoesNotContain(
+            schema.Rows.Cast<DataRow>().GroupBy(row => (string)row["TypeName"], StringComparer.OrdinalIgnoreCase),
+            group => group.Count() > 1);
+    }
+
+    [Fact]
+    public void DataTypesUseCanonicalNamesOnFreshConnection()
+    {
+        using var connection = new DuckDBConnection("DataSource=:memory:");
+        connection.Open();
+
+        var schema = connection.GetSchema("DataTypes");
+        var integer = schema.Rows.Cast<DataRow>().Single(row => (string)row["TypeName"] == "INTEGER");
+        var list = schema.Rows.Cast<DataRow>().Single(row => (string)row["TypeName"] == "LIST");
+
+        Assert.Equal("INTEGER", integer["CreateFormat"]);
+        Assert.Equal(DBNull.Value, list["CreateFormat"]);
+    }
+
+    [Fact]
+    public void DataTypesExposeEveryDuckDbType()
+    {
+        var schema = Connection.GetSchema("DataTypes");
+        var providerTypes = schema.Rows.Cast<DataRow>()
+            .Select(row => (DuckDBType)(int)row["ProviderDbType"])
+            .ToHashSet();
+
+        var expectedTypes = Enum.GetValues<DuckDBType>()
+            .Where(type => type is not (DuckDBType.Invalid or DuckDBType.Any or DuckDBType.StringLiteral or DuckDBType.IntegerLiteral))
+            .ToHashSet();
+
+        Assert.True(expectedTypes.IsSubsetOf(providerTypes),
+            $"Missing: {string.Join(", ", expectedTypes.Except(providerTypes))}");
+    }
+
     [Fact]
     public void ColumnsTooManyRestrictions()
     {
@@ -173,4 +257,4 @@ public void IndexesTooManyRestrictions()
 
     private static IEnumerable<string> GetValues(DataTable schema, string columnName) =>
         schema.Rows.Cast<DataRow>().Select(r => (string)r[columnName]);
-}
+}