diff --git a/DuckDB.NET.Data/DuckDBSchema.cs b/DuckDB.NET.Data/DuckDBSchema.cs index 63e0e98b..ea1a1ae6 100644 --- a/DuckDB.NET.Data/DuckDBSchema.cs +++ b/DuckDB.NET.Data/DuckDBSchema.cs @@ -1,5 +1,7 @@ using System.Linq; +using System.IO; using System.Text; +using DuckDB.NET.Native; namespace DuckDB.NET.Data; @@ -13,10 +15,14 @@ internal static class DuckDBSchema private static readonly string[] IndexesRestrictions = ["index_catalog", "index_schema", "table_name", "index_name"]; - public static DataTable GetSchema(DuckDBConnection connection, string collectionName, string?[]? restrictionValues) => - collectionName.ToUpperInvariant() switch + public static DataTable GetSchema(DuckDBConnection connection, string collectionName, string?[]? restrictionValues) + { + ValidateRestrictions(collectionName, restrictionValues); + return collectionName.ToUpperInvariant() switch { "METADATACOLLECTIONS" => GetMetaDataCollections(), + "DATASOURCEINFORMATION" => GetDataSourceInformation(connection), + "DATATYPES" => GetDataTypes(connection), "RESTRICTIONS" => GetRestrictions(), "RESERVEDWORDS" => GetReservedWords(connection), "TABLES" => GetTables(connection, restrictionValues), @@ -25,6 +31,16 @@ public static DataTable GetSchema(DuckDBConnection connection, string collection "INDEXES" => GetIndexes(connection, restrictionValues), _ => throw new ArgumentOutOfRangeException(nameof(collectionName), collectionName, "Invalid collection name.") }; + } + + private static void ValidateRestrictions(string collectionName, string?[]? restrictionValues) + { + if (restrictionValues is { Length: > 0 } + && collectionName.ToUpperInvariant() is "METADATACOLLECTIONS" or "DATASOURCEINFORMATION" or "DATATYPES" or "RESTRICTIONS" or "RESERVEDWORDS") + { + throw new ArgumentException("This collection does not support restrictions.", nameof(restrictionValues)); + } + } private static DataTable GetMetaDataCollections() => new(DbMetaDataCollectionNames.MetaDataCollections) @@ -37,9 +53,11 @@ private static DataTable GetMetaDataCollections() => }, Rows = { + { DbMetaDataCollectionNames.DataSourceInformation, 0, 0 }, + { DbMetaDataCollectionNames.DataTypes, 0, 0 }, { DbMetaDataCollectionNames.MetaDataCollections, 0, 0 }, - { DbMetaDataCollectionNames.Restrictions, 0, 0 }, { DbMetaDataCollectionNames.ReservedWords, 0, 0 }, + { DbMetaDataCollectionNames.Restrictions, 0, 0 }, { DuckDbMetaDataCollectionNames.Tables, TableRestrictions.Length, 3 }, { DuckDbMetaDataCollectionNames.Columns, ColumnRestrictions.Length, 4 }, { DuckDbMetaDataCollectionNames.ForeignKeys, ForeignKeyRestrictions.Length, 3 }, @@ -47,6 +65,215 @@ private static DataTable GetMetaDataCollections() => } }; + private static DataTable GetDataSourceInformation(DuckDBConnection connection) => + new(DbMetaDataCollectionNames.DataSourceInformation) + { + Columns = + { + { DbMetaDataColumnNames.CompositeIdentifierSeparatorPattern, typeof(string) }, + { DbMetaDataColumnNames.DataSourceProductName, typeof(string) }, + { DbMetaDataColumnNames.DataSourceProductVersion, typeof(string) }, + { DbMetaDataColumnNames.DataSourceProductVersionNormalized, typeof(string) }, + { DbMetaDataColumnNames.GroupByBehavior, typeof(int) }, + { DbMetaDataColumnNames.IdentifierCase, typeof(int) }, + { DbMetaDataColumnNames.IdentifierPattern, typeof(string) }, + { DbMetaDataColumnNames.OrderByColumnsInSelect, typeof(bool) }, + { DbMetaDataColumnNames.ParameterMarkerFormat, typeof(string) }, + { DbMetaDataColumnNames.ParameterMarkerPattern, typeof(string) }, + { DbMetaDataColumnNames.ParameterNameMaxLength, typeof(int) }, + { DbMetaDataColumnNames.ParameterNamePattern, typeof(string) }, + { DbMetaDataColumnNames.QuotedIdentifierCase, typeof(int) }, + { DbMetaDataColumnNames.QuotedIdentifierPattern, typeof(string) }, + { DbMetaDataColumnNames.StatementSeparatorPattern, typeof(string) }, + { DbMetaDataColumnNames.StringLiteralPattern, typeof(string) }, + { DbMetaDataColumnNames.SupportedJoinOperators, typeof(int) } + }, + Rows = + { + { + @"\.", "DuckDB", connection.ServerVersion, NormalizeVersion(connection.ServerVersion), + (int)GroupByBehavior.MustContainAll, + (int)IdentifierCase.Insensitive, + @"^[\p{L}_][\p{L}\p{N}_$]*$", false, + "${0}", @"(\$[\p{L}_][\p{L}\p{N}_]*)", DBNull.Value, + @"^[\p{L}_][\p{L}\p{N}_$]*$", (int)IdentifierCase.Insensitive, + "^(([^\\\"]|\\\"\\\")*)$", ";", @"'(([^']|'')*)'", (int)(SupportedJoinOperators.Inner | SupportedJoinOperators.LeftOuter + | SupportedJoinOperators.RightOuter | SupportedJoinOperators.FullOuter) + } + } + }; + + private static DataTable GetDataTypes(DuckDBConnection connection) + { + var table = new DataTable(DbMetaDataCollectionNames.DataTypes); + foreach (var column in new (string Name, Type Type)[] + { + (DbMetaDataColumnNames.TypeName, typeof(string)), (DbMetaDataColumnNames.ProviderDbType, typeof(int)), + (DbMetaDataColumnNames.DataType, typeof(string)), (DbMetaDataColumnNames.ColumnSize, typeof(long)), + (DbMetaDataColumnNames.CreateFormat, typeof(string)), (DbMetaDataColumnNames.CreateParameters, typeof(string)), + (DbMetaDataColumnNames.IsAutoIncrementable, typeof(bool)), (DbMetaDataColumnNames.IsBestMatch, typeof(bool)), + (DbMetaDataColumnNames.IsCaseSensitive, typeof(bool)), (DbMetaDataColumnNames.IsConcurrencyType, typeof(bool)), + (DbMetaDataColumnNames.IsFixedLength, typeof(bool)), (DbMetaDataColumnNames.IsFixedPrecisionScale, typeof(bool)), + (DbMetaDataColumnNames.IsLiteralSupported, typeof(bool)), (DbMetaDataColumnNames.IsLong, typeof(bool)), + (DbMetaDataColumnNames.IsNullable, typeof(bool)), (DbMetaDataColumnNames.IsSearchable, typeof(bool)), + (DbMetaDataColumnNames.IsSearchableWithLike, typeof(bool)), (DbMetaDataColumnNames.IsUnsigned, typeof(bool)), + (DbMetaDataColumnNames.LiteralPrefix, typeof(string)), (DbMetaDataColumnNames.LiteralSuffix, typeof(string)), + (DbMetaDataColumnNames.MaximumScale, typeof(short)), (DbMetaDataColumnNames.MinimumScale, typeof(short)), + ("NativeDataType", typeof(string)) + }) + { + table.Columns.Add(column.Name, column.Type); + } + + using var command = connection.CreateCommand(); + command.CommandText = "SELECT DISTINCT type_name, logical_type FROM duckdb_types() ORDER BY type_name"; + using var reader = command.ExecuteReader(); + var types = new Dictionary(StringComparer.OrdinalIgnoreCase); + while (reader.Read()) + { + var typeName = reader.GetString(0); + var logicalTypeName = reader.GetString(1); + + if (!types.TryGetValue(typeName, out var existing) + || typeName.Equals(logicalTypeName, StringComparison.Ordinal) && !existing.TypeName.Equals(existing.LogicalType, StringComparison.Ordinal)) + { + types[typeName] = (typeName, logicalTypeName); + } + } + + foreach (var (typeName, logicalTypeName) in types.Values.OrderBy(type => type.TypeName, StringComparer.OrdinalIgnoreCase)) + { + var providerType = ParseDuckDBType(logicalTypeName); + var displayTypeName = typeName.Equals(logicalTypeName, StringComparison.OrdinalIgnoreCase) ? logicalTypeName : typeName; + + AddType( + table, displayTypeName, providerType, GetClrType(providerType), + createFormat: GetCreateFormat(providerType, displayTypeName), + createParameters: providerType == DuckDBType.Decimal ? "precision,scale" : null, + isUnsigned: providerType is DuckDBType.UnsignedTinyInt or DuckDBType.UnsignedSmallInt + or DuckDBType.UnsignedInteger or DuckDBType.UnsignedBigInt or DuckDBType.UnsignedHugeInt, + isCaseSensitive: providerType == DuckDBType.Varchar, + isLong: providerType == DuckDBType.Blob, + isBestMatch: displayTypeName.Equals(logicalTypeName, StringComparison.Ordinal), + nativeTypeName: logicalTypeName); + } + + return table; + } + + private static DuckDBType ParseDuckDBType(string typeName) => typeName.ToUpperInvariant().Replace(" WITH TIME ZONE", "_TZ") switch + { + "BOOLEAN" => DuckDBType.Boolean, + "TINYINT" => DuckDBType.TinyInt, + "SMALLINT" => DuckDBType.SmallInt, + "INTEGER" => DuckDBType.Integer, + "BIGINT" => DuckDBType.BigInt, + "UTINYINT" => DuckDBType.UnsignedTinyInt, + "USMALLINT" => DuckDBType.UnsignedSmallInt, + "UINTEGER" => DuckDBType.UnsignedInteger, + "UBIGINT" => DuckDBType.UnsignedBigInt, + "FLOAT" => DuckDBType.Float, + "DOUBLE" => DuckDBType.Double, + "DECIMAL" => DuckDBType.Decimal, + "VARCHAR" => DuckDBType.Varchar, + "BLOB" => DuckDBType.Blob, + "UUID" => DuckDBType.Uuid, + "DATE" => DuckDBType.Date, + "TIME" => DuckDBType.Time, + "TIME_TZ" => DuckDBType.TimeTz, + "TIMESTAMP" => DuckDBType.Timestamp, + "TIMESTAMP_S" => DuckDBType.TimestampS, + "TIMESTAMP_MS" => DuckDBType.TimestampMs, + "TIMESTAMP_NS" => DuckDBType.TimestampNs, + "TIMESTAMP_TZ" => DuckDBType.TimestampTz, + "INTERVAL" => DuckDBType.Interval, + "HUGEINT" => DuckDBType.HugeInt, + "UHUGEINT" => DuckDBType.UnsignedHugeInt, + "VARINT" or "BIGNUM" => DuckDBType.VarInt, + "LIST" => DuckDBType.List, + "ARRAY" => DuckDBType.Array, + "STRUCT" => DuckDBType.Struct, + "MAP" => DuckDBType.Map, + "UNION" => DuckDBType.Union, + "ENUM" => DuckDBType.Enum, + "BIT" => DuckDBType.Bit, + "ANY" => DuckDBType.Any, + "SQLNULL" or "NULL" => DuckDBType.SqlNull, + "STRING_LITERAL" => DuckDBType.StringLiteral, + "INTEGER_LITERAL" => DuckDBType.IntegerLiteral, + "TIME_NS" => DuckDBType.TimeNs, + "GEOMETRY" => DuckDBType.Geometry, + "VARIANT" => DuckDBType.Variant, + _ => DuckDBType.Invalid + }; + + private static string? GetCreateFormat(DuckDBType type, string typeName) => type switch + { + DuckDBType.Decimal => "DECIMAL({0},{1})", + DuckDBType.List or DuckDBType.Array or DuckDBType.Struct or DuckDBType.Map or DuckDBType.Union + or DuckDBType.Enum or DuckDBType.SqlNull or DuckDBType.Invalid or DuckDBType.Any => null, + _ => typeName + }; + + private static Type GetClrType(DuckDBType type) => type switch + { + DuckDBType.Boolean => typeof(bool), + DuckDBType.TinyInt => typeof(sbyte), + DuckDBType.SmallInt => typeof(short), + DuckDBType.Integer => typeof(int), + DuckDBType.BigInt => typeof(long), + DuckDBType.UnsignedTinyInt => typeof(byte), + DuckDBType.UnsignedSmallInt => typeof(ushort), + DuckDBType.UnsignedInteger => typeof(uint), + DuckDBType.UnsignedBigInt => typeof(ulong), + DuckDBType.Float => typeof(float), + DuckDBType.Double => typeof(double), + DuckDBType.Decimal => typeof(decimal), + DuckDBType.Varchar => typeof(string), + DuckDBType.Blob => typeof(Stream), + DuckDBType.Uuid => typeof(Guid), + DuckDBType.Date => typeof(DateOnly), + DuckDBType.Time => typeof(TimeOnly), + DuckDBType.TimeTz => typeof(DateTimeOffset), + DuckDBType.TimestampTz => typeof(DateTime), + DuckDBType.Timestamp or DuckDBType.TimestampS or DuckDBType.TimestampMs or DuckDBType.TimestampNs => typeof(DateTime), + DuckDBType.Interval => typeof(TimeSpan), + DuckDBType.HugeInt or DuckDBType.UnsignedHugeInt or DuckDBType.VarInt => typeof(System.Numerics.BigInteger), + DuckDBType.Enum or DuckDBType.Bit => typeof(string), + DuckDBType.SqlNull => typeof(DBNull), + _ => typeof(object) + }; + + private static void AddType( + DataTable table, + string typeName, + DuckDBType providerType, + Type dataType, + long columnSize = -1, + string? createFormat = null, + string? createParameters = null, + bool isUnsigned = false, + bool isCaseSensitive = false, + bool isLong = false, + bool isBestMatch = false, + string? nativeTypeName = null) + { + table.Rows.Add( + typeName, (int)providerType, dataType.FullName!, columnSize, + createFormat ?? (object)DBNull.Value, createParameters ?? (object)DBNull.Value, + false, isBestMatch, isCaseSensitive, false, false, false, true, isLong, + true, true, isCaseSensitive, isUnsigned ? (object)true : DBNull.Value, + DBNull.Value, DBNull.Value, DBNull.Value, DBNull.Value, nativeTypeName ?? typeName); + } + + private static string NormalizeVersion(string serverVersion) + { + var value = serverVersion.TrimStart('v', 'V'); + return Version.TryParse(value, out var version) + ? $"{version.Major:00}.{version.Minor:000}.{(version.Build < 0 ? 0 : version.Build):0000}.{(version.Revision < 0 ? 0 : version.Revision):0000}" + : serverVersion; + } + private static DataTable GetRestrictions() => new(DbMetaDataCollectionNames.Restrictions) { @@ -74,10 +301,10 @@ private static DataTable GetRestrictions() => { DuckDbMetaDataCollectionNames.ForeignKeys, "Table", "table_name", 3 }, { DuckDbMetaDataCollectionNames.ForeignKeys, "Constraint", "constraint_name", 4 }, - { DuckDbMetaDataCollectionNames.Indexes, "Catalog", "constraint_catalog", 1 }, - { DuckDbMetaDataCollectionNames.Indexes, "Schema", "constraint_schema", 2 }, + { DuckDbMetaDataCollectionNames.Indexes, "Catalog", "index_catalog", 1 }, + { DuckDbMetaDataCollectionNames.Indexes, "Schema", "index_schema", 2 }, { DuckDbMetaDataCollectionNames.Indexes, "Table", "table_name", 3 }, - { DuckDbMetaDataCollectionNames.Indexes, "Constraint", "constraint_name", 4 }, + { DuckDbMetaDataCollectionNames.Indexes, "Index", "index_name", 4 }, }, }; @@ -218,4 +445,4 @@ private static DataTable GetDataTable(string tableName, DuckDBCommand command) return table; } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Test/SchemaTests.cs b/DuckDB.NET.Test/SchemaTests.cs index f9c2a4f4..24950fb1 100644 --- a/DuckDB.NET.Test/SchemaTests.cs +++ b/DuckDB.NET.Test/SchemaTests.cs @@ -20,6 +20,17 @@ public void InvalidCollection() { Assert.Throws(() => Connection.GetSchema("Invalid")); } + + [Theory] + [InlineData("MetaDataCollections")] + [InlineData("DataSourceInformation")] + [InlineData("DataTypes")] + [InlineData("Restrictions")] + [InlineData("ReservedWords")] + public void CollectionsWithoutRestrictionsRejectRestrictionValues(string collectionName) + { + Assert.Throws(() => Connection.GetSchema(collectionName, ["unexpected"])); + } [Fact] public void MetaDataCollections() @@ -119,6 +130,79 @@ public void ColumnsWithRestrictions() Assert.Equal("foo_id", schema.Rows[0]["column_name"]); } + [Fact] + public void ColumnsExposeDataType() + { + var schema = Connection.GetSchema("Columns", [null, null, "foo", "foo_id"]); + Assert.Equal("INTEGER", schema.Rows[0]["data_type"]); + } + + [Fact] + public void DataSourceInformationSupportsProbing() + { + var schema = Connection.GetSchema(DbMetaDataCollectionNames.DataSourceInformation); + var row = Assert.IsType(Assert.Single(schema.Rows)); + Assert.Equal("DuckDB", row["DataSourceProductName"]); + Assert.Equal(Connection.ServerVersion, row["DataSourceProductVersion"]); + Assert.Equal("${0}", row["ParameterMarkerFormat"]); + } + + [Fact] + public void DataTypesExposeStandardAdoNetMetadata() + { + var schema = Connection.GetSchema("DataTypes"); + var integer = schema.Rows.Cast().Single(row => (string)row["TypeName"] == "INTEGER"); + + Assert.Equal(typeof(int).FullName, integer["DataType"]); + Assert.Equal((int)DuckDBType.Integer, integer["ProviderDbType"]); + Assert.Equal("INTEGER", integer["CreateFormat"]); + Assert.True((bool)integer["IsBestMatch"]); + Assert.True((bool)integer["IsSearchable"]); + + var blob = schema.Rows.Cast().First(row => (int)row["ProviderDbType"] == (int)DuckDBType.Blob); + Assert.Equal(typeof(Stream).FullName, blob["DataType"]); + + var timestampTz = schema.Rows.Cast().First(row => (int)row["ProviderDbType"] == (int)DuckDBType.TimestampTz); + Assert.Equal(typeof(DateTime).FullName, timestampTz["DataType"]); + + var bignum = schema.Rows.Cast().First(row => (int)row["ProviderDbType"] == (int)DuckDBType.VarInt); + Assert.Equal(typeof(System.Numerics.BigInteger).FullName, bignum["DataType"]); + + Assert.DoesNotContain( + schema.Rows.Cast().GroupBy(row => (string)row["TypeName"], StringComparer.OrdinalIgnoreCase), + group => group.Count() > 1); + } + + [Fact] + public void DataTypesUseCanonicalNamesOnFreshConnection() + { + using var connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + var schema = connection.GetSchema("DataTypes"); + var integer = schema.Rows.Cast().Single(row => (string)row["TypeName"] == "INTEGER"); + var list = schema.Rows.Cast().Single(row => (string)row["TypeName"] == "LIST"); + + Assert.Equal("INTEGER", integer["CreateFormat"]); + Assert.Equal(DBNull.Value, list["CreateFormat"]); + } + + [Fact] + public void DataTypesExposeEveryDuckDbType() + { + var schema = Connection.GetSchema("DataTypes"); + var providerTypes = schema.Rows.Cast() + .Select(row => (DuckDBType)(int)row["ProviderDbType"]) + .ToHashSet(); + + var expectedTypes = Enum.GetValues() + .Where(type => type is not (DuckDBType.Invalid or DuckDBType.Any or DuckDBType.StringLiteral or DuckDBType.IntegerLiteral)) + .ToHashSet(); + + Assert.True(expectedTypes.IsSubsetOf(providerTypes), + $"Missing: {string.Join(", ", expectedTypes.Except(providerTypes))}"); + } + [Fact] public void ColumnsTooManyRestrictions() { @@ -173,4 +257,4 @@ public void IndexesTooManyRestrictions() private static IEnumerable GetValues(DataTable schema, string columnName) => schema.Rows.Cast().Select(r => (string)r[columnName]); -} \ No newline at end of file +}