diff --git a/README.md b/README.md index 744880c..0369b4a 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,15 @@ number) and booleans written as `1`/`0`. Sorts accept any property type and dott (`Customer.Address.City`). In `SimplyWorks.EfCoreExtensions.PgSql`, `StartsWith` and `Contains` are case-insensitive `ILIKE`, with `%`, `_` and `\` in the value matched literally. +#### Query shaping +>`RowProjection.Of(columns)` / `RowProjection.Many(...)`: project to `object[]` rows of the chosen columns, nested collections included, so a query reads only what the caller asked for. + +>`AfterKeyset` / `OrderByKeyset`: keyset ("seek") paging; a deep page costs the same as the first. + +>`ToPageAsync(size)`: a page and whether more rows follow, without a count. `CountUpToAsync(limit)`: a count that stops at a limit ("10,000+"). + +>`QueryExpressions.Parameter(value, type)` / `QueryExpressions.TextMatch(field, text, kind)`: for code that builds its own filters, a value EF Core sends as a parameter, and the package's text match (escaped `ILIKE` in PgSql). + ## Getting support 👷 If you encounter any bugs, don't hesitate to submit an [issue](https://github.com/simplify9/EfCoreExtensions/issues). We'll get back to you promptly! diff --git a/SW.EfCoreExtensions.PgSql.IntegrationTests/QueryShapingTests.cs b/SW.EfCoreExtensions.PgSql.IntegrationTests/QueryShapingTests.cs new file mode 100644 index 0000000..f92ca04 --- /dev/null +++ b/SW.EfCoreExtensions.PgSql.IntegrationTests/QueryShapingTests.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using static SW.EfCoreExtensions.PgSql.IntegrationTests.OrdersDatabase; + +namespace SW.EfCoreExtensions.PgSql.IntegrationTests; + +[TestClass] +public class QueryShapingTests +{ + static LambdaExpression Column(Expression> column) => column; + + [TestMethod] + public async Task A_row_reads_only_the_chosen_columns() + { + ClearCommands(); + await using var db = NewContext(); + var rows = await db.Orders.OrderBy(o => o.Id).Take(3) + .Select(RowProjection.Of(Column((Order o) => o.Number), Column((Order o) => o.Customer.Name))) + .ToListAsync(); + + Assert.AreEqual(All[0].Number, rows[0][0]); + Assert.AreEqual(All[0].Customer.Name, rows[0][1]); + + var sql = Commands.Last(); + var select = sql[sql.IndexOf("SELECT", StringComparison.Ordinal)..sql.IndexOf("FROM", StringComparison.Ordinal)]; + StringAssert.Contains(select, "\"Number\""); + StringAssert.Contains(select, "\"Name\""); + foreach (var unselected in new[] { "\"Total\"", "\"Discount\"", "\"ExternalId\"", "\"PlacedOn\"", "\"Country\"" }) + Assert.IsFalse(select.Contains(unselected), $"{unselected} was read:\n{sql}"); + } + + [TestMethod] + public async Task A_collection_column_reads_its_own_chosen_columns_in_order_and_capped() + { + await using var db = NewContext(); + var row = RowProjection.Of( + Column((Order o) => o.Number), + RowProjection.Many(o => o.Lines, take: 1, + orderBy: Column((OrderLine l) => l.Quantity), descending: true, + Column((OrderLine l) => l.Sku), Column((OrderLine l) => l.Quantity))); + + ClearCommands(); + var rows = await db.Orders.Where(o => o.Id <= 3).OrderBy(o => o.Id).Select(row).ToListAsync(); + + var first = (List)rows[0][1]; + Assert.AreEqual(1, first.Count); + Assert.AreEqual(All[0].Lines.Single().Sku, first[0][0]); + Assert.IsFalse(Commands.Last().Contains("\"Total\""), Commands.Last()); + } + + [TestMethod] + public async Task The_same_columns_compile_once() + { + await using (var warm = NewContext()) + await warm.Orders.Where(o => o.Paid).Select(RowProjection.Of(Column((Order o) => o.Number))).ToListAsync(); + var before = Compilations; + + for (var i = 0; i < 5; i++) + { + await using var db = NewContext(); + // A new expression tree each time, as a request builds it. + await db.Orders.Where(o => o.Paid).Select(RowProjection.Of(Column((Order o) => o.Number))).ToListAsync(); + } + + Assert.AreEqual(before, Compilations); + } + + public static IEnumerable Keysets => new[] + { + // Unique key alone. + new object[] { new[] { KeysetKey.Asc((Order o) => o.Id) }, 4 }, + // Many orders share a status: the id breaks the ties. + new object[] { new[] { KeysetKey.Asc((Order o) => o.Status), KeysetKey.Asc((Order o) => o.Id) }, 4 }, + new object[] { new[] { KeysetKey.Desc((Order o) => o.Total), KeysetKey.Desc((Order o) => o.Id) }, 7 }, + new object[] { new[] { KeysetKey.Asc((Order o) => o.Customer.Name), KeysetKey.Desc((Order o) => o.PlacedOn), KeysetKey.Asc((Order o) => o.Id) }, 5 }, + }; + + [TestMethod] + [DynamicData(nameof(Keysets))] + public async Task Keyset_pages_visit_every_row_once_in_order(KeysetKey[] keys, int pageSize) + { + await using var db = NewContext(); + var expected = await db.Orders.OrderByKeyset(keys).Select(o => o.Id).ToListAsync(); + + var visited = new List(); + object[] after = null; + var compiled = 0; + for (var page = 0; page < 100; page++) + { + var before = Compilations; + var (items, hasMore) = await db.Orders.Include(o => o.Customer).AfterKeyset(keys, after).OrderByKeyset(keys) + .AsNoTracking().ToPageAsync(pageSize); + if (page > 1) compiled += Compilations - before; + + visited.AddRange(items.Select(o => o.Id)); + if (!hasMore) break; + var last = items.Last(); + after = keys.Select(k => k.Key.Compile().DynamicInvoke(last)).ToArray(); + } + + CollectionAssert.AreEqual(expected, visited); + Assert.AreEqual(0, compiled, "a new position must not compile a new query"); + } + + [TestMethod] + public void A_nullable_key_is_refused() + { + Assert.ThrowsException(() => + NewContext().Orders.OrderByKeyset(new[] { KeysetKey.Asc((Order o) => o.Discount) })); + } + + [TestMethod] + public async Task A_page_says_whether_more_rows_follow() + { + await using var db = NewContext(); + var (items, hasMore) = await db.Orders.OrderBy(o => o.Id).ToPageAsync(All.Count - 1); + Assert.AreEqual(All.Count - 1, items.Count); + Assert.IsTrue(hasMore); + + (items, hasMore) = await db.Orders.OrderBy(o => o.Id).ToPageAsync(All.Count); + Assert.AreEqual(All.Count, items.Count); + Assert.IsFalse(hasMore); + } + + [TestMethod] + public async Task A_count_stops_at_its_limit() + { + await using var db = NewContext(); + Assert.AreEqual(new CountUpTo(10, true), await db.Orders.CountUpToAsync(10)); + Assert.AreEqual(new CountUpTo(All.Count, false), await db.Orders.CountUpToAsync(All.Count)); + Assert.AreEqual(new CountUpTo(0, false), await db.Orders.Where(o => o.Id < 0).CountUpToAsync(10)); + } + + [TestMethod] + public async Task Public_helpers_parameterise_and_match_text() + { + var row = Expression.Parameter(typeof(Order), "o"); + var number = Expression.Property(row, nameof(Order.Number)); + var suffix = Expression.Lambda>(QueryExpressions.TextMatch(number, "50%", TextMatchKind.Suffix), row); + var total = Expression.Lambda>( + Expression.GreaterThan(Expression.Property(row, nameof(Order.Total)), QueryExpressions.Parameter("250.5", typeof(decimal))), row); + + ClearCommands(); + await using var db = NewContext(); + CollectionAssert.AreEqual(All.Where(o => o.Number.EndsWith("50%")).Select(o => o.Id).ToList(), + await db.Orders.Where(suffix).OrderBy(o => o.Id).Select(o => o.Id).ToListAsync()); + CollectionAssert.AreEqual(All.Where(o => o.Total > 250.5m).Select(o => o.Id).OrderBy(i => i).ToList(), + await db.Orders.Where(total).OrderBy(o => o.Id).Select(o => o.Id).ToListAsync()); + Assert.IsFalse(string.Join("\n", Commands).Contains("250.5")); + Assert.AreEqual(OrderStatus.Shipped, QueryExpressions.Convert("shipped", typeof(OrderStatus))); + } +} diff --git a/SW.EfCoreExtensions.PgSql/SW.EfCoreExtensions.PgSql.csproj b/SW.EfCoreExtensions.PgSql/SW.EfCoreExtensions.PgSql.csproj index b544ce3..b43c224 100644 --- a/SW.EfCoreExtensions.PgSql/SW.EfCoreExtensions.PgSql.csproj +++ b/SW.EfCoreExtensions.PgSql/SW.EfCoreExtensions.PgSql.csproj @@ -23,6 +23,8 @@ + + diff --git a/SW.EfCoreExtensions.PgSql/SearchyTextMatch.cs b/SW.EfCoreExtensions.PgSql/SearchyTextMatch.cs index 44929d8..080a948 100644 --- a/SW.EfCoreExtensions.PgSql/SearchyTextMatch.cs +++ b/SW.EfCoreExtensions.PgSql/SearchyTextMatch.cs @@ -12,14 +12,19 @@ internal static partial class ExpressionExtensions const string LikeEscapeCharacter = "\\"; // ILIKE: case-insensitive, and served by pg_trgm indexes like LIKE is. - private static partial Expression TextMatch(Expression field, string value, bool prefixOnly) => + private static partial Expression TextMatch(Expression field, string value, TextMatchKind kind) => Expression.Call( typeof(NpgsqlDbFunctionsExtensions), nameof(NpgsqlDbFunctionsExtensions.ILike), Type.EmptyTypes, Expression.Property(null, typeof(EF), nameof(EF.Functions)), field, - SearchyValue.Captured(prefixOnly ? $"{EscapeLikePattern(value)}%" : $"%{EscapeLikePattern(value)}%", typeof(string)), + SearchyValue.Captured(kind switch + { + TextMatchKind.Prefix => $"{EscapeLikePattern(value)}%", + TextMatchKind.Suffix => $"%{EscapeLikePattern(value)}", + _ => $"%{EscapeLikePattern(value)}%", + }, typeof(string)), Expression.Constant(LikeEscapeCharacter) ); diff --git a/SW.EfCoreExtensions/Querying/KeysetExtensions.cs b/SW.EfCoreExtensions/Querying/KeysetExtensions.cs new file mode 100644 index 0000000..0190b72 --- /dev/null +++ b/SW.EfCoreExtensions/Querying/KeysetExtensions.cs @@ -0,0 +1,147 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace SW.EfCoreExtensions +{ + /// One key of a keyset ordering: a lambda over the entity and its direction. + public sealed record KeysetKey(LambdaExpression Key, bool Descending = false) + { + public static KeysetKey Asc(Expression> key) => new(key); + + public static KeysetKey Desc(Expression> key) => new(key, Descending: true); + } + + /// The count of a query up to a limit, and whether there were more rows than that. + public readonly record struct CountUpTo(int Count, bool Exceeded); + + /// + /// Keyset ("seek") paging and cheap counts. A keyset page reads the next rows after the last row + /// already seen, so a deep page costs the same as the first, unlike Skip, which reads and discards + /// every row before it. + /// + public static class KeysetExtensions + { + /// + /// Orders by . The keys must identify a row, so end them with a unique + /// column, the primary key for instance. + /// + public static IOrderedQueryable OrderByKeyset(this IQueryable query, IReadOnlyList keys) + { + if (keys == null || keys.Count == 0) + throw new ArgumentException("A keyset needs at least one key.", nameof(keys)); + + IOrderedQueryable ordered = null; + foreach (var key in keys) + { + var lambda = Lambda(key.Key); + var name = (ordered == null ? "OrderBy" : "ThenBy") + (key.Descending ? "Descending" : ""); + var expression = Expression.Call(typeof(Queryable), name, new[] { typeof(T), lambda.ReturnType }, + (ordered ?? query).Expression, Expression.Quote(lambda)); + ordered = (IOrderedQueryable)query.Provider.CreateQuery(expression); + } + return ordered; + } + + /// + /// The rows that come after the row whose key values are , in the + /// order of : (k1 > v1) OR (k1 = v1 AND k2 > v2) OR ..., with + /// < for descending keys. Apply before . The values are sent + /// as parameters. Keys can't be nullable: SQL comparisons with NULL match nothing, so rows + /// would go missing. + /// + public static IQueryable AfterKeyset(this IQueryable query, IReadOnlyList keys, IReadOnlyList after) + { + if (after == null) + return query; + if (keys == null || keys.Count == 0 || after.Count != keys.Count) + throw new ArgumentException("A keyset position needs one value per key.", nameof(after)); + + var row = Expression.Parameter(typeof(T), "row"); + Expression predicate = null; + + for (var i = keys.Count - 1; i >= 0; i--) + { + var member = RowProjection.Inline(Lambda(keys[i].Key), row); + var value = SearchyValue.Captured(after[i], member.Type); + var beyond = Compare(member, value, keys[i].Descending ? ExpressionType.LessThan : ExpressionType.GreaterThan); + // Rows past this key, or level with it and past the keys that follow. + predicate = predicate == null + ? beyond + : Expression.OrElse(beyond, Expression.AndAlso(Compare(member, value, ExpressionType.Equal), predicate)); + } + + return query.Where(Expression.Lambda>(predicate, row)); + } + + /// + /// Up to rows, and whether more follow, from one query that reads + /// one row beyond the page: no separate count. + /// + public static async Task<(List Items, bool HasMore)> ToPageAsync(this IQueryable query, int pageSize, + CancellationToken cancellationToken = default) + { + if (pageSize <= 0) + throw new ArgumentOutOfRangeException(nameof(pageSize), "A page holds at least one row."); + + var items = await query.Take(pageSize + 1).ToListAsync(cancellationToken); + var hasMore = items.Count > pageSize; + if (hasMore) + items.RemoveAt(pageSize); + return (items, hasMore); + } + + /// + /// Counts rows up to , reading at most one row beyond it: a list shows + /// "10,000+" instead of paying for an exact count of millions. + /// + public static async Task CountUpToAsync(this IQueryable query, int limit, + CancellationToken cancellationToken = default) + { + if (limit < 0) + throw new ArgumentOutOfRangeException(nameof(limit)); + + var count = await query.Take(limit + 1).CountAsync(cancellationToken); + return count > limit ? new CountUpTo(limit, true) : new CountUpTo(count, false); + } + + static LambdaExpression Lambda(LambdaExpression key) + { + if (key.Parameters.Count != 1 || !key.Parameters[0].Type.IsAssignableFrom(typeof(T))) + throw new ArgumentException($"A key is a lambda over {typeof(T).Name}.", nameof(key)); + if (Nullable.GetUnderlyingType(key.ReturnType) != null) + throw new ArgumentException($"A keyset key can't be nullable ({key}).", nameof(key)); + return key; + } + + // Strings and enums have no comparison operators in expression trees: strings go through + // string.Compare, which EF Core translates to a plain comparison, enums through their + // underlying type. + static Expression Compare(Expression left, Expression right, ExpressionType comparison) + { + if (left.Type == typeof(string)) + { + if (comparison == ExpressionType.Equal) + return Expression.Equal(left, right); + var compare = Expression.Call(typeof(string).GetMethod(nameof(string.Compare), new[] { typeof(string), typeof(string) }), left, right); + return Expression.MakeBinary(comparison, compare, Expression.Constant(0)); + } + + if (left.Type.IsEnum) + { + var underlying = Enum.GetUnderlyingType(left.Type); + left = Expression.Convert(left, underlying); + right = Expression.Convert(right, underlying); + } + + if (left.Type == typeof(bool) || left.Type == typeof(Guid)) + throw new ArgumentException($"A keyset key of type {left.Type.Name} can't be compared in SQL; use another key."); + + return Expression.MakeBinary(comparison, left, right); + } + } +} diff --git a/SW.EfCoreExtensions/Querying/QueryExpressions.cs b/SW.EfCoreExtensions/Querying/QueryExpressions.cs new file mode 100644 index 0000000..e0c65ff --- /dev/null +++ b/SW.EfCoreExtensions/Querying/QueryExpressions.cs @@ -0,0 +1,39 @@ +using System; +using System.Linq.Expressions; + +namespace SW.EfCoreExtensions +{ + /// How a text match anchors the search text. + public enum TextMatchKind + { + Contains, + Prefix, + Suffix, + } + + /// + /// Building blocks for code that builds its own filter expressions: values as SQL parameters, and + /// the package's text match (case-insensitive ILIKE in the PostgreSQL package). + /// + public static class QueryExpressions + { + /// + /// , converted to , as an expression EF Core sends + /// as a parameter rather than a literal, so a query shape compiles once for every value. + /// + public static Expression Parameter(object value, Type type) => SearchyValue.Captured(value, type); + + /// + /// Converts to with the invariant culture, + /// covering Guid, enums by name or number and booleans written as 1/0. + /// + public static object Convert(object value, Type type) => SearchyValue.Convert(value, type); + + /// + /// (a string expression) matching , with + /// wildcard characters in the text matched literally. + /// + public static Expression TextMatch(Expression field, string text, TextMatchKind kind) => + ExpressionExtensions.ProviderTextMatch(field, text, kind); + } +} diff --git a/SW.EfCoreExtensions/Querying/RowProjection.cs b/SW.EfCoreExtensions/Querying/RowProjection.cs new file mode 100644 index 0000000..e8b3957 --- /dev/null +++ b/SW.EfCoreExtensions/Querying/RowProjection.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; + +namespace SW.EfCoreExtensions +{ + /// + /// Builds a projection to object[] rows from a list of column expressions, so a query reads + /// exactly the columns a caller asked for. The shape depends only on which columns are chosen, + /// never on values, so EF Core compiles each distinct choice of columns once. + /// + /// + /// + /// var row = RowProjection.Of<Order>( + /// (Expression<Func<Order, string>>)(o => o.Number), + /// (Expression<Func<Order, string>>)(o => o.Customer.Name), + /// RowProjection.Many<Order, OrderLine>(o => o.Lines, take: 10, + /// (Expression<Func<OrderLine, string>>)(l => l.Sku))); + /// var rows = await db.Orders.Select(row).ToListAsync(); // [["ORD-1", "Contoso", [["SKU-1"]]], ...] + /// + /// + public static class RowProjection + { + /// + /// A row of , each a lambda over , in order. + /// Use for a column holding a collection's rows. + /// + public static Expression> Of(params LambdaExpression[] columns) => + Of((IEnumerable)columns); + + /// + public static Expression> Of(IEnumerable columns) + { + var row = Expression.Parameter(typeof(T), "row"); + return Expression.Lambda>( + Expression.NewArrayInit(typeof(object), columns.Select(c => Expression.Convert(Inline(c, row), typeof(object)))), + row); + } + + /// + /// A column holding the rows of a collection: for each element + /// of , in order, at most + /// of them when given. + /// + public static Expression>> Many( + Expression>> collection, + int? take, + params LambdaExpression[] childColumns) => + Many(collection, take, orderBy: null, descending: false, childColumns); + + /// + public static Expression>> Many( + Expression>> collection, + int? take, + LambdaExpression orderBy, + bool descending, + params LambdaExpression[] childColumns) + { + Expression children = collection.Body; + + if (orderBy != null) + children = Expression.Call(typeof(Enumerable), descending ? nameof(Enumerable.OrderByDescending) : nameof(Enumerable.OrderBy), + new[] { typeof(TChild), orderBy.ReturnType }, children, orderBy); + + if (take != null) + children = Expression.Call(typeof(Enumerable), nameof(Enumerable.Take), new[] { typeof(TChild) }, + children, Expression.Constant(take.Value)); + + var rows = Expression.Call(typeof(Enumerable), nameof(Enumerable.Select), new[] { typeof(TChild), typeof(object[]) }, + children, Of(childColumns)); + var list = Expression.Call(typeof(Enumerable), nameof(Enumerable.ToList), new[] { typeof(object[]) }, rows); + + return Expression.Lambda>>(list, collection.Parameters); + } + + /// The body of with its parameter replaced by . + internal static Expression Inline(LambdaExpression lambda, Expression argument) + { + if (lambda.Parameters.Count != 1) + throw new ArgumentException("A column is a lambda with exactly one parameter.", nameof(lambda)); + if (!lambda.Parameters[0].Type.IsAssignableFrom(argument.Type)) + throw new ArgumentException($"A column over {lambda.Parameters[0].Type.Name} can't be read from {argument.Type.Name}.", nameof(lambda)); + + return new ReplaceParameter(lambda.Parameters[0], argument).Visit(lambda.Body); + } + + private sealed class ReplaceParameter(ParameterExpression from, Expression to) : ExpressionVisitor + { + protected override Expression VisitParameter(ParameterExpression node) => node == from ? to : node; + } + } +} diff --git a/SW.EfCoreExtensions/Searchy/ExpressionExtensions.cs b/SW.EfCoreExtensions/Searchy/ExpressionExtensions.cs index 29aa8d6..7cfe5ae 100644 --- a/SW.EfCoreExtensions/Searchy/ExpressionExtensions.cs +++ b/SW.EfCoreExtensions/Searchy/ExpressionExtensions.cs @@ -67,10 +67,10 @@ static Expression BuildFilterExpression(this Expression parameter, ISea switch (filter.Rule) { case SearchyRule.StartsWith: - return TextMatch(fieldNameExpression, $"{filter.Value}", prefixOnly: true); + return TextMatch(fieldNameExpression, $"{filter.Value}", TextMatchKind.Prefix); case SearchyRule.Contains: - return TextMatch(fieldNameExpression, $"{filter.Value}", prefixOnly: false); + return TextMatch(fieldNameExpression, $"{filter.Value}", TextMatchKind.Contains); case SearchyRule.EqualsTo: return Expression.Equal(fieldNameExpression, SearchyValue.Captured(filter.Value, fieldType)); @@ -114,7 +114,10 @@ static Expression BuildFilterExpression(this Expression parameter, ISea /// The text rules, which differ by provider. is the raw search text; /// the implementation must pass it as a captured value, never as a constant. /// - private static partial Expression TextMatch(Expression field, string value, bool prefixOnly); + private static partial Expression TextMatch(Expression field, string value, TextMatchKind kind); + + /// The provider's text match, for . + internal static Expression ProviderTextMatch(Expression field, string value, TextMatchKind kind) => TextMatch(field, value, kind); static RangeValues PopulateRangeValues(object valueCollection) { diff --git a/SW.EfCoreExtensions/SearchyTextMatch.cs b/SW.EfCoreExtensions/SearchyTextMatch.cs index ac98cce..f63ca4c 100644 --- a/SW.EfCoreExtensions/SearchyTextMatch.cs +++ b/SW.EfCoreExtensions/SearchyTextMatch.cs @@ -4,10 +4,15 @@ namespace SW.EfCoreExtensions { internal static partial class ExpressionExtensions { - // string.StartsWith/Contains: LIKE on relational providers, case as the column's collation has it. - private static partial Expression TextMatch(Expression field, string value, bool prefixOnly) => + // string.StartsWith/Contains/EndsWith: LIKE on relational providers, case as the column's collation has it. + private static partial Expression TextMatch(Expression field, string value, TextMatchKind kind) => Expression.Call(field, - typeof(string).GetMethod(prefixOnly ? nameof(string.StartsWith) : nameof(string.Contains), new[] { typeof(string) }), + typeof(string).GetMethod(kind switch + { + TextMatchKind.Prefix => nameof(string.StartsWith), + TextMatchKind.Suffix => nameof(string.EndsWith), + _ => nameof(string.Contains), + }, new[] { typeof(string) }), SearchyValue.Captured(value, typeof(string))); } }