feat(querying): row projections, keyset paging and capped counts

README.md

@@ -68,6 +68,15 @@ number) and booleans written as `1`/`0`. Sorts accept any property type and dott
 (`Customer.Address.City`). In `SimplyWorks.EfCoreExtensions.PgSql`, `StartsWith` and `Contains` are
 case-insensitive `ILIKE`, with `%`, `_` and `\` in the value matched literally.
 
+#### Query shaping
+>`RowProjection.Of<T>(columns)` / `RowProjection.Many<T, TChild>(...)`: project to `object[]` rows of the chosen columns, nested collections included, so a query reads only what the caller asked for.
+
+>`AfterKeyset` / `OrderByKeyset`: keyset ("seek") paging; a deep page costs the same as the first.
+
+>`ToPageAsync(size)`: a page and whether more rows follow, without a count. `CountUpToAsync(limit)`: a count that stops at a limit ("10,000+").
+
+>`QueryExpressions.Parameter(value, type)` / `QueryExpressions.TextMatch(field, text, kind)`: for code that builds its own filters, a value EF Core sends as a parameter, and the package's text match (escaped `ILIKE` in PgSql).
+
 
 ## Getting support 👷
 If you encounter any bugs, don't hesitate to submit an [issue](https://github.com/simplify9/EfCoreExtensions/issues). We'll get back to you promptly!

SW.EfCoreExtensions.PgSql.IntegrationTests/QueryShapingTests.cs

@@ -0,0 +1,157 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using static SW.EfCoreExtensions.PgSql.IntegrationTests.OrdersDatabase;
+
+namespace SW.EfCoreExtensions.PgSql.IntegrationTests;
+
+[TestClass]
+public class QueryShapingTests
+{
+    static LambdaExpression Column<T, TValue>(Expression<Func<T, TValue>> column) => column;
+
+    [TestMethod]
+    public async Task A_row_reads_only_the_chosen_columns()
+    {
+        ClearCommands();
+        await using var db = NewContext();
+        var rows = await db.Orders.OrderBy(o => o.Id).Take(3)
+            .Select(RowProjection.Of<Order>(Column((Order o) => o.Number), Column((Order o) => o.Customer.Name)))
+            .ToListAsync();
+
+        Assert.AreEqual(All[0].Number, rows[0][0]);
+        Assert.AreEqual(All[0].Customer.Name, rows[0][1]);
+
+        var sql = Commands.Last();
+        var select = sql[sql.IndexOf("SELECT", StringComparison.Ordinal)..sql.IndexOf("FROM", StringComparison.Ordinal)];
+        StringAssert.Contains(select, "\"Number\"");
+        StringAssert.Contains(select, "\"Name\"");
+        foreach (var unselected in new[] { "\"Total\"", "\"Discount\"", "\"ExternalId\"", "\"PlacedOn\"", "\"Country\"" })
+            Assert.IsFalse(select.Contains(unselected), $"{unselected} was read:\n{sql}");
+    }
+
+    [TestMethod]
+    public async Task A_collection_column_reads_its_own_chosen_columns_in_order_and_capped()
+    {
+        await using var db = NewContext();
+        var row = RowProjection.Of<Order>(
+            Column((Order o) => o.Number),
+            RowProjection.Many<Order, OrderLine>(o => o.Lines, take: 1,
+                orderBy: Column((OrderLine l) => l.Quantity), descending: true,
+                Column((OrderLine l) => l.Sku), Column((OrderLine l) => l.Quantity)));
+
+        ClearCommands();
+        var rows = await db.Orders.Where(o => o.Id <= 3).OrderBy(o => o.Id).Select(row).ToListAsync();
+
+        var first = (List<object[]>)rows[0][1];
+        Assert.AreEqual(1, first.Count);
+        Assert.AreEqual(All[0].Lines.Single().Sku, first[0][0]);
+        Assert.IsFalse(Commands.Last().Contains("\"Total\""), Commands.Last());
+    }
+
+    [TestMethod]
+    public async Task The_same_columns_compile_once()
+    {
+        await using (var warm = NewContext())
+            await warm.Orders.Where(o => o.Paid).Select(RowProjection.Of<Order>(Column((Order o) => o.Number))).ToListAsync();
+        var before = Compilations;
+
+        for (var i = 0; i < 5; i++)
+        {
+            await using var db = NewContext();
+            // A new expression tree each time, as a request builds it.
+            await db.Orders.Where(o => o.Paid).Select(RowProjection.Of<Order>(Column((Order o) => o.Number))).ToListAsync();
+        }
+
+        Assert.AreEqual(before, Compilations);
+    }
+
+    public static IEnumerable<object[]> Keysets => new[]
+    {
+        // Unique key alone.
+        new object[] { new[] { KeysetKey.Asc((Order o) => o.Id) }, 4 },
+        // Many orders share a status: the id breaks the ties.
+        new object[] { new[] { KeysetKey.Asc((Order o) => o.Status), KeysetKey.Asc((Order o) => o.Id) }, 4 },
+        new object[] { new[] { KeysetKey.Desc((Order o) => o.Total), KeysetKey.Desc((Order o) => o.Id) }, 7 },
+        new object[] { new[] { KeysetKey.Asc((Order o) => o.Customer.Name), KeysetKey.Desc((Order o) => o.PlacedOn), KeysetKey.Asc((Order o) => o.Id) }, 5 },
+    };
+
+    [TestMethod]
+    [DynamicData(nameof(Keysets))]
+    public async Task Keyset_pages_visit_every_row_once_in_order(KeysetKey[] keys, int pageSize)
+    {
+        await using var db = NewContext();
+        var expected = await db.Orders.OrderByKeyset(keys).Select(o => o.Id).ToListAsync();
+
+        var visited = new List<int>();
+        object[] after = null;
+        var compiled = 0;
+        for (var page = 0; page < 100; page++)
+        {
+            var before = Compilations;
+            var (items, hasMore) = await db.Orders.Include(o => o.Customer).AfterKeyset(keys, after).OrderByKeyset(keys)
+                .AsNoTracking().ToPageAsync(pageSize);
+            if (page > 1) compiled += Compilations - before;
+
+            visited.AddRange(items.Select(o => o.Id));
+            if (!hasMore) break;
+            var last = items.Last();
+            after = keys.Select(k => k.Key.Compile().DynamicInvoke(last)).ToArray();
+        }
+
+        CollectionAssert.AreEqual(expected, visited);
+        Assert.AreEqual(0, compiled, "a new position must not compile a new query");
+    }
+
+    [TestMethod]
+    public void A_nullable_key_is_refused()
+    {
+        Assert.ThrowsException<ArgumentException>(() =>
+            NewContext().Orders.OrderByKeyset(new[] { KeysetKey.Asc((Order o) => o.Discount) }));
+    }
+
+    [TestMethod]
+    public async Task A_page_says_whether_more_rows_follow()
+    {
+        await using var db = NewContext();
+        var (items, hasMore) = await db.Orders.OrderBy(o => o.Id).ToPageAsync(All.Count - 1);
+        Assert.AreEqual(All.Count - 1, items.Count);
+        Assert.IsTrue(hasMore);
+
+        (items, hasMore) = await db.Orders.OrderBy(o => o.Id).ToPageAsync(All.Count);
+        Assert.AreEqual(All.Count, items.Count);
+        Assert.IsFalse(hasMore);
+    }
+
+    [TestMethod]
+    public async Task A_count_stops_at_its_limit()
+    {
+        await using var db = NewContext();
+        Assert.AreEqual(new CountUpTo(10, true), await db.Orders.CountUpToAsync(10));
+        Assert.AreEqual(new CountUpTo(All.Count, false), await db.Orders.CountUpToAsync(All.Count));
+        Assert.AreEqual(new CountUpTo(0, false), await db.Orders.Where(o => o.Id < 0).CountUpToAsync(10));
+    }
+
+    [TestMethod]
+    public async Task Public_helpers_parameterise_and_match_text()
+    {
+        var row = Expression.Parameter(typeof(Order), "o");
+        var number = Expression.Property(row, nameof(Order.Number));
+        var suffix = Expression.Lambda<Func<Order, bool>>(QueryExpressions.TextMatch(number, "50%", TextMatchKind.Suffix), row);
+        var total = Expression.Lambda<Func<Order, bool>>(
+            Expression.GreaterThan(Expression.Property(row, nameof(Order.Total)), QueryExpressions.Parameter("250.5", typeof(decimal))), row);
+
+        ClearCommands();
+        await using var db = NewContext();
+        CollectionAssert.AreEqual(All.Where(o => o.Number.EndsWith("50%")).Select(o => o.Id).ToList(),
+            await db.Orders.Where(suffix).OrderBy(o => o.Id).Select(o => o.Id).ToListAsync());
+        CollectionAssert.AreEqual(All.Where(o => o.Total > 250.5m).Select(o => o.Id).OrderBy(i => i).ToList(),
+            await db.Orders.Where(total).OrderBy(o => o.Id).Select(o => o.Id).ToListAsync());
+        Assert.IsFalse(string.Join("\n", Commands).Contains("250.5"));
+        Assert.AreEqual(OrderStatus.Shipped, QueryExpressions.Convert("shipped", typeof(OrderStatus)));
+    }
+}

SW.EfCoreExtensions.PgSql/SW.EfCoreExtensions.PgSql.csproj

@@ -23,6 +23,8 @@
   <ItemGroup>
     <!-- Searchy is provider-neutral apart from SearchyTextMatch.cs, so the source is shared with the core package. -->
     <Compile Include="..\SW.EfCoreExtensions\Searchy\**\*.cs" LinkBase="Searchy" />
+    <!-- Provider-neutral query shaping (projection rows, keyset paging, capped counts). -->
+    <Compile Include="..\SW.EfCoreExtensions\Querying\**\*.cs" LinkBase="Querying" />
   </ItemGroup>
 
   <ItemGroup>

SW.EfCoreExtensions.PgSql/SearchyTextMatch.cs

@@ -12,14 +12,19 @@ internal static partial class ExpressionExtensions
         const string LikeEscapeCharacter = "\\";
 
         // ILIKE: case-insensitive, and served by pg_trgm indexes like LIKE is.
-        private static partial Expression TextMatch(Expression field, string value, bool prefixOnly) =>
+        private static partial Expression TextMatch(Expression field, string value, TextMatchKind kind) =>
             Expression.Call(
                 typeof(NpgsqlDbFunctionsExtensions),
                 nameof(NpgsqlDbFunctionsExtensions.ILike),
                 Type.EmptyTypes,
                 Expression.Property(null, typeof(EF), nameof(EF.Functions)),
                 field,
-                SearchyValue.Captured(prefixOnly ? $"{EscapeLikePattern(value)}%" : $"%{EscapeLikePattern(value)}%", typeof(string)),
+                SearchyValue.Captured(kind switch
+                {
+                    TextMatchKind.Prefix => $"{EscapeLikePattern(value)}%",
+                    TextMatchKind.Suffix => $"%{EscapeLikePattern(value)}",
+                    _ => $"%{EscapeLikePattern(value)}%",
+                }, typeof(string)),
                 Expression.Constant(LikeEscapeCharacter)
             );
 

SW.EfCoreExtensions/Querying/KeysetExtensions.cs

@@ -0,0 +1,147 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+
+namespace SW.EfCoreExtensions
+{
+    /// <summary>One key of a keyset ordering: a lambda over the entity and its direction.</summary>
+    public sealed record KeysetKey(LambdaExpression Key, bool Descending = false)
+    {
+        public static KeysetKey Asc<T, TKey>(Expression<Func<T, TKey>> key) => new(key);
+
+        public static KeysetKey Desc<T, TKey>(Expression<Func<T, TKey>> key) => new(key, Descending: true);
+    }
+
+    /// <summary>The count of a query up to a limit, and whether there were more rows than that.</summary>
+    public readonly record struct CountUpTo(int Count, bool Exceeded);
+
+    /// <summary>
+    /// Keyset ("seek") paging and cheap counts. A keyset page reads the next rows after the last row
+    /// already seen, so a deep page costs the same as the first, unlike Skip, which reads and discards
+    /// every row before it.
+    /// </summary>
+    public static class KeysetExtensions
+    {
+        /// <summary>
+        /// Orders by <paramref name="keys"/>. The keys must identify a row, so end them with a unique
+        /// column, the primary key for instance.
+        /// </summary>
+        public static IOrderedQueryable<T> OrderByKeyset<T>(this IQueryable<T> query, IReadOnlyList<KeysetKey> keys)
+        {
+            if (keys == null || keys.Count == 0)
+                throw new ArgumentException("A keyset needs at least one key.", nameof(keys));
+
+            IOrderedQueryable<T> ordered = null;
+            foreach (var key in keys)
+            {
+                var lambda = Lambda<T>(key.Key);
+                var name = (ordered == null ? "OrderBy" : "ThenBy") + (key.Descending ? "Descending" : "");
+                var expression = Expression.Call(typeof(Queryable), name, new[] { typeof(T), lambda.ReturnType },
+                    (ordered ?? query).Expression, Expression.Quote(lambda));
+                ordered = (IOrderedQueryable<T>)query.Provider.CreateQuery<T>(expression);
+            }
+            return ordered;
+        }
+
+        /// <summary>
+        /// The rows that come after the row whose key values are <paramref name="after"/>, in the
+        /// order of <paramref name="keys"/>: (k1 &gt; v1) OR (k1 = v1 AND k2 &gt; v2) OR ..., with
+        /// &lt; for descending keys. Apply before <see cref="OrderByKeyset{T}"/>. The values are sent
+        /// as parameters. Keys can't be nullable: SQL comparisons with NULL match nothing, so rows
+        /// would go missing.
+        /// </summary>
+        public static IQueryable<T> AfterKeyset<T>(this IQueryable<T> query, IReadOnlyList<KeysetKey> keys, IReadOnlyList<object> after)
+        {
+            if (after == null)
+                return query;
+            if (keys == null || keys.Count == 0 || after.Count != keys.Count)
+                throw new ArgumentException("A keyset position needs one value per key.", nameof(after));
+
+            var row = Expression.Parameter(typeof(T), "row");
+            Expression predicate = null;
+
+            for (var i = keys.Count - 1; i >= 0; i--)
+            {
+                var member = RowProjection.Inline(Lambda<T>(keys[i].Key), row);
+                var value = SearchyValue.Captured(after[i], member.Type);
+                var beyond = Compare(member, value, keys[i].Descending ? ExpressionType.LessThan : ExpressionType.GreaterThan);
+                // Rows past this key, or level with it and past the keys that follow.
+                predicate = predicate == null
+                    ? beyond
+                    : Expression.OrElse(beyond, Expression.AndAlso(Compare(member, value, ExpressionType.Equal), predicate));
+            }
+
+            return query.Where(Expression.Lambda<Func<T, bool>>(predicate, row));
+        }
+
+        /// <summary>
+        /// Up to <paramref name="pageSize"/> rows, and whether more follow, from one query that reads
+        /// one row beyond the page: no separate count.
+        /// </summary>
+        public static async Task<(List<T> Items, bool HasMore)> ToPageAsync<T>(this IQueryable<T> query, int pageSize,
+            CancellationToken cancellationToken = default)
+        {
+            if (pageSize <= 0)
+                throw new ArgumentOutOfRangeException(nameof(pageSize), "A page holds at least one row.");
+
+            var items = await query.Take(pageSize + 1).ToListAsync(cancellationToken);
+            var hasMore = items.Count > pageSize;
+            if (hasMore)
+                items.RemoveAt(pageSize);
+            return (items, hasMore);
+        }
+
+        /// <summary>
+        /// Counts rows up to <paramref name="limit"/>, reading at most one row beyond it: a list shows
+        /// "10,000+" instead of paying for an exact count of millions.
+        /// </summary>
+        public static async Task<CountUpTo> CountUpToAsync<T>(this IQueryable<T> query, int limit,
+            CancellationToken cancellationToken = default)
+        {
+            if (limit < 0)
+                throw new ArgumentOutOfRangeException(nameof(limit));
+
+            var count = await query.Take(limit + 1).CountAsync(cancellationToken);
+            return count > limit ? new CountUpTo(limit, true) : new CountUpTo(count, false);
+        }
+
+        static LambdaExpression Lambda<T>(LambdaExpression key)
+        {
+            if (key.Parameters.Count != 1 || !key.Parameters[0].Type.IsAssignableFrom(typeof(T)))
+                throw new ArgumentException($"A key is a lambda over {typeof(T).Name}.", nameof(key));
+            if (Nullable.GetUnderlyingType(key.ReturnType) != null)
+                throw new ArgumentException($"A keyset key can't be nullable ({key}).", nameof(key));
+            return key;
+        }
+
+        // Strings and enums have no comparison operators in expression trees: strings go through
+        // string.Compare, which EF Core translates to a plain comparison, enums through their
+        // underlying type.
+        static Expression Compare(Expression left, Expression right, ExpressionType comparison)
+        {
+            if (left.Type == typeof(string))
+            {
+                if (comparison == ExpressionType.Equal)
+                    return Expression.Equal(left, right);
+                var compare = Expression.Call(typeof(string).GetMethod(nameof(string.Compare), new[] { typeof(string), typeof(string) }), left, right);
+                return Expression.MakeBinary(comparison, compare, Expression.Constant(0));
+            }
+
+            if (left.Type.IsEnum)
+            {
+                var underlying = Enum.GetUnderlyingType(left.Type);
+                left = Expression.Convert(left, underlying);
+                right = Expression.Convert(right, underlying);
+            }
+
+            if (left.Type == typeof(bool) || left.Type == typeof(Guid))
+                throw new ArgumentException($"A keyset key of type {left.Type.Name} can't be compared in SQL; use another key.");
+
+            return Expression.MakeBinary(comparison, left, right);
+        }
+    }
+}

SW.EfCoreExtensions/Querying/QueryExpressions.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Linq.Expressions;
+
+namespace SW.EfCoreExtensions
+{
+    /// <summary>How a text match anchors the search text.</summary>
+    public enum TextMatchKind
+    {
+        Contains,
+        Prefix,
+        Suffix,
+    }
+
+    /// <summary>
+    /// Building blocks for code that builds its own filter expressions: values as SQL parameters, and
+    /// the package's text match (case-insensitive ILIKE in the PostgreSQL package).
+    /// </summary>
+    public static class QueryExpressions
+    {
+        /// <summary>
+        /// <paramref name="value"/>, converted to <paramref name="type"/>, as an expression EF Core sends
+        /// as a parameter rather than a literal, so a query shape compiles once for every value.
+        /// </summary>
+        public static Expression Parameter(object value, Type type) => SearchyValue.Captured(value, type);
+
+        /// <summary>
+        /// Converts <paramref name="value"/> to <paramref name="type"/> with the invariant culture,
+        /// covering Guid, enums by name or number and booleans written as 1/0.
+        /// </summary>
+        public static object Convert(object value, Type type) => SearchyValue.Convert(value, type);
+
+        /// <summary>
+        /// <paramref name="field"/> (a string expression) matching <paramref name="text"/>, with
+        /// wildcard characters in the text matched literally.
+        /// </summary>
+        public static Expression TextMatch(Expression field, string text, TextMatchKind kind) =>
+            ExpressionExtensions.ProviderTextMatch(field, text, kind);
+    }
+}