fix(editor): rerank incremental completions for the current prefix

CHANGELOG.md

@@ -13,8 +13,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Data Rewind settings in Settings > Data & Results, with an off switch and Clear Saved Changes.
 - Restore Previous Values in the toolbar's Table Actions group.
 
+### Changed
+
+- Editor tabs drawn as a segmented tab picker rather than in Liquid Glass, on every macOS version. (#2439)
+
 ### Fixed
 
+- Autocomplete keeping an earlier prefix's ordering after the typed word becomes an exact match. (#2444)
 - Whole MySQL and MariaDB result set fetched before a capped query returned its first rows. (#2427)
 - KILL sent to a different server when a MySQL or MariaDB connection's host is spelled `localhost`.
 - Save reporting the number of statements it ran as the number of rows it changed.
@@ -30,13 +35,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - A save reported as failed on an engine without transactions after some statements had been written.
 - Keep Open for a preview tab, by double-clicking it in the tab strip or from its contextual menu. (#2436)
 - Kafka driver plugin: topics in the grid, consumer group lag, and KafkaQL for seeking and producing. (#2419)
-
-### Changed
-
-- Editor tabs drawn as a segmented tab picker rather than in Liquid Glass, on every macOS version. (#2439)
-
-### Fixed
-
 - Active editor tab indistinguishable from the inactive ones in light appearance. (#2439)
 - Active editor tab drawn darker than its track on macOS 27, and inverting when the window lost focus.
 - Editor tab selection changing with the desktop picture behind the window.

TablePro/Core/Autocomplete/CompletionEngine.swift

@@ -11,6 +11,9 @@ import TableProPluginKit
 /// Completion context returned by the engine
 struct CompletionContext {
     let items: [SQLCompletionItem]
+    /// The wider ranked pool an open popup re-ranks against as the prefix grows. `items` is the
+    /// slice of it the popup shows.
+    let candidates: [SQLCompletionItem]
     let replacementRange: NSRange
     let sqlContext: SQLContext
 }
@@ -95,6 +98,7 @@ final class CompletionEngine {
 
         return CompletionContext(
             items: context.items,
+            candidates: context.candidates,
             replacementRange: mappedRange,
             sqlContext: context.sqlContext
         )
@@ -128,7 +132,7 @@ final class CompletionEngine {
 
         let adjustedCursor = cursorPosition - windowOffset
 
-        let (items, context) = await provider.getCompletions(
+        let (items, candidates, context) = await provider.completionSession(
             text: analysisText,
             cursorPosition: adjustedCursor,
             forcedTableReferences: forcedTableReferences
@@ -164,6 +168,7 @@ final class CompletionEngine {
 
         return CompletionContext(
             items: items,
+            candidates: candidates,
             replacementRange: replacementRange,
             sqlContext: adjustedContext
         )

TablePro/Core/Autocomplete/Mongo/MongoCompletionService.swift

@@ -8,6 +8,12 @@ final class MongoCompletionService: QueryCompletionService {
 
     private static let windowRadius = 5_000
 
+    /// How many candidates a session may keep for re-ranking as the prefix grows. Every position's
+    /// vocabulary is well under this except a collection's sampled field paths, which a wide
+    /// document schema can run into the thousands; those sessions prefer what the popup showed,
+    /// and the cap holds either way because filtering an empty opening prefix returns everything.
+    private static let sessionPoolLimit = 400
+
     init(schemaProvider: SQLSchemaProvider?, databaseType: DatabaseType?) {
         self.schemaProvider = schemaProvider
         _ = databaseType
@@ -29,14 +35,17 @@ final class MongoCompletionService: QueryCompletionService {
         MongoContextAnalyzer.prefixRange(in: text, endingAt: offset).location
     }
 
-    func filter(_ items: [SQLCompletionItem], prefix: String) -> [SQLCompletionItem] {
+    private func filter(_ items: [SQLCompletionItem], prefix: String) -> [SQLCompletionItem] {
         guard !prefix.isEmpty else { return items }
         let needle = prefix.lowercased()
         return items.filter { $0.filterText.hasPrefix(needle) || $0.filterText.contains(needle) }
             .sorted { lhs, rhs in
-                let lhsExact = lhs.filterText.hasPrefix(needle)
-                let rhsExact = rhs.filterText.hasPrefix(needle)
-                if lhsExact != rhsExact { return lhsExact }
+                let lhsComplete = lhs.filterText == needle
+                let rhsComplete = rhs.filterText == needle
+                if lhsComplete != rhsComplete { return lhsComplete }
+                let lhsAnchored = lhs.filterText.hasPrefix(needle)
+                let rhsAnchored = rhs.filterText.hasPrefix(needle)
+                if lhsAnchored != rhsAnchored { return lhsAnchored }
                 if lhs.sortPriority != rhs.sortPriority { return lhs.sortPriority < rhs.sortPriority }
                 return lhs.label < rhs.label
             }
@@ -58,8 +67,12 @@ final class MongoCompletionService: QueryCompletionService {
         let items = await items(for: context.position)
         guard !items.isEmpty else { return nil }
 
+        let shown = filter(items, prefix: context.prefix)
+        let pool = items.count <= Self.sessionPoolLimit ? items : shown
+
         return QueryCompletionSession(
-            items: filter(items, prefix: context.prefix),
+            items: shown,
+            candidates: Array(pool.prefix(Self.sessionPoolLimit)),
             replacementRange: NSRange(
                 location: context.prefixRange.location + windowStart,
                 length: context.prefixRange.length

TablePro/Core/Autocomplete/QueryCompletionService.swift

@@ -2,7 +2,11 @@ import Foundation
 import TableProPluginKit
 
 struct QueryCompletionSession {
+    /// What the popup shows when it opens.
     let items: [SQLCompletionItem]
+    /// What it re-ranks against while it stays open, which is wider than `items` so a longer
+    /// prefix can promote a candidate the opening prefix ranked out of view.
+    let candidates: [SQLCompletionItem]
     let replacementRange: NSRange
 }
 
@@ -13,7 +17,6 @@ protocol QueryCompletionService: AnyObject {
     func seedItems() -> [SQLCompletionItem]
     func prepare() async
     func completions(in text: NSString, at offset: Int, isManualTrigger: Bool) async -> QueryCompletionSession?
-    func filter(_ items: [SQLCompletionItem], prefix: String) -> [SQLCompletionItem]
     func rank(_ items: [SQLCompletionItem], prefix: String) -> [SQLCompletionItem]
     func tokenStart(in text: NSString, endingAt offset: Int) -> Int
     func updateFavoriteKeywords(_ keywords: [String: (name: String, query: String)])

TablePro/Core/Autocomplete/SQLCompletionProvider.swift

@@ -76,13 +76,28 @@ final class SQLCompletionProvider {
         cursorPosition: Int,
         forcedTableReferences: [TableReference]? = nil
     ) async -> (items: [SQLCompletionItem], context: SQLContext) {
+        let session = await completionSession(
+            text: text,
+            cursorPosition: cursorPosition,
+            forcedTableReferences: forcedTableReferences
+        )
+        return (session.items, session.context)
+    }
+
+    /// The completions for the cursor position, as both what the popup shows and the wider pool
+    /// it re-ranks against while it stays open.
+    func completionSession(
+        text: String,
+        cursorPosition: Int,
+        forcedTableReferences: [TableReference]? = nil
+    ) async -> (items: [SQLCompletionItem], candidates: [SQLCompletionItem], context: SQLContext) {
         var context = contextAnalyzer.analyze(query: text, cursorPosition: cursorPosition)
         if let forcedTableReferences {
             context = context.replacingTableReferences(forcedTableReferences)
         }
 
         if context.isInsideString || context.isInsideComment {
-            return ([], context)
+            return ([], [], context)
         }
 
         var candidates = await getCandidates(for: context)
@@ -93,11 +108,36 @@ final class SQLCompletionProvider {
 
         candidates = rankResults(candidates, prefix: context.prefix, context: context)
 
-        let limited = Array(candidates.prefix(maxSuggestions(for: context.clauseType)))
+        let limit = maxSuggestions(for: context.clauseType)
 
-        return (limited, context)
+        return (Array(candidates.prefix(limit)), Array(candidates.prefix(sessionPool(for: limit))), context)
     }
 
+    /// Filter, rank and cut a session's candidates down to what the popup shows.
+    ///
+    /// The popup shows `maxSuggestions`, but the session holds `sessionPool` of them, because an
+    /// open popup re-ranks against what it kept rather than asking again. A candidate cut at the
+    /// opening prefix can never lead for a longer one however well the survivors are ordered, and
+    /// PostgreSQL declares more `T`-prefixed functions than the popup shows rows, every one of
+    /// them outranking the `TRUE` keyword for the single letter `t`.
+    func filterRankAndLimit(
+        _ items: [SQLCompletionItem],
+        prefix: String,
+        context: SQLContext
+    ) -> [SQLCompletionItem] {
+        Array(filterAndRank(items, prefix: prefix, context: context).prefix(maxSuggestions(for: context.clauseType)))
+    }
+
+    /// Ten times what the popup shows. Ranking is linear in the pool and the filter beside it
+    /// already walks every candidate: measured at 36us for 40 candidates and 340us for 400, against
+    /// 4ms at 5,000, which is why the pool is bounded rather than kept whole.
+    private func sessionPool(for limit: Int) -> Int { limit * 10 }
+
+    /// The ceiling for a session built without going through `completionSession`. The seeded
+    /// window is statement keywords plus every saved favorite, which has no natural bound, and it
+    /// is replaced by an analyzed session as soon as the request lands.
+    var seedPoolLimit: Int { sessionPool(for: maxSuggestions(for: .unknown)) }
+
     /// Generic SQL functions plus the active dialect's own functions (deduplicated).
     /// Cached per dialect; invalidated in `setDatabaseType`.
     private func functionItems() -> [SQLCompletionItem] {
@@ -716,15 +756,24 @@ final class SQLCompletionProvider {
 
     // MARK: - Ranking
 
-    /// Rank results by relevance
+    /// Rank results by relevance, lowest score first.
+    ///
+    /// Scores are resolved once per candidate rather than inside the comparator, which called
+    /// `calculateScore` twice per comparison. Ranking runs on every keystroke of an open popup,
+    /// so the candidate set is walked once and the sort then compares integers.
+    ///
+    /// Equal scores fall back to the candidate's own position, which is the order the generator
+    /// emitted it in and carries meaning `sorted(by:)` would otherwise be free to discard: the
+    /// standard library documents the sort as not stable.
     func rankResults(_ items: [SQLCompletionItem], prefix: String, context: SQLContext) -> [SQLCompletionItem] {
         let lowerPrefix = prefix.lowercased()
-
-        return items.sorted { a, b in
-            let aScore = calculateScore(for: a, prefix: lowerPrefix, context: context)
-            let bScore = calculateScore(for: b, prefix: lowerPrefix, context: context)
-            return aScore < bScore // Lower score = higher priority
+        let scored = items.enumerated().map { position, item in
+            (position: position, item: item, score: calculateScore(for: item, prefix: lowerPrefix, context: context))
         }
+
+        return scored.sorted { lhs, rhs in
+            lhs.score == rhs.score ? lhs.position < rhs.position : lhs.score < rhs.score
+        }.map(\.item)
     }
 
     /// Calculate ranking score for an item (lower = better).

TablePro/Core/Autocomplete/SQLCompletionService.swift

@@ -4,7 +4,7 @@ import TableProPluginKit
 @MainActor
 final class SQLCompletionService: QueryCompletionService {
     private let engine: CompletionEngine
-    private var lastContext: SQLContext?
+    private var lastContext = SQLContext.unanalyzed
 
     private static let windowRadius = 5_000
 
@@ -28,8 +28,13 @@ final class SQLCompletionService: QueryCompletionService {
 
     var triggerCharacters: Set<String> { [".", " ", ":", "(", ","] }
 
+    /// Seeding starts a session the analyzer has not seen, so the context a previous session
+    /// left behind stops describing anything. Ranking a seeded session against it would score
+    /// the new prefix under the old clause.
     func seedItems() -> [SQLCompletionItem] {
-        engine.keywordCompletions() + engine.allFavoriteItems()
+        lastContext = .unanalyzed
+        let items = engine.keywordCompletions() + engine.allFavoriteItems()
+        return Array(items.prefix(engine.provider.seedPoolLimit))
     }
 
     func prepare() async {
@@ -44,13 +49,8 @@ final class SQLCompletionService: QueryCompletionService {
         SQLTokenBoundary.segmentStart(in: text, endingAt: offset)
     }
 
-    func filter(_ items: [SQLCompletionItem], prefix: String) -> [SQLCompletionItem] {
-        engine.provider.filterByPrefix(items, prefix: prefix)
-    }
-
     func rank(_ items: [SQLCompletionItem], prefix: String) -> [SQLCompletionItem] {
-        guard let context = lastContext else { return filter(items, prefix: prefix) }
-        return engine.provider.filterAndRank(items, prefix: prefix, context: context)
+        engine.provider.filterRankAndLimit(items, prefix: prefix, context: lastContext)
     }
 
     func completions(in text: NSString, at offset: Int, isManualTrigger: Bool) async -> QueryCompletionSession? {
@@ -68,6 +68,7 @@ final class SQLCompletionService: QueryCompletionService {
         lastContext = context.sqlContext
         return QueryCompletionSession(
             items: context.items,
+            candidates: context.candidates,
             replacementRange: NSRange(
                 location: context.replacementRange.location + windowStart,
                 length: context.replacementRange.length

TablePro/Core/Autocomplete/SQLContextAnalyzer.swift

@@ -142,6 +142,20 @@ struct SQLContext {
             comparisonColumn: comparisonColumn
         )
     }
+
+    /// The context to rank against before the analyzer has produced one, which is the window
+    /// between the popup seeding itself with statement-start keywords and the first analyzed
+    /// request completing. It carries no clause and no tables, so ranking falls back to the
+    /// prefix and exact-match bonuses alone rather than not ranking at all.
+    static let unanalyzed = SQLContext(
+        clauseType: .unknown,
+        prefix: "",
+        prefixRange: 0..<0,
+        dotPrefix: nil,
+        tableReferences: [],
+        isInsideString: false,
+        isInsideComment: false
+    )
 }
 
 /// Analyzes SQL query to determine completion context