From fd8128f62701a5b8f87720b17c8c586f53c2be5c Mon Sep 17 00:00:00 2001 From: Max Chesnikov Date: Wed, 21 Oct 2026 01:26:59 -0700 Subject: [PATCH] Fix cursor bookkeeping in auto-possessification The two traversals in AutoPossessification.swift disagreed about where they left `position` in the flat node list: on the node itself, on the first node of a child's subtree, or on the subtree's last node. An optional quantification, for one, answered "no required first atom" without stepping over its child, so the enclosing concatenation went on to read a node from inside that quantification instead of the next sibling. In `(?i)(?:a*\+*)A{2}`, `a*` was compared against the `+` inside `\+*` rather than against `A`, made possessive on the strength of that, and the pattern stopped matching "aA". Give both functions the same invariant as the forward-progress walk: find the end of the subtree up front with `skipNode` and restore the cursor there on the way out, so that a case is free to look at as many or as few children as it needs. The concatenation case of `autoPossessifyNextQuantification` is now a single left-to-right pass carrying the quantification it is trying to possessify. A component with no determinable required first atom drops that quantification instead of handing it to the enclosing concatenation, which was possessifying it against an unrelated atom: `(?:a*[ab])b` no longer loses "aab". Every quantification of a concatenation is considered, not just the first one, so `\w+\s\d+a` possessifies both of its quantifiers. Fixes #898. --- .../Optimizations/AutoPossessification.swift | 133 ++++++++++-------- Tests/RegexTests/MatchTests.swift | 22 +++ Tests/RegexTests/OptimizationTests.swift | 53 +++++++ 3 files changed, 147 insertions(+), 61 deletions(-) diff --git a/Sources/_StringProcessing/Optimizations/AutoPossessification.swift b/Sources/_StringProcessing/Optimizations/AutoPossessification.swift index e2f266f05..87c3e3481 100644 --- a/Sources/_StringProcessing/Optimizations/AutoPossessification.swift +++ b/Sources/_StringProcessing/Optimizations/AutoPossessification.swift @@ -12,6 +12,16 @@ internal import _RegexParser extension DSLList { + /// Returns the required first atom of the node at `position`, leaving + /// `position` just past that node's entire subtree, so that a caller walking + /// a list of nodes reads the next sibling. + /// + /// A `nil` result means this node has no effect on what has to match first, + /// and `.some(nil)` means that no required first atom can be determined. + /// + /// Some of the cases below stop looking at children as soon as the answer is + /// known, and others don't look at them at all, so the end of the subtree is + /// found up front rather than left to the traversal to reach. private func _requiredAtomImpl( _ position: inout Int, options: inout MatchingOptions, @@ -20,7 +30,12 @@ extension DSLList { guard position < nodes.count else { return nil } - + + var endOfSubtree = position + skipNode(&endOfSubtree) + endOfSubtree += 1 + defer { position = endOfSubtree } + switch nodes[position] { case .atom(let atom): switch atom { @@ -36,26 +51,15 @@ extension DSLList { return atom } - // In a concatenation, the first definitive child provides the answer, - // and then we need to skip past (in some cases at least) the remaining - // concatenation elements. + // In a concatenation, the first definitive child provides the answer. case .concatenation(let count): - var result: DSLTree.Atom?? = nil - var i = 0 - while i < count { - i += 1 - position += 1 + position += 1 + for _ in 0..) throws { var list = pattern.program.list @@ -65,4 +66,56 @@ import Testing } } } + + /// The number of quantifications in `pattern` that auto-possessification + /// converts to possessive, and the total number of quantifications. + @available(macOS 9999, *) + func possessificationCounts( + in pattern: String + ) throws -> (possessive: Int, total: Int) { + var list = try Regex(pattern).program.list + list.autoPossessify() + var possessive = 0 + var total = 0 + for node in list.nodes { + guard case .quantification(_, let kind) = node else { continue } + total += 1 + if kind.isExplicit && kind.quantificationKind?.ast == .possessive { + possessive += 1 + } + } + return (possessive, total) + } + + /// Auto-possessification walks a flat list of nodes, so a case that stops + /// looking at a node's children has to leave the cursor past the whole + /// subtree anyway. Losing track of it possessifies the wrong quantification, + /// which drops matches, or stops the traversal short, which silently gives + /// up the optimization. + @available(macOS 9999, *) + @Test func partialAutoPossessify() throws { + // pattern, expected possessive count, expected quantification count + let expectations: [(String, Int, Int)] = [ + // Every quantification of a concatenation participates, not just the + // first one. + (#"a+b\d+c"#, 2, 2), + (#"x*a+b"#, 2, 2), + (#"\w+\s\d+a"#, 2, 2), + // `\s?` is excluded by the following `b`, but `b{1,3}` isn't + // possessifiable, because `\s?` is allowed to match nothing. + (#"(b{1,3}\s?)b"#, 1, 2), + // Same, one level down: only `\+*` is excluded by what follows it. + (#"(?i)(?:a*\+*)A{2}"#, 1, 3), + // Nothing is known about what has to match after `[ab]`, so `a*` can't + // be possessified at any level. + (#"(?:a*[ab])b"#, 0, 1), + (#"(?:(?:a*[ab])x?)b"#, 1, 2), + ] + for (pattern, possessive, total) in expectations { + let counts = try possessificationCounts(in: pattern) + #expect( + counts == (possessive, total), + "Unexpected possessification in '\(pattern)'") + } + } }