diff --git a/Sources/_StringProcessing/Optimizations/AutoPossessification.swift b/Sources/_StringProcessing/Optimizations/AutoPossessification.swift index e2f266f05..e45ebb651 100644 --- a/Sources/_StringProcessing/Optimizations/AutoPossessification.swift +++ b/Sources/_StringProcessing/Optimizations/AutoPossessification.swift @@ -12,11 +12,21 @@ internal import _RegexParser extension DSLList { + /// Finds the required first atom of the node at `position`. On return, + /// `position` is always at the last node of that node's subtree, regardless + /// of how early the search stopped, so callers can safely advance to the + /// next sibling. private func _requiredAtomImpl( _ position: inout Int, options: inout MatchingOptions, allowOptionsChanges: Bool ) -> DSLTree.Atom?? { + let start = position + defer { + position = start + skipNode(&position) + } + guard position < nodes.count else { return nil } @@ -112,7 +122,11 @@ extension DSLList { // A quantification that doesn't require its child to exist can still // allow a start-only match. (e.g. `/(foo)?^bar/`) case .quantification(let amount, _): - if amount.requiresAtLeastOne { + // A quantification with a maximum of zero never matches anything and is + // skipped entirely during code generation, so it has no effect here. + if amount.ast.bounds.1 == 0 { + return nil + } else if amount.requiresAtLeastOne { position += 1 return _requiredAtomImpl(&position, options: &options, allowOptionsChanges: allowOptionsChanges) } else { @@ -155,6 +169,12 @@ extension DSLList { default: var innerPosition = position _ = autoPossessifyNextQuantification(&innerPosition, options: &options) + + // Move past the whole quantification, since the recursive call may + // have stopped early inside the quantified node. + position = quantPosition + skipNode(&position) + position += 1 return nil } @@ -348,7 +368,11 @@ extension DSLTree.Atom.CharacterClass { case .digit: switch other { case .whitespace, .horizontalWhitespace, .verticalWhitespace, .newlineSequence, - .notWord, .notDigit: true + .notDigit: true + case .notWord: + // Digits are a subset of word characters, unless word characters are + // restricted to ASCII while digits are not. + !(options.usesASCIIWord && !options.usesASCIIDigits) default: false } case .notDigit: diff --git a/Tests/RegexTests/MatchTests.swift b/Tests/RegexTests/MatchTests.swift index acf156d76..c8ad56073 100644 --- a/Tests/RegexTests/MatchTests.swift +++ b/Tests/RegexTests/MatchTests.swift @@ -3038,4 +3038,15 @@ extension RegexTests { XCTAssertNotNil("testS".wholeMatch(of: regex)) XCTAssertNotNil("tesTs".wholeMatch(of: regex)) } + + func testAutoPossessificationSkippedNodes() throws { + firstMatchTest(#"(\w* {0})a"#, input: "a", match: "a") + firstMatchTest(#"((\w* {0})){1}()a"#, input: "a", match: "a") + firstMatchTest(#"(?:\w{1,}? *?)b"#, input: "bb", match: "bb") + firstMatchTest(#"(?i)]()*|A{,1}a{1}"#, input: "a", match: "a") + firstMatchTest(#"(b?W?)(?i)([B])"#, input: "b", match: "b") + + // Digits aren't a subset of word characters when only \w is ASCII-only. + firstMatchTest(#"(?W)\d{,1}\W"#, input: "\u{661}", match: "\u{661}") + } } diff --git a/Tests/RegexTests/OptimizationTests.swift b/Tests/RegexTests/OptimizationTests.swift index d40c8c8ac..83b7e420c 100644 --- a/Tests/RegexTests/OptimizationTests.swift +++ b/Tests/RegexTests/OptimizationTests.swift @@ -14,7 +14,7 @@ import Testing @testable import _RegexParser @Suite struct OptimizationTests { - @available(macOS 9999, *) + @available(macOS 26, *) @Test(arguments: [#/a/#, #/a+/#, #/(?:a+)/#, #/(?:a)+/#, #/(?m)a+/#, #/ab?c/#, #/(?:a+)+$/#, #/(?:(?:a+b)+b)/#]) func requiredFirstAtom(pattern: Regex) throws { let list = pattern.program.list @@ -22,7 +22,7 @@ import Testing #expect(atom?.literalCharacterValue == "a", "Missing first character atom in '\(pattern._literalPattern!)'") } - @available(macOS 9999, *) + @available(macOS 26, *) @Test(arguments: [#/a?/#, #/(?:a|b)/#, #/[a]/#, #/a?bc/#]) func noRequiredFirstAtom(pattern: Regex) throws { let list = pattern.program.list @@ -30,8 +30,8 @@ import Testing #expect(atom == nil, "Unexpected required first atom in '\(pattern._literalPattern!)'") } - @available(macOS 9999, *) - @Test(arguments: [#/a+b/#, #/a*b/#, #/\w+\s/#, #/(?:a+b|b+a)/#, #/\d+a/#, #/a+A/#]) + @available(macOS 26, *) + @Test(arguments: [#/a+b/#, #/a*b/#, #/\w+\s/#, #/(?:a+b|b+a)/#, #/\d+a/#, #/a+A/#, #/(?:x|a+b)/#]) func autoPossessify(pattern: Regex) throws { var list = pattern.program.list list.autoPossessify() @@ -46,11 +46,12 @@ import Testing } } - @available(macOS 9999, *) + @available(macOS 26, *) @Test(arguments: [ #/a?/#, #/a+a/#, #/a+(?:b|c)/#, #/(?:a+|b+)/#, #/[a]/#, #/a?a/#, #/(?i)a+A/#, #/(?i:a+A)/#, // case insensitivity when checking exclusion #/(?:(?:ab)+b)/#, // single atom quantifications only + #/\w*(?:\s*)a/#, // optional quantifier in between ]) func noAutoPossessify(pattern: Regex) throws { var list = pattern.program.list @@ -65,4 +66,58 @@ import Testing } } } + + @available(macOS 26, *) + @Test(arguments: [ + #"(\w* {1})a"#, #"(\w* {2})a"#, #"(\w* {0} {1})a"#, #"(\w*)\s"#, #"((\w+)) b"#, + #"(\w{1,} +)b"#, #"(?:\w* {1})a"#, #"\d{,1}\W"#, + ]) + func autoPossessifyFirstQuantification(pattern: String) throws { + // Valid possessification near skipped/zero-width nodes must still happen. + var list = try Regex(pattern).program.list + list.autoPossessify() + let first = list.nodes.first { + if case .quantification = $0 { return true } else { return false } + } + guard case .quantification(_, let kind)? = first else { + Issue.record("No quantification in '\(pattern)'") + return + } + #expect(kind.isExplicit && kind.quantificationKind?.ast == .possessive, + "Expected possessification in '\(pattern)'") + } + + @available(macOS 26, *) + @Test(arguments: [#"(\w* {0})a"#, #"((\w* {0})){1}()a"#, #"(\w*)(?: {0})a"#]) + func noAutoPossessifyAcrossSkippedNode(pattern: String) throws { + // The zero-width quantification doesn't hide the `a` that follows `\w*`. + var list = try Regex(pattern).program.list + list.autoPossessify() + let first = list.nodes.first { + if case .quantification = $0 { return true } else { return false } + } + guard case .quantification(_, let kind)? = first else { + Issue.record("No quantification in '\(pattern)'") + return + } + #expect(kind.quantificationKind?.ast != .possessive, + "Unexpected possessification in '\(pattern)'") + } + + @available(macOS 26, *) + @Test func digitWordExclusionDependsOnOptions() throws { + func firstIsPossessive(_ regex: Regex) -> Bool { + var list = regex.program.list + list.autoPossessify() + for case .quantification(_, let kind) in list.nodes { + return kind.quantificationKind?.ast == .possessive + } + return false + } + // Digits are always word characters, so `\d?` can't give up input `\W` needs. + #expect(firstIsPossessive(try Regex(#"\d?\W"#))) + #expect(firstIsPossessive(try Regex(#"\d?\W"#).asciiOnlyDigits().asciiOnlyWordCharacters())) + // Unless only `\w` is ASCII-only: then non-ASCII digits are `\W`. + #expect(!firstIsPossessive(try Regex(#"\d?\W"#).asciiOnlyWordCharacters())) + } }