[6.4] Fix autopossessification errors

Sources/_StringProcessing/Optimizations/AutoPossessification.swift

@@ -12,11 +12,21 @@
 internal import _RegexParser
 
 extension DSLList {
+  /// Finds the required first atom of the node at `position`. On return,
+  /// `position` is always at the last node of that node's subtree, regardless
+  /// of how early the search stopped, so callers can safely advance to the
+  /// next sibling.
   private func _requiredAtomImpl(
     _ position: inout Int,
     options: inout MatchingOptions,
     allowOptionsChanges: Bool
   ) -> DSLTree.Atom?? {
+    let start = position
+    defer {
+      position = start
+      skipNode(&position)
+    }
+
     guard position < nodes.count else {
       return nil
     }
@@ -112,7 +122,11 @@ extension DSLList {
     // A quantification that doesn't require its child to exist can still
     // allow a start-only match. (e.g. `/(foo)?^bar/`)
     case .quantification(let amount, _):
-      if amount.requiresAtLeastOne {
+      // A quantification with a maximum of zero never matches anything and is
+      // skipped entirely during code generation, so it has no effect here.
+      if amount.ast.bounds.1 == 0 {
+        return nil
+      } else if amount.requiresAtLeastOne {
         position += 1
         return _requiredAtomImpl(&position, options: &options, allowOptionsChanges: allowOptionsChanges)
       } else {
@@ -155,6 +169,12 @@ extension DSLList {
       default:
         var innerPosition = position
         _ = autoPossessifyNextQuantification(&innerPosition, options: &options)
+
+        // Move past the whole quantification, since the recursive call may
+        // have stopped early inside the quantified node.
+        position = quantPosition
+        skipNode(&position)
+        position += 1
         return nil
       }
 
@@ -348,7 +368,11 @@ extension DSLTree.Atom.CharacterClass {
     case .digit:
       switch other {
       case .whitespace, .horizontalWhitespace, .verticalWhitespace, .newlineSequence,
-          .notWord, .notDigit: true
+          .notDigit: true
+      case .notWord:
+        // Digits are a subset of word characters, unless word characters are
+        // restricted to ASCII while digits are not.
+        !(options.usesASCIIWord && !options.usesASCIIDigits)
       default: false
       }
     case .notDigit:

Tests/RegexTests/MatchTests.swift

@@ -3038,4 +3038,15 @@ extension RegexTests {
     XCTAssertNotNil("testS".wholeMatch(of: regex))
     XCTAssertNotNil("tesTs".wholeMatch(of: regex))
   }
+
+  func testAutoPossessificationSkippedNodes() throws {
+    firstMatchTest(#"(\w* {0})a"#, input: "a", match: "a")
+    firstMatchTest(#"((\w* {0})){1}()a"#, input: "a", match: "a")
+    firstMatchTest(#"(?:\w{1,}? *?)b"#, input: "bb", match: "bb")
+    firstMatchTest(#"(?i)]()*|A{,1}a{1}"#, input: "a", match: "a")
+    firstMatchTest(#"(b?W?)(?i)([B])"#, input: "b", match: "b")
+
+    // Digits aren't a subset of word characters when only \w is ASCII-only.
+    firstMatchTest(#"(?W)\d{,1}\W"#, input: "\u{661}", match: "\u{661}")
+  }
 }

Tests/RegexTests/OptimizationTests.swift

@@ -14,24 +14,24 @@ import Testing
 @testable import _RegexParser
 
 @Suite struct OptimizationTests {
-  @available(macOS 9999, *)
+  @available(macOS 26, *)
   @Test(arguments: [#/a/#, #/a+/#, #/(?:a+)/#, #/(?:a)+/#, #/(?m)a+/#, #/ab?c/#, #/(?:a+)+$/#, #/(?:(?:a+b)+b)/#])
   func requiredFirstAtom(pattern: Regex<Substring>) throws {
     let list = pattern.program.list
     let atom = list.requiredFirstAtom(allowOptionsChanges: true)
     #expect(atom?.literalCharacterValue == "a", "Missing first character atom in '\(pattern._literalPattern!)'")
   }
 
-  @available(macOS 9999, *)
+  @available(macOS 26, *)
   @Test(arguments: [#/a?/#, #/(?:a|b)/#, #/[a]/#, #/a?bc/#])
   func noRequiredFirstAtom(pattern: Regex<Substring>) throws {
     let list = pattern.program.list
     let atom = list.requiredFirstAtom(allowOptionsChanges: true)
     #expect(atom == nil, "Unexpected required first atom in '\(pattern._literalPattern!)'")
   }
 
-  @available(macOS 9999, *)
-  @Test(arguments: [#/a+b/#, #/a*b/#, #/\w+\s/#, #/(?:a+b|b+a)/#, #/\d+a/#, #/a+A/#])
+  @available(macOS 26, *)
+  @Test(arguments: [#/a+b/#, #/a*b/#, #/\w+\s/#, #/(?:a+b|b+a)/#, #/\d+a/#, #/a+A/#, #/(?:x|a+b)/#])
   func autoPossessify(pattern: Regex<Substring>) throws {
     var list = pattern.program.list
     list.autoPossessify()
@@ -46,11 +46,12 @@ import Testing
     }
   }
 
-  @available(macOS 9999, *)
+  @available(macOS 26, *)
   @Test(arguments: [
     #/a?/#, #/a+a/#, #/a+(?:b|c)/#, #/(?:a+|b+)/#, #/[a]/#, #/a?a/#,
     #/(?i)a+A/#, #/(?i:a+A)/#,  // case insensitivity when checking exclusion
     #/(?:(?:ab)+b)/#,           // single atom quantifications only
+    #/\w*(?:\s*)a/#,            // optional quantifier in between
   ])
   func noAutoPossessify(pattern: Regex<Substring>) throws {
     var list = pattern.program.list
@@ -65,4 +66,58 @@ import Testing
       }
     }
   }
+
+  @available(macOS 26, *)
+  @Test(arguments: [
+    #"(\w* {1})a"#, #"(\w* {2})a"#, #"(\w* {0} {1})a"#, #"(\w*)\s"#, #"((\w+)) b"#,
+    #"(\w{1,} +)b"#, #"(?:\w* {1})a"#, #"\d{,1}\W"#,
+  ])
+  func autoPossessifyFirstQuantification(pattern: String) throws {
+    // Valid possessification near skipped/zero-width nodes must still happen.
+    var list = try Regex(pattern).program.list
+    list.autoPossessify()
+    let first = list.nodes.first {
+      if case .quantification = $0 { return true } else { return false }
+    }
+    guard case .quantification(_, let kind)? = first else {
+      Issue.record("No quantification in '\(pattern)'")
+      return
+    }
+    #expect(kind.isExplicit && kind.quantificationKind?.ast == .possessive,
+            "Expected possessification in '\(pattern)'")
+  }
+
+  @available(macOS 26, *)
+  @Test(arguments: [#"(\w* {0})a"#, #"((\w* {0})){1}()a"#, #"(\w*)(?: {0})a"#])
+  func noAutoPossessifyAcrossSkippedNode(pattern: String) throws {
+    // The zero-width quantification doesn't hide the `a` that follows `\w*`.
+    var list = try Regex(pattern).program.list
+    list.autoPossessify()
+    let first = list.nodes.first {
+      if case .quantification = $0 { return true } else { return false }
+    }
+    guard case .quantification(_, let kind)? = first else {
+      Issue.record("No quantification in '\(pattern)'")
+      return
+    }
+    #expect(kind.quantificationKind?.ast != .possessive,
+            "Unexpected possessification in '\(pattern)'")
+  }
+
+  @available(macOS 26, *)
+  @Test func digitWordExclusionDependsOnOptions() throws {
+    func firstIsPossessive(_ regex: Regex<AnyRegexOutput>) -> Bool {
+      var list = regex.program.list
+      list.autoPossessify()
+      for case .quantification(_, let kind) in list.nodes {
+        return kind.quantificationKind?.ast == .possessive
+      }
+      return false
+    }
+    // Digits are always word characters, so `\d?` can't give up input `\W` needs.
+    #expect(firstIsPossessive(try Regex(#"\d?\W"#)))
+    #expect(firstIsPossessive(try Regex(#"\d?\W"#).asciiOnlyDigits().asciiOnlyWordCharacters()))
+    // Unless only `\w` is ASCII-only: then non-ASCII digits are `\W`.
+    #expect(!firstIsPossessive(try Regex(#"\d?\W"#).asciiOnlyWordCharacters()))
+  }
 }