Split a glob class too large to list into its ranges [patch]

TextFilter.Test/ReadmeTests.cs

@@ -4,6 +4,7 @@ namespace TextFilter.Test;
 
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.IO;
 using System.Linq;
 using System.Reflection;
@@ -89,7 +90,10 @@ public void EveryMethodInTheReadmeApiReferenceExistsWithItsDocumentedDefaults()
 		foreach (Match row in rows)
 		{
 			string methodName = row.Groups["method"].Value;
-			MethodInfo[] overloads = [.. methods.Where(m => m.Name == methodName)];
+			// The binary-compatibility overloads are hidden from IntelliSense and deliberately carry no
+			// defaults, since defaults on both sets would make calls ambiguous. The README documents the
+			// visible overloads, so only those are held to its defaults.
+			MethodInfo[] overloads = [.. methods.Where(m => m.Name == methodName && !IsHiddenFromCallers(m))];
 			if (overloads.Length == 0)
 			{
 				problems.Add($"{methodName} does not exist");
@@ -115,6 +119,9 @@ public void EveryMethodInTheReadmeApiReferenceExistsWithItsDocumentedDefaults()
 		Assert.IsEmpty(problems, string.Join("; ", problems));
 	}
 
+	private static bool IsHiddenFromCallers(MethodInfo method) =>
+		method.GetCustomAttribute<EditorBrowsableAttribute>()?.State == EditorBrowsableState.Never;
+
 	[TestMethod]
 	public void ReadmeMatchOptionsSampleGivesTheDocumentedResults()
 	{

TextFilter.Test/TextFilterTests.cs

@@ -1031,14 +1031,72 @@
 	[DataRow("x[a^]", "x[a^]")]
 	[DataRow("x[^a", "x[^a")]
 	public void ExpandGlobClassesRewritesOnlyClassesDotNetGlobMisreads(string token, string expected) =>
-		Assert.AreEqual(expected, TextFilter.ExpandGlobClasses(token));
+		CollectionAssert.AreEqual(new[] { expected }, TextFilter.ExpandGlobClasses(token)?.ToArray());
 
 	[TestMethod]
 	[DataRow("[#-!]")]
-	[DataRow("[\u0001-\uffff_]")]
 	public void ExpandGlobClassesRejectsClassesThatCannotBeListed(string token) =>
 		Assert.IsNull(TextFilter.ExpandGlobClasses(token));
 
+	[TestMethod]
+	[DataRow("[가-힣a-z]*", new[] { "[가-힣]*", "[a-z]*" })]
+	[DataRow("x[一-龥0-9_]", new[] { "x[一-龥]", "x[0-9]", "x[_]" })]
+	[DataRow("[가-힣a-z][一-龥0-9]", new[] { "[가-힣][一-龥]", "[가-힣][0-9]", "[a-z][一-龥]", "[a-z][0-9]" })]
+	[DataRow("[가-힣z-a]", new[] { "[가-힣]" })]
+	public void ExpandGlobClassesSplitsAClassTooLargeToListIntoItsRanges(string token, string[] expected) =>
+		CollectionAssert.AreEqual(expected, TextFilter.ExpandGlobClasses(token)?.ToArray());
+
+	[TestMethod]
+	[DataRow("[\u0001-\uffff_]")]
+	[DataRow("[!가-힣a-z]")]
+	[DataRow("[A-힣]")]
+	public void ExpandGlobClassesListsAClassTooLargeToListThatCannotBeSplit(string token)
+	{
+		IReadOnlyList<string>? expanded = TextFilter.ExpandGlobClasses(token);
+		Assert.IsNotNull(expanded);
+		Assert.HasCount(1, expanded);
+		Assert.IsGreaterThan(4096, expanded[0].Length);
+	}
+
+	[TestMethod]
+	[DataRow("README.md", "[가-힣a-z]*", false)]
+	[DataRow("README.md", "[一-龥a-z]*", false)]
+	[DataRow("README.md", "[一-龥0-9]*", false)]
+	// Matching is case-sensitive, so "README.md" is not among the members and is not excluded.
+	[DataRow("README.md", "-[一-龥a-z]*", true)]
+	[DataRow("README.md", "-[一-龥A-Z]*", false)]
+	[DataRow("@", "[A-힣]", false)]
+	[DataRow("readme.md", "[가-힣a-z]*", true)]
+	[DataRow("한국.md", "[가-힣a-z]*", true)]
+	[DataRow("中文", "[一-龥a-z]*", true)]
+	[DataRow("7z", "[一-龥0-9]*", true)]
+	[DataRow("readme.md", "-[一-龥a-z]*", false)]
+	[DataRow("中文", "-[一-龥a-z]*", false)]
+	[DataRow("README.md", "+[가-힣a-z]*", false)]
+	[DataRow("한국", "+[가-힣a-z]*", true)]
+	[DataRow("A", "[A-힣]", true)]
+	[DataRow("힣", "[A-힣]", true)]
+	[DataRow("a\\b", "a[A-힣]b", true)]
+	[DataRow("README.md", "[!가-힣a-z]*", true)]
+	[DataRow("readme.md", "[!가-힣a-z]*", false)]
+	[DataRow("한국", "[!가-힣a-z]*", false)]
+	[DataRow("x_", "x[가-힣a-z_]", true)]
+	[DataRow("x-", "x[가-힣a-z_]", false)]
+	public void GlobClassTooLargeToListMatchesOnlyItsMembers(string text, string filter, bool expected) =>
+		Assert.AreEqual(expected, TextFilter.IsMatch(text, filter, TextFilterType.Glob, TextFilterMatchOptions.ByWholeString));
+
+	[TestMethod]
+	public void GlobClassTooLargeToListStillLeavesAHalfTypedTokenIgnored()
+	{
+		string[] items = ["README.md", "한국.md"];
+		string[] korean = ["한국.md"];
+		string[] readme = ["README.md"];
+		CollectionAssert.AreEqual(korean, TextFilter.Filter(items, "[가-힣a-z]*", TextFilterType.Glob).ToArray());
+		CollectionAssert.AreEqual(readme, TextFilter.Filter(items, "-[가-힣a-z]*", TextFilterType.Glob).ToArray());
+		CollectionAssert.AreEqual(items, TextFilter.Filter(items, "file[0-", TextFilterType.Glob).ToArray());
+		CollectionAssert.AreEqual(korean, TextFilter.Filter(items, "[가-힣a-z]* file[0-", TextFilterType.Glob).ToArray());
+	}
+
 	[TestMethod]
 	[DataRow("[0-9a-f]", "0 5 9 a c f", "g G - x _")]
 	[DataRow("[!0-9a-f]", "g - x _", "0 5 a c f")]
@@ -1144,7 +1202,7 @@
 	[DataRow("[]]]", "[]][]]")]
 	[DataRow("*[[][0-9]]", "*[[][0-9][]]")]
 	public void ExpandGlobClassesWritesALiteralBracketAfterAClassAsItsOwnClass(string token, string expected) =>
-		Assert.AreEqual(expected, TextFilter.ExpandGlobClasses(token));
+		CollectionAssert.AreEqual(new[] { expected }, TextFilter.ExpandGlobClasses(token)?.ToArray());
 
 	[TestMethod]
 	[DataRow("*[[][0-9]]", "items[3]", "items[3")]
@@ -1169,8 +1227,8 @@
 	public void FilterWithAClassFollowedByALiteralBracketKeepsAndExcludesTheRightItems()
 	{
 		List<string> items = ["items[3]", "items[3", "notes.txt"];
 		CollectionAssert.AreEqual(new List<string> { "items[3]" }, TextFilter.Filter(items, "*[[][0-9]]").ToList());
 		CollectionAssert.AreEqual(new List<string> { "items[3", "notes.txt" }, TextFilter.Filter(items, "-*[[][0-9]]").ToList());
 	}
 
 	[TestMethod]
@@ -1326,7 +1384,7 @@
 	{
 		List<string> result = [.. TextFilter.Filter(["readme.md", "src/a.cs", "docs/", "a b"], filter)];
 
 		Assert.AreEqual(1, result.Count);
 		Assert.AreEqual("docs/", result[0]);
 	}
 
@@ -1337,7 +1395,7 @@
 	{
 		List<string> result = [.. TextFilter.Filter(["readme.md", "src/a.cs", "docs/", "a b"], filter)];
 
 		Assert.AreEqual(3, result.Count);
 		Assert.AreEqual("readme.md", result[0]);
 		Assert.AreEqual("src/a.cs", result[1]);
 		Assert.AreEqual("a b", result[2]);
@@ -1348,7 +1406,7 @@
 	{
 		List<string> result = [.. TextFilter.Filter(["readme.md", "src/a.cs", "docs/"], "+**/")];
 
 		Assert.AreEqual(1, result.Count);
 		Assert.AreEqual("docs/", result[0]);
 	}
 
@@ -1559,11 +1617,11 @@
 	[TestMethod]
 	public void Pre17GenericFilterSignatureStillExists()
 	{
 		Assert.IsTrue(typeof(TextFilter).GetMethods().Any(m =>
 			m.Name == nameof(TextFilter.Filter)
 			&& m.IsGenericMethodDefinition
 			&& m.GetParameters().Length == 5
 			&& m.GetParameters()[4].ParameterType == typeof(TextFilterMatchOptions)));
 	}
 
 	[TestMethod]

TextFilter/TextFilter.cs

@@ -303,7 +303,7 @@
 
 			return ExcludedTokenPrefixes.Contains(prefix)
 				? TextFilterTokenType.Excluded
 				: RequiredTokenPrefixes.Contains(prefix)
 				? TextFilterTokenType.Required
 				: TextFilterTokenType.Optional;
 		})
@@ -334,7 +334,7 @@
 	/// matches nor rejects anything; as a required token it matches everything; and as an excluded
 	/// token it excludes nothing.
 	/// </remarks>
 	public static bool DoesMatchGlob(string text, string filter, TextFilterMatchOptions textFilterMatchOptions, TextFilterCaseSensitivity caseSensitivity = TextFilterCaseSensitivity.CaseSensitive)
 	{
 		Ensure.NotNull(text);
 		Ensure.NotNull(filter);
@@ -408,7 +408,7 @@
 	/// <param name="textTokens">The set of text tokens to match against.</param>
 	/// <param name="caseSensitivity">Whether the match distinguishes uppercase from lowercase.</param>
 	/// <returns><c>true</c> if any token matches the glob filter token; otherwise, <c>false</c>.</returns>
 	public static bool AnyTokenMatchesGlobFilter(string filterToken, HashSet<string> textTokens, TextFilterCaseSensitivity caseSensitivity = TextFilterCaseSensitivity.CaseSensitive)
 	{
 		Ensure.NotNull(filterToken);
 		Ensure.NotNull(textTokens);
@@ -425,7 +425,7 @@
 	/// <param name="textTokens">The set of text tokens to match against.</param>
 	/// <param name="caseSensitivity">Whether the match distinguishes uppercase from lowercase.</param>
 	/// <returns><c>true</c> if all tokens match the glob filter token; otherwise, <c>false</c>.</returns>
 	public static bool AllTokensMatchGlobFilter(string filterToken, HashSet<string> textTokens, TextFilterCaseSensitivity caseSensitivity = TextFilterCaseSensitivity.CaseSensitive)
 	{
 		Ensure.NotNull(filterToken);
 		Ensure.NotNull(textTokens);
@@ -505,18 +505,26 @@
 	// letter or digit range is left for DotNet.Glob to match natively, since it reads that form
 	// correctly and it may span far more characters than are worth listing. A class with no closing
 	// "]" is left alone and stays unparseable.
-	// Returns null for a class that cannot be written as a list: one whose ranges span more than
-	// MaxGlobClassMembers characters, or that has no members at all because every range is reversed.
+	// A class that would list more than MaxGlobClassMembers characters ("[가-힣a-z]") is costly to match
+	// as a list, so a class that is not negated is split instead: each letter or digit range in it is
+	// matched natively, the rest is listed, and the token becomes one alternative per piece, matching if
+	// any of them does. A negated class cannot be split that way, since it must miss every piece at once,
+	// so it is listed in full however long it is, as is a class whose split would give the token more than
+	// MaxGlobClassAlternatives alternatives.
+	// Returns the token's alternatives, or null for a class that has no members at all because every
+	// range is reversed.
 	private const int MaxGlobClassMembers = 4096;
 
-	internal static string? ExpandGlobClasses(string token)
+	private const int MaxGlobClassAlternatives = 16;
+
+	internal static IReadOnlyList<string>? ExpandGlobClasses(string token)
 	{
 		if (!token.Contains(']'))
 		{
-			return token;
+			return [token];
 		}
 
-		StringBuilder result = new(token.Length);
+		List<StringBuilder> results = [new(token.Length)];
 		int index = 0;
 		while (index < token.Length)
 		{
@@ -533,29 +541,66 @@
 			}
 
 			string content = token[contentStart..close];
-			string opening = negated ? "[!" : "[";
-			string? rewritten = !content.Contains('-') || IsNativeGlobRange(content)
-				? opening + content + "]"
-				: ToGlobCharacterList(content, negated);
+			List<string>? rewritten = RewriteGlobClass(content, negated, MaxGlobClassAlternatives / results.Count);
 			if (rewritten is null)
 			{
 				return null;
 			}
 
-			result.Append(token, index, open - index).Append(rewritten);
+			string literal = token[index..open];
+			results = [.. results.SelectMany(result => rewritten.Select(piece => new StringBuilder(result.ToString()).Append(literal).Append(piece)))];
 			index = close + 1;
 
 			// DotNet.Glob reads a "]" straight after a class as one more member of it, so "[a]]" would
 			// match "a" and not "a]". Each such "]" is written as the one-member class "[]]", which it
 			// reads as the literal the pattern means.
 			while (index < token.Length && token[index] == ']')
 			{
-				result.Append("[]]");
+				results.ForEach(result => result.Append("[]]"));
 				index++;
 			}
 		}
 
-		return result.Append(token, index, token.Length - index).ToString();
+		string tail = token[index..];
+		return [.. results.Select(result => result.Append(tail).ToString())];
+	}
+
+	// Returns the pieces a class is matched as, any of which may match, at most maxPieces of them; or
+	// null if the class has no members.
+	private static List<string>? RewriteGlobClass(string content, bool negated, int maxPieces)
+	{
+		if (!content.Contains('-') || IsNativeGlobRange(content))
+		{
+			return [(negated ? "[!" : "[") + content + "]"];
+		}
+
+		List<char> singles = [];
+		List<(char First, char Last)> ranges = [];
+		ReadGlobClassMembers(content, singles, ranges);
+
+		long memberCount = singles.Count + ranges.Sum(range => Math.Max(0L, range.Last - range.First + 1L));
+		if (!negated && memberCount > MaxGlobClassMembers)
+		{
+			// A reversed range has no members, so it adds no piece.
+			List<(char First, char Last)> listedRanges = [.. ranges.Where(range => range.First <= range.Last && !IsNativeGlobRange(range.First, range.Last))];
+			List<string> pieces = [.. ranges
+				.Where(range => range.First <= range.Last && IsNativeGlobRange(range.First, range.Last))
+				.Select(range => $"[{range.First}-{range.Last}]")
+				.Distinct(StringComparer.Ordinal)];
+
+			string? rest = ToGlobCharacterList(CollectGlobClassMembers(singles, listedRanges), negated: false);
+			if (rest is not null)
+			{
+				pieces.Add(rest);
+			}
+
+			if (pieces.Count <= maxPieces)
+			{
+				return pieces;
+			}
+		}
+
+		return ToGlobCharacterList(CollectGlobClassMembers(singles, ranges), negated) is string list ? [list] : null;
 	}
 
 	// Returns the index of the "]" closing the class opened at "open", or -1 if it is never closed.
@@ -577,47 +622,55 @@
 	// A range that spans "/" or "\" is listed instead, so the separator in it is masked like any other
 	// literal one; left native, it would never contain the masked separator in the text ("[A-z]").
 	private static bool IsNativeGlobRange(string content) =>
-		content.Length == 3 && content[1] == '-' && content[0] != ']'
-		&& ((char.IsLetter(content[0]) && char.IsLetter(content[2])) || (char.IsAsciiDigit(content[0]) && char.IsAsciiDigit(content[2])))
-		&& !SpansPathSeparator(content[0], content[2]);
+		content.Length == 3 && content[1] == '-' && content[0] != ']' && IsNativeGlobRange(content[0], content[2]);
+
+	private static bool IsNativeGlobRange(char first, char last) =>
+		((char.IsLetter(first) && char.IsLetter(last)) || (char.IsAsciiDigit(first) && char.IsAsciiDigit(last)))
+		&& !SpansPathSeparator(first, last);
 
 	private static bool SpansPathSeparator(char first, char last) =>
 		(first <= '/' && '/' <= last) || (first <= '\\' && '\\' <= last);
 
-	// Reads the class members with the usual glob rules (a leading "]" and a leading or trailing "-" are
-	// literal, "x-y" is an inclusive range) and writes them in an order DotNet.Glob reads as a plain list:
-	// "]" first, so it does not close the class, then "-", so no member pair looks like a range, and "!"
-	// never first, so it is not read as negation.
-	private static string? ToGlobCharacterList(string content, bool negated)
+	// Reads the class members with the usual glob rules: a leading "]" and a leading or trailing "-" are
+	// literal, and "x-y" is an inclusive range.
+	private static void ReadGlobClassMembers(string content, List<char> singles, List<(char First, char Last)> ranges)
 	{
-		SortedSet<char> members = [];
 		int i = 0;
 		while (i < content.Length)
 		{
-			char first = content[i];
 			// A leading "]" can start a range too: "[]-a]" is "]".."a", as in POSIX.
 			bool isRange = i + 2 < content.Length && content[i + 1] == '-';
 			if (!isRange)
 			{
-				members.Add(first);
+				singles.Add(content[i]);
 				i++;
 				continue;
 			}
 
-			char last = content[i + 2];
-			if (last - first + members.Count >= MaxGlobClassMembers)
-			{
-				return null;
-			}
+			ranges.Add((content[i], content[i + 2]));
+			i += 3;
+		}
+	}
 
+	private static SortedSet<char> CollectGlobClassMembers(List<char> singles, List<(char First, char Last)> ranges)
+	{
+		SortedSet<char> members = [.. singles];
+		foreach ((char first, char last) in ranges)
+		{
 			for (int member = first; member <= last; member++)
 			{
 				members.Add((char)member);
 			}
-
-			i += 3;
 		}
 
+		return members;
+	}
+
+	// Writes the members in an order DotNet.Glob reads as a plain list: "]" first, so it does not close
+	// the class, then "-", so no member pair looks like a range, and "!" never first, so it is not read
+	// as negation. Returns null if there are no members.
+	private static string? ToGlobCharacterList(SortedSet<char> members, bool negated)
+	{
 		if (members.Count == 0)
 		{
 			return null;
@@ -663,12 +716,14 @@
 		{
 			try
 			{
-				string? expandedToken = ExpandGlobClasses(filterToken);
+				IReadOnlyList<string>? expandedTokens = ExpandGlobClasses(filterToken);
 				// A token made only of "**/" segments expands to the empty string as well as to "*/".
 				// Glob.Parse throws on an empty pattern, which would mark the whole token unparseable and
 				// discard the valid expansions with it. An empty pattern could only match empty text, which
 				// never reaches the matcher, so drop it; a token with nothing left is still unparseable.
-				Glob[]? parsed = expandedToken is null ? null : [.. ExpandGlobStarSegments(MaskPathSeparators(expandedToken))
+				Glob[]? parsed = expandedTokens is null ? null : [.. expandedTokens
+					.SelectMany(expandedToken => ExpandGlobStarSegments(MaskPathSeparators(expandedToken)))
+					.Distinct(StringComparer.Ordinal)
 					.Where(pattern => pattern.Length != 0)
 					.Select(pattern => caseSensitivity is TextFilterCaseSensitivity.CaseInsensitive
 						? Glob.Parse(pattern, CaseInsensitiveGlobOptions)
@@ -712,7 +767,7 @@
 	/// <see cref="TextFilterMatchOptions.ByWholeString"/> for those.
 	/// </para>
 	/// </remarks>
 	public static bool DoesMatchRegex(string text, string filter, TextFilterMatchOptions textFilterMatchOptions, TextFilterCaseSensitivity caseSensitivity = TextFilterCaseSensitivity.CaseSensitive)
 	{
 		Ensure.NotNull(text);
 		Ensure.NotNull(filter);