diff --git a/TextFilter.Test/ReadmeTests.cs b/TextFilter.Test/ReadmeTests.cs index 7ced909..17fc9c2 100644 --- a/TextFilter.Test/ReadmeTests.cs +++ b/TextFilter.Test/ReadmeTests.cs @@ -4,6 +4,7 @@ namespace TextFilter.Test; using System; using System.Collections.Generic; +using System.ComponentModel; using System.IO; using System.Linq; using System.Reflection; @@ -89,7 +90,10 @@ public void EveryMethodInTheReadmeApiReferenceExistsWithItsDocumentedDefaults() foreach (Match row in rows) { string methodName = row.Groups["method"].Value; - MethodInfo[] overloads = [.. methods.Where(m => m.Name == methodName)]; + // The binary-compatibility overloads are hidden from IntelliSense and deliberately carry no + // defaults, since defaults on both sets would make calls ambiguous. The README documents the + // visible overloads, so only those are held to its defaults. + MethodInfo[] overloads = [.. methods.Where(m => m.Name == methodName && !IsHiddenFromCallers(m))]; if (overloads.Length == 0) { problems.Add($"{methodName} does not exist"); @@ -115,6 +119,9 @@ public void EveryMethodInTheReadmeApiReferenceExistsWithItsDocumentedDefaults() Assert.IsEmpty(problems, string.Join("; ", problems)); } + private static bool IsHiddenFromCallers(MethodInfo method) => + method.GetCustomAttribute()?.State == EditorBrowsableState.Never; + [TestMethod] public void ReadmeMatchOptionsSampleGivesTheDocumentedResults() { diff --git a/TextFilter.Test/TextFilterTests.cs b/TextFilter.Test/TextFilterTests.cs index 886b9cd..2e918a9 100644 --- a/TextFilter.Test/TextFilterTests.cs +++ b/TextFilter.Test/TextFilterTests.cs @@ -1031,14 +1031,72 @@ public void GlobClassWithATrailingDashMatchesTheDashLiterally() [DataRow("x[a^]", "x[a^]")] [DataRow("x[^a", "x[^a")] public void ExpandGlobClassesRewritesOnlyClassesDotNetGlobMisreads(string token, string expected) => - Assert.AreEqual(expected, TextFilter.ExpandGlobClasses(token)); + CollectionAssert.AreEqual(new[] { expected }, TextFilter.ExpandGlobClasses(token)?.ToArray()); [TestMethod] [DataRow("[#-!]")] - [DataRow("[\u0001-\uffff_]")] public void ExpandGlobClassesRejectsClassesThatCannotBeListed(string token) => Assert.IsNull(TextFilter.ExpandGlobClasses(token)); + [TestMethod] + [DataRow("[가-힣a-z]*", new[] { "[가-힣]*", "[a-z]*" })] + [DataRow("x[一-龥0-9_]", new[] { "x[一-龥]", "x[0-9]", "x[_]" })] + [DataRow("[가-힣a-z][一-龥0-9]", new[] { "[가-힣][一-龥]", "[가-힣][0-9]", "[a-z][一-龥]", "[a-z][0-9]" })] + [DataRow("[가-힣z-a]", new[] { "[가-힣]" })] + public void ExpandGlobClassesSplitsAClassTooLargeToListIntoItsRanges(string token, string[] expected) => + CollectionAssert.AreEqual(expected, TextFilter.ExpandGlobClasses(token)?.ToArray()); + + [TestMethod] + [DataRow("[\u0001-\uffff_]")] + [DataRow("[!가-힣a-z]")] + [DataRow("[A-힣]")] + public void ExpandGlobClassesListsAClassTooLargeToListThatCannotBeSplit(string token) + { + IReadOnlyList? expanded = TextFilter.ExpandGlobClasses(token); + Assert.IsNotNull(expanded); + Assert.HasCount(1, expanded); + Assert.IsGreaterThan(4096, expanded[0].Length); + } + + [TestMethod] + [DataRow("README.md", "[가-힣a-z]*", false)] + [DataRow("README.md", "[一-龥a-z]*", false)] + [DataRow("README.md", "[一-龥0-9]*", false)] + // Matching is case-sensitive, so "README.md" is not among the members and is not excluded. + [DataRow("README.md", "-[一-龥a-z]*", true)] + [DataRow("README.md", "-[一-龥A-Z]*", false)] + [DataRow("@", "[A-힣]", false)] + [DataRow("readme.md", "[가-힣a-z]*", true)] + [DataRow("한국.md", "[가-힣a-z]*", true)] + [DataRow("中文", "[一-龥a-z]*", true)] + [DataRow("7z", "[一-龥0-9]*", true)] + [DataRow("readme.md", "-[一-龥a-z]*", false)] + [DataRow("中文", "-[一-龥a-z]*", false)] + [DataRow("README.md", "+[가-힣a-z]*", false)] + [DataRow("한국", "+[가-힣a-z]*", true)] + [DataRow("A", "[A-힣]", true)] + [DataRow("힣", "[A-힣]", true)] + [DataRow("a\\b", "a[A-힣]b", true)] + [DataRow("README.md", "[!가-힣a-z]*", true)] + [DataRow("readme.md", "[!가-힣a-z]*", false)] + [DataRow("한국", "[!가-힣a-z]*", false)] + [DataRow("x_", "x[가-힣a-z_]", true)] + [DataRow("x-", "x[가-힣a-z_]", false)] + public void GlobClassTooLargeToListMatchesOnlyItsMembers(string text, string filter, bool expected) => + Assert.AreEqual(expected, TextFilter.IsMatch(text, filter, TextFilterType.Glob, TextFilterMatchOptions.ByWholeString)); + + [TestMethod] + public void GlobClassTooLargeToListStillLeavesAHalfTypedTokenIgnored() + { + string[] items = ["README.md", "한국.md"]; + string[] korean = ["한국.md"]; + string[] readme = ["README.md"]; + CollectionAssert.AreEqual(korean, TextFilter.Filter(items, "[가-힣a-z]*", TextFilterType.Glob).ToArray()); + CollectionAssert.AreEqual(readme, TextFilter.Filter(items, "-[가-힣a-z]*", TextFilterType.Glob).ToArray()); + CollectionAssert.AreEqual(items, TextFilter.Filter(items, "file[0-", TextFilterType.Glob).ToArray()); + CollectionAssert.AreEqual(korean, TextFilter.Filter(items, "[가-힣a-z]* file[0-", TextFilterType.Glob).ToArray()); + } + [TestMethod] [DataRow("[0-9a-f]", "0 5 9 a c f", "g G - x _")] [DataRow("[!0-9a-f]", "g - x _", "0 5 a c f")] @@ -1144,7 +1202,7 @@ public void GlobClassRangeSpanningAPathSeparatorMatchesIt(string pattern, string [DataRow("[]]]", "[]][]]")] [DataRow("*[[][0-9]]", "*[[][0-9][]]")] public void ExpandGlobClassesWritesALiteralBracketAfterAClassAsItsOwnClass(string token, string expected) => - Assert.AreEqual(expected, TextFilter.ExpandGlobClasses(token)); + CollectionAssert.AreEqual(new[] { expected }, TextFilter.ExpandGlobClasses(token)?.ToArray()); [TestMethod] [DataRow("*[[][0-9]]", "items[3]", "items[3")] diff --git a/TextFilter/TextFilter.cs b/TextFilter/TextFilter.cs index 16701d8..ccd7525 100644 --- a/TextFilter/TextFilter.cs +++ b/TextFilter/TextFilter.cs @@ -505,18 +505,26 @@ internal static IReadOnlyList ExpandGlobStarSegments(string maskedToken) // letter or digit range is left for DotNet.Glob to match natively, since it reads that form // correctly and it may span far more characters than are worth listing. A class with no closing // "]" is left alone and stays unparseable. - // Returns null for a class that cannot be written as a list: one whose ranges span more than - // MaxGlobClassMembers characters, or that has no members at all because every range is reversed. + // A class that would list more than MaxGlobClassMembers characters ("[가-힣a-z]") is costly to match + // as a list, so a class that is not negated is split instead: each letter or digit range in it is + // matched natively, the rest is listed, and the token becomes one alternative per piece, matching if + // any of them does. A negated class cannot be split that way, since it must miss every piece at once, + // so it is listed in full however long it is, as is a class whose split would give the token more than + // MaxGlobClassAlternatives alternatives. + // Returns the token's alternatives, or null for a class that has no members at all because every + // range is reversed. private const int MaxGlobClassMembers = 4096; - internal static string? ExpandGlobClasses(string token) + private const int MaxGlobClassAlternatives = 16; + + internal static IReadOnlyList? ExpandGlobClasses(string token) { if (!token.Contains(']')) { - return token; + return [token]; } - StringBuilder result = new(token.Length); + List results = [new(token.Length)]; int index = 0; while (index < token.Length) { @@ -533,16 +541,14 @@ internal static IReadOnlyList ExpandGlobStarSegments(string maskedToken) } string content = token[contentStart..close]; - string opening = negated ? "[!" : "["; - string? rewritten = !content.Contains('-') || IsNativeGlobRange(content) - ? opening + content + "]" - : ToGlobCharacterList(content, negated); + List? rewritten = RewriteGlobClass(content, negated, MaxGlobClassAlternatives / results.Count); if (rewritten is null) { return null; } - result.Append(token, index, open - index).Append(rewritten); + string literal = token[index..open]; + results = [.. results.SelectMany(result => rewritten.Select(piece => new StringBuilder(result.ToString()).Append(literal).Append(piece)))]; index = close + 1; // DotNet.Glob reads a "]" straight after a class as one more member of it, so "[a]]" would @@ -550,12 +556,51 @@ internal static IReadOnlyList ExpandGlobStarSegments(string maskedToken) // reads as the literal the pattern means. while (index < token.Length && token[index] == ']') { - result.Append("[]]"); + results.ForEach(result => result.Append("[]]")); index++; } } - return result.Append(token, index, token.Length - index).ToString(); + string tail = token[index..]; + return [.. results.Select(result => result.Append(tail).ToString())]; + } + + // Returns the pieces a class is matched as, any of which may match, at most maxPieces of them; or + // null if the class has no members. + private static List? RewriteGlobClass(string content, bool negated, int maxPieces) + { + if (!content.Contains('-') || IsNativeGlobRange(content)) + { + return [(negated ? "[!" : "[") + content + "]"]; + } + + List singles = []; + List<(char First, char Last)> ranges = []; + ReadGlobClassMembers(content, singles, ranges); + + long memberCount = singles.Count + ranges.Sum(range => Math.Max(0L, range.Last - range.First + 1L)); + if (!negated && memberCount > MaxGlobClassMembers) + { + // A reversed range has no members, so it adds no piece. + List<(char First, char Last)> listedRanges = [.. ranges.Where(range => range.First <= range.Last && !IsNativeGlobRange(range.First, range.Last))]; + List pieces = [.. ranges + .Where(range => range.First <= range.Last && IsNativeGlobRange(range.First, range.Last)) + .Select(range => $"[{range.First}-{range.Last}]") + .Distinct(StringComparer.Ordinal)]; + + string? rest = ToGlobCharacterList(CollectGlobClassMembers(singles, listedRanges), negated: false); + if (rest is not null) + { + pieces.Add(rest); + } + + if (pieces.Count <= maxPieces) + { + return pieces; + } + } + + return ToGlobCharacterList(CollectGlobClassMembers(singles, ranges), negated) is string list ? [list] : null; } // Returns the index of the "]" closing the class opened at "open", or -1 if it is never closed. @@ -577,47 +622,55 @@ private static int FindGlobClassEnd(string token, int open, out bool negated, ou // A range that spans "/" or "\" is listed instead, so the separator in it is masked like any other // literal one; left native, it would never contain the masked separator in the text ("[A-z]"). private static bool IsNativeGlobRange(string content) => - content.Length == 3 && content[1] == '-' && content[0] != ']' - && ((char.IsLetter(content[0]) && char.IsLetter(content[2])) || (char.IsAsciiDigit(content[0]) && char.IsAsciiDigit(content[2]))) - && !SpansPathSeparator(content[0], content[2]); + content.Length == 3 && content[1] == '-' && content[0] != ']' && IsNativeGlobRange(content[0], content[2]); + + private static bool IsNativeGlobRange(char first, char last) => + ((char.IsLetter(first) && char.IsLetter(last)) || (char.IsAsciiDigit(first) && char.IsAsciiDigit(last))) + && !SpansPathSeparator(first, last); private static bool SpansPathSeparator(char first, char last) => (first <= '/' && '/' <= last) || (first <= '\\' && '\\' <= last); - // Reads the class members with the usual glob rules (a leading "]" and a leading or trailing "-" are - // literal, "x-y" is an inclusive range) and writes them in an order DotNet.Glob reads as a plain list: - // "]" first, so it does not close the class, then "-", so no member pair looks like a range, and "!" - // never first, so it is not read as negation. - private static string? ToGlobCharacterList(string content, bool negated) + // Reads the class members with the usual glob rules: a leading "]" and a leading or trailing "-" are + // literal, and "x-y" is an inclusive range. + private static void ReadGlobClassMembers(string content, List singles, List<(char First, char Last)> ranges) { - SortedSet members = []; int i = 0; while (i < content.Length) { - char first = content[i]; // A leading "]" can start a range too: "[]-a]" is "]".."a", as in POSIX. bool isRange = i + 2 < content.Length && content[i + 1] == '-'; if (!isRange) { - members.Add(first); + singles.Add(content[i]); i++; continue; } - char last = content[i + 2]; - if (last - first + members.Count >= MaxGlobClassMembers) - { - return null; - } + ranges.Add((content[i], content[i + 2])); + i += 3; + } + } + private static SortedSet CollectGlobClassMembers(List singles, List<(char First, char Last)> ranges) + { + SortedSet members = [.. singles]; + foreach ((char first, char last) in ranges) + { for (int member = first; member <= last; member++) { members.Add((char)member); } - - i += 3; } + return members; + } + + // Writes the members in an order DotNet.Glob reads as a plain list: "]" first, so it does not close + // the class, then "-", so no member pair looks like a range, and "!" never first, so it is not read + // as negation. Returns null if there are no members. + private static string? ToGlobCharacterList(SortedSet members, bool negated) + { if (members.Count == 0) { return null; @@ -663,12 +716,14 @@ private static bool SpansPathSeparator(char first, char last) => { try { - string? expandedToken = ExpandGlobClasses(filterToken); + IReadOnlyList? expandedTokens = ExpandGlobClasses(filterToken); // A token made only of "**/" segments expands to the empty string as well as to "*/". // Glob.Parse throws on an empty pattern, which would mark the whole token unparseable and // discard the valid expansions with it. An empty pattern could only match empty text, which // never reaches the matcher, so drop it; a token with nothing left is still unparseable. - Glob[]? parsed = expandedToken is null ? null : [.. ExpandGlobStarSegments(MaskPathSeparators(expandedToken)) + Glob[]? parsed = expandedTokens is null ? null : [.. expandedTokens + .SelectMany(expandedToken => ExpandGlobStarSegments(MaskPathSeparators(expandedToken))) + .Distinct(StringComparer.Ordinal) .Where(pattern => pattern.Length != 0) .Select(pattern => caseSensitivity is TextFilterCaseSensitivity.CaseInsensitive ? Glob.Parse(pattern, CaseInsensitiveGlobOptions)