diff --git a/benchmarks/F23.StringSimilarity.Benchmarks/Benchmarks.cs b/benchmarks/F23.StringSimilarity.Benchmarks/Benchmarks.cs index bc750f9..e23c614 100644 --- a/benchmarks/F23.StringSimilarity.Benchmarks/Benchmarks.cs +++ b/benchmarks/F23.StringSimilarity.Benchmarks/Benchmarks.cs @@ -1,4 +1,5 @@ using BenchmarkDotNet.Attributes; +using F23.StringSimilarity.Experimental; namespace F23.StringSimilarity.Benchmarks; @@ -89,6 +90,13 @@ public void RatcliffObershelp() _ = ratcliffObershelp.Distance("hello", "world"); } + [Benchmark] + public void Sift4() + { + var sift4 = new Sift4(); + _ = sift4.Distance("hello", "world"); + } + [Benchmark] public void SorensenDice() { @@ -103,6 +111,53 @@ public void WeightedLevenshtein() _ = weightedLevenshtein.Distance("hello", "world"); } +#if STATIC_METHODS + [Benchmark] + public void CosineStatic() => _ = F23.StringSimilarity.Cosine.GetDistance("hello", "world"); + + [Benchmark] + public void DamerauStatic() => _ = F23.StringSimilarity.Damerau.GetDistance("hello", "world"); + + [Benchmark] + public void JaccardStatic() => _ = F23.StringSimilarity.Jaccard.GetDistance("hello", "world"); + + [Benchmark] + public void JaroWinklerStatic() => _ = F23.StringSimilarity.JaroWinkler.GetDistance("hello", "world"); + + [Benchmark] + public void LevenshteinStatic() => _ = F23.StringSimilarity.Levenshtein.GetDistance("hello", "world"); + + [Benchmark] + public void LongestCommonSubsequenceStatic() => _ = F23.StringSimilarity.LongestCommonSubsequence.GetDistance("hello", "world"); + + [Benchmark] + public void MetricLCSStatic() => _ = F23.StringSimilarity.MetricLCS.GetDistance("hello", "world"); + + [Benchmark] + public void NGramStatic() => _ = F23.StringSimilarity.NGram.GetDistance("hello", "world"); + + [Benchmark] + public void NormalizedLevenshteinStatic() => _ = F23.StringSimilarity.NormalizedLevenshtein.GetDistance("hello", "world"); + + [Benchmark] + public void OptimalStringAlignmentStatic() => _ = F23.StringSimilarity.OptimalStringAlignment.GetDistance("hello", "world"); + + [Benchmark] + public void QGramStatic() => _ = F23.StringSimilarity.QGram.GetDistance("hello", "world"); + + [Benchmark] + public void RatcliffObershelpStatic() => _ = F23.StringSimilarity.RatcliffObershelp.GetDistance("hello", "world"); + + [Benchmark] + public void Sift4Static() => _ = F23.StringSimilarity.Experimental.Sift4.GetDistance("hello", "world"); + + [Benchmark] + public void SorensenDiceStatic() => _ = F23.StringSimilarity.SorensenDice.GetDistance("hello", "world"); + + [Benchmark] + public void WeightedLevenshteinStatic() => _ = F23.StringSimilarity.WeightedLevenshtein.GetDistance("hello", "world", new ExampleCharSub()); +#endif + private class ExampleCharSub : ICharacterSubstitution { public double Cost(char c1, char c2) diff --git a/benchmarks/F23.StringSimilarity.Benchmarks/F23.StringSimilarity.Benchmarks.csproj b/benchmarks/F23.StringSimilarity.Benchmarks/F23.StringSimilarity.Benchmarks.csproj index c838a74..ebb11e0 100644 --- a/benchmarks/F23.StringSimilarity.Benchmarks/F23.StringSimilarity.Benchmarks.csproj +++ b/benchmarks/F23.StringSimilarity.Benchmarks/F23.StringSimilarity.Benchmarks.csproj @@ -6,6 +6,10 @@ enable enable false + + $(DefineConstants);STATIC_METHODS diff --git a/src/F23.StringSimilarity/Cosine.cs b/src/F23.StringSimilarity/Cosine.cs index 324378f..6064020 100644 --- a/src/F23.StringSimilarity/Cosine.cs +++ b/src/F23.StringSimilarity/Cosine.cs @@ -58,6 +58,17 @@ public Cosine() { } /// The cosine similarity in the range [0, 1] /// If s1 or s2 is null. public double Similarity(string s1, string s2) + => GetSimilarity(s1, s2, k); + + /// + /// Compute the cosine similarity between strings. + /// + /// The first string to compare. + /// The second string to compare. + /// The length of the k-shingles (sequences of k characters) to compare. + /// The cosine similarity in the range [0, 1] + /// If s1 or s2 is null. + public static double GetSimilarity(string s1, string s2, int k = DEFAULT_K) { if (s1 == null) { @@ -79,8 +90,8 @@ public double Similarity(string s1, string s2) return 0; } - var profile1 = GetProfile(s1); - var profile2 = GetProfile(s2); + var profile1 = GetProfile(s1, k); + var profile2 = GetProfile(s2, k); return DotProduct(profile1, profile2) / (Norm(profile1) * Norm(profile2)); } @@ -134,15 +145,30 @@ private static double DotProduct(IDictionary profile1, /// 1.0 - the cosine similarity in the range [0, 1] /// If s1 or s2 is null. public double Distance(string s1, string s2) - => 1.0 - Similarity(s1, s2); - + => GetDistance(s1, s2, k); + /// - /// + /// Returns 1.0 - similarity. + /// + /// The first string to compare. + /// The second string to compare. + /// The length of the k-shingles (sequences of k characters) to compare. + /// 1.0 - the cosine similarity in the range [0, 1] + /// If s1 or s2 is null. + public static double GetDistance(string s1, string s2, int k = DEFAULT_K) + => 1.0 - GetSimilarity(s1, s2, k); + + /// + /// /// /// /// /// public double Similarity(IDictionary profile1, IDictionary profile2) + => GetSimilarity(profile1, profile2); + + /// + public static double GetSimilarity(IDictionary profile1, IDictionary profile2) => DotProduct(profile1, profile2) / (Norm(profile1) * Norm(profile2)); } diff --git a/src/F23.StringSimilarity/Damerau.cs b/src/F23.StringSimilarity/Damerau.cs index 1a349f2..e4c3718 100644 --- a/src/F23.StringSimilarity/Damerau.cs +++ b/src/F23.StringSimilarity/Damerau.cs @@ -54,7 +54,11 @@ public class Damerau : IMetricStringDistance, IMetricSpanDistance /// The computed distance. /// If s1 or s2 is null. public double Distance(string s1, string s2) - => Distance(s1.AsSpan(), s2.AsSpan()); + => GetDistance(s1, s2); + + /// + public static double GetDistance(string s1, string s2) + => GetDistance(s1.AsSpan(), s2.AsSpan()); /// /// Calculates the Damerau-Levenshtein distance between two sequences. @@ -74,6 +78,11 @@ public double Distance(string s1, string s2) /// Thrown if or is . public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable + => GetDistance(s1, s2); + + /// + public static double GetDistance(ReadOnlySpan s1, ReadOnlySpan s2) + where T : IEquatable { if (s1 == null) { diff --git a/src/F23.StringSimilarity/Experimental/Sift4.cs b/src/F23.StringSimilarity/Experimental/Sift4.cs index b085820..9d629f8 100644 --- a/src/F23.StringSimilarity/Experimental/Sift4.cs +++ b/src/F23.StringSimilarity/Experimental/Sift4.cs @@ -37,7 +37,10 @@ namespace F23.StringSimilarity.Experimental /// public class Sift4 : IStringDistance { - private const int DEFAULT_MAX_OFFSET = 10; + /// + /// The default maximum distance to search for character transposition. + /// + public const int DEFAULT_MAX_OFFSET = 10; /// /// Gets or sets the maximum distance to search for character transposition. @@ -77,6 +80,20 @@ internal Offset(int c1, int c2, bool trans) /// /// public double Distance(string s1, string s2) + => GetDistance(s1, s2, MaxOffset); + + /// + /// Sift4 - a general purpose string distance algorithm inspired by JaroWinkler + /// and Longest Common Subsequence. + /// Original JavaScript algorithm by siderite, java port by Nathan Fischer 2016. + /// https://siderite.dev/blog/super-fast-and-accurate-string-distance.html + /// https://blackdoor.github.io/blog/sift4-java/ + /// + /// + /// + /// The maximum distance to search for character transposition. + /// + public static double GetDistance(string s1, string s2, int maxOffset = DEFAULT_MAX_OFFSET) { if (string.IsNullOrEmpty(s1)) { @@ -169,7 +186,7 @@ public double Distance(string s1, string s2) // (they get incremented at the end of the loop) // so that we can have only one code block handling matches for (int i = 0; - i < MaxOffset && (c1 + i < l1 || c2 + i < l2); + i < maxOffset && (c1 + i < l1 || c2 + i < l2); i++) { if ((c1 + i < l1) && (s1[c1 + i] == s2[c2])) diff --git a/src/F23.StringSimilarity/Jaccard.cs b/src/F23.StringSimilarity/Jaccard.cs index 433de1d..48dad00 100644 --- a/src/F23.StringSimilarity/Jaccard.cs +++ b/src/F23.StringSimilarity/Jaccard.cs @@ -65,6 +65,17 @@ public Jaccard() { } /// The Jaccard index in the range [0, 1] /// If s1 or s2 is null. public double Similarity(string s1, string s2) + => GetSimilarity(s1, s2, k); + + /// + /// Compute jaccard index: |A inter B| / |A union B|. + /// + /// The first string to compare. + /// The second string to compare. + /// The length of the k-shingles (sequences of k characters) to compare. + /// The Jaccard index in the range [0, 1] + /// If s1 or s2 is null. + public static double GetSimilarity(string s1, string s2, int k = DEFAULT_K) { if (s1 == null) { @@ -81,8 +92,8 @@ public double Similarity(string s1, string s2) return 1; } - var profile1 = GetProfile(s1); - var profile2 = GetProfile(s2); + var profile1 = GetProfile(s1, k); + var profile2 = GetProfile(s2, k); // SSNET Specific: use LINQ for more optimal distinct count var unionCount = profile1.Keys.Concat(profile2.Keys).Distinct().Count(); @@ -102,6 +113,17 @@ public double Similarity(string s1, string s2) /// 1 - the Jaccard similarity. /// If s1 or s2 is null. public double Distance(string s1, string s2) - => 1.0 - Similarity(s1, s2); + => GetDistance(s1, s2, k); + + /// + /// Distance is computed as 1 - similarity. + /// + /// The first string to compare. + /// The second string to compare. + /// The length of the k-shingles (sequences of k characters) to compare. + /// 1 - the Jaccard similarity. + /// If s1 or s2 is null. + public static double GetDistance(string s1, string s2, int k = DEFAULT_K) + => 1.0 - GetSimilarity(s1, s2, k); } } diff --git a/src/F23.StringSimilarity/JaroWinkler.cs b/src/F23.StringSimilarity/JaroWinkler.cs index 835c0fe..4ef109e 100644 --- a/src/F23.StringSimilarity/JaroWinkler.cs +++ b/src/F23.StringSimilarity/JaroWinkler.cs @@ -78,7 +78,19 @@ public JaroWinkler(double threshold) /// The Jaro-Winkler similarity in the range [0, 1] /// If s1 or s2 is null. public double Similarity(string s1, string s2) - => Similarity(s1.AsSpan(), s2.AsSpan()); + => GetSimilarity(s1, s2, Threshold); + + /// + /// Compute Jaro-Winkler similarity. + /// + /// The first string to compare. + /// The second string to compare. + /// The threshold used for adding the Winkler bonus. Set to a + /// negative value to get the Jaro similarity. + /// The Jaro-Winkler similarity in the range [0, 1] + /// If s1 or s2 is null. + public static double GetSimilarity(string s1, string s2, double threshold = DEFAULT_THRESHOLD) + => GetSimilarity(s1.AsSpan(), s2.AsSpan(), threshold); /// /// Calculates the similarity between two sequences using the Jaro-Winkler distance metric. @@ -94,6 +106,21 @@ public double Similarity(string s1, string s2) /// Thrown if or is null. public double Similarity(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable + => GetSimilarity(s1, s2, Threshold); + + /// + /// Calculates the similarity between two sequences using the Jaro-Winkler distance metric. + /// + /// The type of elements in the sequences. Must implement . + /// The first sequence to compare. Cannot be null. + /// The second sequence to compare. Cannot be null. + /// The threshold used for adding the Winkler bonus. Set to a + /// negative value to get the Jaro similarity. + /// A value between 0 and 1 representing the similarity between the two sequences, where 1 indicates identical + /// sequences and 0 indicates no similarity. + /// Thrown if or is null. + public static double GetSimilarity(ReadOnlySpan s1, ReadOnlySpan s2, double threshold = DEFAULT_THRESHOLD) + where T : IEquatable { if (s1 == null) { @@ -120,7 +147,7 @@ public double Similarity(ReadOnlySpan s1, ReadOnlySpan s2) / THREE; double jw = j; - if (j > Threshold) + if (j > threshold) { jw = j + Math.Min(JW_COEF, 1.0 / mtp[THREE]) * mtp[2] * (1 - j); } @@ -135,7 +162,19 @@ public double Similarity(ReadOnlySpan s1, ReadOnlySpan s2) /// 1 - similarity /// If s1 or s2 is null. public double Distance(string s1, string s2) - => 1.0 - Similarity(s1, s2); + => GetDistance(s1, s2, Threshold); + + /// + /// Return 1 - similarity. + /// + /// The first string to compare. + /// The second string to compare. + /// The threshold used for adding the Winkler bonus. Set to a + /// negative value to get the Jaro distance. + /// 1 - similarity + /// If s1 or s2 is null. + public static double GetDistance(string s1, string s2, double threshold = DEFAULT_THRESHOLD) + => 1.0 - GetSimilarity(s1, s2, threshold); /// /// Calculates the distance between two sequences based on their similarity. @@ -149,7 +188,21 @@ public double Distance(string s1, string s2) /// 0.0 indicates identical sequences and 1.0 indicates completely dissimilar sequences. public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable - => 1.0 - Similarity(s1, s2); + => GetDistance(s1, s2, Threshold); + + /// + /// Calculates the distance between two sequences based on their similarity. + /// + /// The type of elements in the sequences. Must implement . + /// The first sequence to compare. + /// The second sequence to compare. + /// The threshold used for adding the Winkler bonus. Set to a + /// negative value to get the Jaro distance. + /// A double value representing the distance between the two sequences. The value ranges from 0.0 to 1.0, where + /// 0.0 indicates identical sequences and 1.0 indicates completely dissimilar sequences. + public static double GetDistance(ReadOnlySpan s1, ReadOnlySpan s2, double threshold = DEFAULT_THRESHOLD) + where T : IEquatable + => 1.0 - GetSimilarity(s1, s2, threshold); private static int[] Matches(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable diff --git a/src/F23.StringSimilarity/Levenshtein.cs b/src/F23.StringSimilarity/Levenshtein.cs index 74e801b..6bfb1c6 100644 --- a/src/F23.StringSimilarity/Levenshtein.cs +++ b/src/F23.StringSimilarity/Levenshtein.cs @@ -42,7 +42,10 @@ public class Levenshtein : IMetricStringDistance, IMetricSpanDistance /// The first string to compare. /// The second string to compare. /// The Levenshtein distance between strings - public double Distance(string s1, string s2) => Distance(s1, s2, int.MaxValue); + public double Distance(string s1, string s2) => GetDistance(s1, s2); + + /// + public static double GetDistance(string s1, string s2) => GetDistance(s1, s2, int.MaxValue); /// /// The Levenshtein distance, or edit distance, between two words is the @@ -74,7 +77,11 @@ public class Levenshtein : IMetricStringDistance, IMetricSpanDistance /// The Levenshtein distance between strings /// If s1 or s2 is null. public double Distance(string s1, string s2, int limit) - => Distance(s1.AsSpan(), s2.AsSpan(), limit); + => GetDistance(s1, s2, limit); + + /// + public static double GetDistance(string s1, string s2, int limit) + => GetDistance(s1.AsSpan(), s2.AsSpan(), limit); /// /// Calculates the distance between two sequences of elements. @@ -88,7 +95,12 @@ public double Distance(string s1, string s2, int limit) /// distance depends on the implementation of the comparison logic. public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable - => Distance(s1, s2, int.MaxValue); + => GetDistance(s1, s2); + + /// + public static double GetDistance(ReadOnlySpan s1, ReadOnlySpan s2) + where T : IEquatable + => GetDistance(s1, s2, int.MaxValue); /// /// Calculates the edit distance (Levenshtein distance) between two sequences, with an optional upper limit. @@ -107,6 +119,11 @@ public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) /// Thrown if or is null. public double Distance(ReadOnlySpan s1, ReadOnlySpan s2, int limit) where T : IEquatable + => GetDistance(s1, s2, limit); + + /// + public static double GetDistance(ReadOnlySpan s1, ReadOnlySpan s2, int limit) + where T : IEquatable { if (s1 == null) { diff --git a/src/F23.StringSimilarity/LongestCommonSubsequence.cs b/src/F23.StringSimilarity/LongestCommonSubsequence.cs index df0cfa3..b0fd03e 100644 --- a/src/F23.StringSimilarity/LongestCommonSubsequence.cs +++ b/src/F23.StringSimilarity/LongestCommonSubsequence.cs @@ -60,7 +60,11 @@ public class LongestCommonSubsequence : IStringDistance, ISpanDistance /// /// If s1 or s2 is null. public double Distance(string s1, string s2) - => Distance(s1.AsSpan(), s2.AsSpan()); + => GetDistance(s1, s2); + + /// + public static double GetDistance(string s1, string s2) + => GetDistance(s1.AsSpan(), s2.AsSpan()); /// /// Calculates the distance between two sequences based on their similarity. @@ -75,6 +79,11 @@ public double Distance(string s1, string s2) /// Thrown if or is . public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable + => GetDistance(s1, s2); + + /// + public static double GetDistance(ReadOnlySpan s1, ReadOnlySpan s2) + where T : IEquatable { if (s1 == null) { @@ -91,7 +100,7 @@ public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) return 0; } - return s1.Length + s2.Length - 2 * Length(s1, s2); + return s1.Length + s2.Length - 2 * GetLength(s1, s2); } /// @@ -103,9 +112,22 @@ public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) /// The length of LCS(s2, s2) /// If s1 or s2 is null. public int Length(string s1, string s2) - => Length(s1.AsSpan(), s2.AsSpan()); + => GetLength(s1, s2); - internal static int Length(ReadOnlySpan s1, ReadOnlySpan s2) + /// + public static int GetLength(string s1, string s2) + => GetLength(s1.AsSpan(), s2.AsSpan()); + + /// + /// Return the length of Longest Common Subsequence (LCS) between sequences s1 + /// and s2. + /// + /// The type of elements in the sequences. Must implement . + /// The first sequence to compare. + /// The second sequence to compare. + /// The length of LCS(s1, s2) + /// If s1 or s2 is null. + public static int GetLength(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable { if (s1 == null) diff --git a/src/F23.StringSimilarity/MetricLCS.cs b/src/F23.StringSimilarity/MetricLCS.cs index a3d2a6d..5ce9e2d 100644 --- a/src/F23.StringSimilarity/MetricLCS.cs +++ b/src/F23.StringSimilarity/MetricLCS.cs @@ -42,7 +42,11 @@ public class MetricLCS : IMetricStringDistance, INormalizedStringDistance, IMetr /// LCS distance metric /// If s1 or s2 is null. public double Distance(string s1, string s2) - => Distance(s1.AsSpan(), s2.AsSpan()); + => GetDistance(s1, s2); + + /// + public static double GetDistance(string s1, string s2) + => GetDistance(s1.AsSpan(), s2.AsSpan()); /// /// Calculates the normalized distance between two sequences based on their longest common subsequence. @@ -59,6 +63,11 @@ public double Distance(string s1, string s2) /// Thrown if or is . public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable + => GetDistance(s1, s2); + + /// + public static double GetDistance(ReadOnlySpan s1, ReadOnlySpan s2) + where T : IEquatable { if (s1 == null) { @@ -80,7 +89,7 @@ public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) if (m_len == 0) return 0.0; return 1.0 - - (1.0 * LongestCommonSubsequence.Length(s1, s2)) + - (1.0 * LongestCommonSubsequence.GetLength(s1, s2)) / m_len; } } diff --git a/src/F23.StringSimilarity/NGram.cs b/src/F23.StringSimilarity/NGram.cs index dfc4e8f..52b9fe9 100644 --- a/src/F23.StringSimilarity/NGram.cs +++ b/src/F23.StringSimilarity/NGram.cs @@ -44,7 +44,11 @@ namespace F23.StringSimilarity /// public class NGram : INormalizedStringDistance { - private const int DEFAULT_N = 2; + /// + /// The default size of the n-grams, used when no value of n is given. + /// + public const int DEFAULT_N = 2; + private readonly int n; /// @@ -72,6 +76,17 @@ public NGram(int n) /// The computed n-gram distance in the range [0, 1] /// If s0 or s1 is null. public double Distance(string s0, string s1) + => GetDistance(s0, s1, n); + + /// + /// Compute n-gram distance. + /// + /// The first string to compare. + /// The second string to compare. + /// The size of the n-grams to compare. + /// The computed n-gram distance in the range [0, 1] + /// If s0 or s1 is null. + public static double GetDistance(string s0, string s1, int n = DEFAULT_N) { if (s0 == null) { diff --git a/src/F23.StringSimilarity/NormalizedLevenshtein.cs b/src/F23.StringSimilarity/NormalizedLevenshtein.cs index dd223be..c0ee53f 100644 --- a/src/F23.StringSimilarity/NormalizedLevenshtein.cs +++ b/src/F23.StringSimilarity/NormalizedLevenshtein.cs @@ -35,8 +35,6 @@ namespace F23.StringSimilarity /// public class NormalizedLevenshtein : INormalizedStringDistance, INormalizedStringSimilarity, INormalizedSpanDistance, INormalizedSpanSimilarity { - private readonly Levenshtein l = new Levenshtein(); - /// /// Compute distance as Levenshtein(s1, s2) / max(|s1|, |s2|). /// @@ -45,7 +43,11 @@ public class NormalizedLevenshtein : INormalizedStringDistance, INormalizedStrin /// The computed distance in the range [0, 1] /// If s1 or s2 is null. public double Distance(string s1, string s2) - => Distance(s1.AsSpan(), s2.AsSpan()); + => GetDistance(s1, s2); + + /// + public static double GetDistance(string s1, string s2) + => GetDistance(s1.AsSpan(), s2.AsSpan()); /// /// Calculates the normalized distance between two sequences of elements. @@ -61,6 +63,11 @@ public double Distance(string s1, string s2) /// Thrown if or is null. public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable + => GetDistance(s1, s2); + + /// + public static double GetDistance(ReadOnlySpan s1, ReadOnlySpan s2) + where T : IEquatable { if (s1 == null) { @@ -84,7 +91,7 @@ public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) return 0.0; } - return l.Distance(s1, s2) / m_len; + return Levenshtein.GetDistance(s1, s2) / m_len; } /// @@ -95,7 +102,11 @@ public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) /// 1 - distance /// If s1 or s2 is null. public double Similarity(string s1, string s2) - => 1.0 - Distance(s1, s2); + => GetSimilarity(s1, s2); + + /// + public static double GetSimilarity(string s1, string s2) + => 1.0 - GetDistance(s1, s2); /// /// Calculates the similarity between two sequences based on their distance. @@ -109,6 +120,11 @@ public double Similarity(string s1, string s2) /// sequences and 0.0 indicates completely dissimilar sequences. public double Similarity(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable - => 1.0 - Distance(s1, s2); + => GetSimilarity(s1, s2); + + /// + public static double GetSimilarity(ReadOnlySpan s1, ReadOnlySpan s2) + where T : IEquatable + => 1.0 - GetDistance(s1, s2); } } diff --git a/src/F23.StringSimilarity/OptimalStringAlignment.cs b/src/F23.StringSimilarity/OptimalStringAlignment.cs index f404fc6..1460b6a 100644 --- a/src/F23.StringSimilarity/OptimalStringAlignment.cs +++ b/src/F23.StringSimilarity/OptimalStringAlignment.cs @@ -53,8 +53,12 @@ public sealed class OptimalStringAlignment : IStringDistance, ISpanDistance /// the OSA distance /// If s1 or s2 is null. public double Distance(string s1, string s2) - => Distance(s1.AsSpan(), s2.AsSpan()); - + => GetDistance(s1, s2); + + /// + public static double GetDistance(string s1, string s2) + => GetDistance(s1.AsSpan(), s2.AsSpan()); + /// /// Calculates the Damerau-Levenshtein distance between two sequences. /// @@ -70,6 +74,11 @@ public double Distance(string s1, string s2) /// Thrown if or is null. public double Distance(ReadOnlySpan s1, ReadOnlySpan s2) where T : IEquatable + => GetDistance(s1, s2); + + /// + public static double GetDistance(ReadOnlySpan s1, ReadOnlySpan s2) + where T : IEquatable { if (s1 == null) { diff --git a/src/F23.StringSimilarity/QGram.cs b/src/F23.StringSimilarity/QGram.cs index 42e1231..c0ba9bb 100644 --- a/src/F23.StringSimilarity/QGram.cs +++ b/src/F23.StringSimilarity/QGram.cs @@ -71,6 +71,18 @@ public QGram() { } /// The computed Q-gram distance. /// If s1 or s2 is null. public double Distance(string s1, string s2) + => GetDistance(s1, s2, k); + + /// + /// The distance between two strings is defined as the L1 norm of the + /// difference of their profiles (the number of occurence of each k-shingle). + /// + /// The first string to compare. + /// The second string to compare. + /// The length of the k-shingles (sequences of k characters) to compare. + /// The computed Q-gram distance. + /// If s1 or s2 is null. + public static double GetDistance(string s1, string s2, int k = DEFAULT_K) { if (s1 == null) { @@ -87,10 +99,10 @@ public double Distance(string s1, string s2) return 0; } - var profile1 = GetProfile(s1); - var profile2 = GetProfile(s2); + var profile1 = GetProfile(s1, k); + var profile2 = GetProfile(s2, k); - return Distance(profile1, profile2); + return GetDistance(profile1, profile2); } /// @@ -100,6 +112,10 @@ public double Distance(string s1, string s2) /// /// public double Distance(IDictionary profile1, IDictionary profile2) + => GetDistance(profile1, profile2); + + /// + public static double GetDistance(IDictionary profile1, IDictionary profile2) { var union = new HashSet(); union.UnionWith(profile1.Keys); diff --git a/src/F23.StringSimilarity/RatcliffObershelp.cs b/src/F23.StringSimilarity/RatcliffObershelp.cs index 4a59af4..c505e1a 100644 --- a/src/F23.StringSimilarity/RatcliffObershelp.cs +++ b/src/F23.StringSimilarity/RatcliffObershelp.cs @@ -29,6 +29,10 @@ public class RatcliffObershelp : INormalizedStringSimilarity, INormalizedStringD /// The RatcliffObershelp similarity in the range [0, 1] /// If s1 or s2 is null. public double Similarity(string s1, string s2) + => GetSimilarity(s1, s2); + + /// + public static double GetSimilarity(string s1, string s2) { if (s1 == null) { @@ -64,9 +68,11 @@ public double Similarity(string s1, string s2) /// 1 - similarity /// If s1 or s2 is null. public double Distance(string s1, string s2) - { - return 1.0d - Similarity(s1, s2); - } + => GetDistance(s1, s2); + + /// + public static double GetDistance(string s1, string s2) + => 1.0d - GetSimilarity(s1, s2); private static IList GetMatchList(ReadOnlySpan s1, ReadOnlySpan s2) { diff --git a/src/F23.StringSimilarity/ShingleBased.cs b/src/F23.StringSimilarity/ShingleBased.cs index f389bd6..84fd63e 100644 --- a/src/F23.StringSimilarity/ShingleBased.cs +++ b/src/F23.StringSimilarity/ShingleBased.cs @@ -33,7 +33,10 @@ namespace F23.StringSimilarity /// public abstract class ShingleBased { - private const int DEFAULT_K = 3; + /// + /// The default length of k-shingles (aka n-grams), used when no value of k is given. + /// + public const int DEFAULT_K = 3; /// /// Return k, the length of k-shingles (aka n-grams). @@ -75,7 +78,24 @@ protected ShingleBased() : this(DEFAULT_K) { } /// A dictionary where the keys are k-length substrings (shingles) extracted from the input string, and the /// values are the number of times each shingle appears. public Dictionary GetProfile(string s) + => GetProfile(s, k); + + /// + /// Generates a profile of k-length substrings (shingles) from the specified string, along with their frequency + /// of occurrence. + /// + /// The input string from which to generate the shingle profile. Cannot be null. + /// The length of the k-shingles (aka n-grams) to extract. + /// A dictionary where the keys are k-length substrings (shingles) extracted from the input string, and the + /// values are the number of times each shingle appears. + /// If k is less than or equal to 0. + public static Dictionary GetProfile(string s, int k) { + if (k <= 0) + { + throw new ArgumentOutOfRangeException(nameof(k), "k should be positive!"); + } + var shingles = new Dictionary(); var string_no_space = SPACE_REG.Replace(s, " "); diff --git a/src/F23.StringSimilarity/SorensenDice.cs b/src/F23.StringSimilarity/SorensenDice.cs index ebb9f08..15d759f 100644 --- a/src/F23.StringSimilarity/SorensenDice.cs +++ b/src/F23.StringSimilarity/SorensenDice.cs @@ -68,6 +68,17 @@ public SorensenDice() { } /// The computed Sorensen-Dice similarity. /// If s1 or s2 is null. public double Similarity(string s1, string s2) + => GetSimilarity(s1, s2, k); + + /// + /// Similarity is computed as 2 * |A inter B| / (|A| + |B|). + /// + /// The first string to compare. + /// The second string to compare. + /// The length of the k-shingles (sequences of k characters) to compare. + /// The computed Sorensen-Dice similarity. + /// If s1 or s2 is null. + public static double GetSimilarity(string s1, string s2, int k = DEFAULT_K) { if (s1 == null) { @@ -84,8 +95,8 @@ public double Similarity(string s1, string s2) return 1; } - var profile1 = GetProfile(s1); - var profile2 = GetProfile(s2); + var profile1 = GetProfile(s1, k); + var profile2 = GetProfile(s2, k); var union = new HashSet(); union.UnionWith(profile1.Keys); @@ -110,6 +121,17 @@ public double Similarity(string s1, string s2) /// 1.0 - the computed similarity /// If s1 or s2 is null. public double Distance(string s1, string s2) - => 1 - Similarity(s1, s2); + => GetDistance(s1, s2, k); + + /// + /// Returns 1 - similarity. + /// + /// The first string to compare. + /// The second string to compare. + /// The length of the k-shingles (sequences of k characters) to compare. + /// 1.0 - the computed similarity + /// If s1 or s2 is null. + public static double GetDistance(string s1, string s2, int k = DEFAULT_K) + => 1 - GetSimilarity(s1, s2, k); } } diff --git a/src/F23.StringSimilarity/WeightedLevenshtein.cs b/src/F23.StringSimilarity/WeightedLevenshtein.cs index 4db07f1..01af0a6 100644 --- a/src/F23.StringSimilarity/WeightedLevenshtein.cs +++ b/src/F23.StringSimilarity/WeightedLevenshtein.cs @@ -69,9 +69,7 @@ public WeightedLevenshtein(ICharacterSubstitution characterSubstitution, /// The second string to compare. /// The computed weighted Levenshtein distance. public double Distance(string s1, string s2) - { - return Distance(s1, s2, double.MaxValue); - } + => GetDistance(s1, s2, _characterSubstitution, _characterInsDel); /// /// Compute Levenshtein distance using provided weights for substitution. @@ -86,6 +84,27 @@ public double Distance(string s1, string s2) /// The computed weighted Levenshtein distance. /// If s1 or s2 is null. public double Distance(string s1, string s2, double limit) + => GetDistance(s1, s2, _characterSubstitution, _characterInsDel, limit); + + /// + /// Compute Levenshtein distance using provided weights for substitution. + /// + /// The first string to compare. + /// The second string to compare. + /// The strategy to determine character substitution weights. + /// The strategy to determine character insertion/deletion weights, + /// or null to use a weight of 1.0 for every insertion and deletion. + /// The maximum result to compute before stopping. This + /// means that the calculation can terminate early if you + /// only care about strings with a certain similarity. + /// Set this to Double.MaxValue if you want to run the + /// calculation to completion in every case. + /// The computed weighted Levenshtein distance. + /// If s1 or s2 is null. + public static double GetDistance(string s1, string s2, + ICharacterSubstitution characterSubstitution, + ICharacterInsDel characterInsDel = null, + double limit = double.MaxValue) { if (s1 == null) { @@ -123,13 +142,13 @@ public double Distance(string s1, string s2, double limit) v0[0] = 0; for (int i = 1; i < v0.Length; i++) { - v0[i] = v0[i - 1] + InsertionCost(s2[i - 1]); + v0[i] = v0[i - 1] + InsertionCost(characterInsDel, s2[i - 1]); } for (int i = 0; i < s1.Length; i++) { char s1i = s1[i]; - double deletionCost = DeletionCost(s1i); + double deletionCost = DeletionCost(characterInsDel, s1i); // calculate v1 (current row distances) from the previous row v0 // first element of v1 is A[i+1][0] @@ -147,10 +166,10 @@ public double Distance(string s1, string s2, double limit) if (s1i != s2j) { - cost = _characterSubstitution.Cost(s1i, s2j); + cost = characterSubstitution.Cost(s1i, s2j); } - double insertionCost = InsertionCost(s2j); + double insertionCost = InsertionCost(characterInsDel, s2j); v1[j + 1] = Math.Min( v1[j] + insertionCost, // Cost of insertion @@ -175,14 +194,14 @@ public double Distance(string s1, string s2, double limit) return v0[s2.Length]; } - private double InsertionCost(char c) + private static double InsertionCost(ICharacterInsDel characterInsDel, char c) { - return _characterInsDel?.InsertionCost(c) ?? 1.0; + return characterInsDel?.InsertionCost(c) ?? 1.0; } - private double DeletionCost(char c) + private static double DeletionCost(ICharacterInsDel characterInsDel, char c) { - return _characterInsDel?.DeletionCost(c) ?? 1.0; + return characterInsDel?.DeletionCost(c) ?? 1.0; } } } diff --git a/test/F23.StringSimilarity.Tests/StaticMethodsTest.cs b/test/F23.StringSimilarity.Tests/StaticMethodsTest.cs new file mode 100644 index 0000000..e8970f9 --- /dev/null +++ b/test/F23.StringSimilarity.Tests/StaticMethodsTest.cs @@ -0,0 +1,190 @@ +/* + * The MIT License + * + * Copyright 2016 feature[23] + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +using System; +using F23.StringSimilarity.Experimental; +using Xunit; + +namespace F23.StringSimilarity.Tests +{ + public class StaticMethodsTest + { + private const string S1 = "My string"; + private const string S2 = "My tsring"; + + [Fact] + public void TestCosine() + { + Assert.Equal(new Cosine().Similarity(S1, S2), Cosine.GetSimilarity(S1, S2)); + Assert.Equal(new Cosine().Distance(S1, S2), Cosine.GetDistance(S1, S2)); + Assert.Equal(new Cosine(2).Similarity(S1, S2), Cosine.GetSimilarity(S1, S2, k: 2)); + Assert.Equal(new Cosine(2).Distance(S1, S2), Cosine.GetDistance(S1, S2, k: 2)); + + var profile1 = ShingleBased.GetProfile(S1, ShingleBased.DEFAULT_K); + var profile2 = ShingleBased.GetProfile(S2, ShingleBased.DEFAULT_K); + + Assert.Equal(new Cosine().Similarity(profile1, profile2), Cosine.GetSimilarity(profile1, profile2)); + } + + [Fact] + public void TestDamerau() + { + Assert.Equal(new Damerau().Distance(S1, S2), Damerau.GetDistance(S1, S2)); + Assert.Equal(new Damerau().Distance(S1.AsSpan(), S2.AsSpan()), Damerau.GetDistance(S1.AsSpan(), S2.AsSpan())); + } + + [Fact] + public void TestJaccard() + { + Assert.Equal(new Jaccard().Similarity(S1, S2), Jaccard.GetSimilarity(S1, S2)); + Assert.Equal(new Jaccard().Distance(S1, S2), Jaccard.GetDistance(S1, S2)); + Assert.Equal(new Jaccard(2).Similarity(S1, S2), Jaccard.GetSimilarity(S1, S2, k: 2)); + Assert.Equal(new Jaccard(2).Distance(S1, S2), Jaccard.GetDistance(S1, S2, k: 2)); + } + + [Fact] + public void TestJaroWinkler() + { + Assert.Equal(new JaroWinkler().Similarity(S1, S2), JaroWinkler.GetSimilarity(S1, S2)); + Assert.Equal(new JaroWinkler().Distance(S1, S2), JaroWinkler.GetDistance(S1, S2)); + Assert.Equal(new JaroWinkler().Similarity(S1.AsSpan(), S2.AsSpan()), JaroWinkler.GetSimilarity(S1.AsSpan(), S2.AsSpan())); + Assert.Equal(new JaroWinkler().Distance(S1.AsSpan(), S2.AsSpan()), JaroWinkler.GetDistance(S1.AsSpan(), S2.AsSpan())); + + Assert.Equal(new JaroWinkler(0.9).Similarity(S1, S2), JaroWinkler.GetSimilarity(S1, S2, threshold: 0.9)); + Assert.Equal(new JaroWinkler(0.9).Distance(S1, S2), JaroWinkler.GetDistance(S1, S2, threshold: 0.9)); + Assert.Equal(new JaroWinkler(0.9).Similarity(S1.AsSpan(), S2.AsSpan()), JaroWinkler.GetSimilarity(S1.AsSpan(), S2.AsSpan(), threshold: 0.9)); + } + + [Fact] + public void TestLevenshtein() + { + Assert.Equal(new Levenshtein().Distance(S1, S2), Levenshtein.GetDistance(S1, S2)); + Assert.Equal(new Levenshtein().Distance(S1, S2, 1), Levenshtein.GetDistance(S1, S2, 1)); + Assert.Equal(new Levenshtein().Distance(S1.AsSpan(), S2.AsSpan()), Levenshtein.GetDistance(S1.AsSpan(), S2.AsSpan())); + Assert.Equal(new Levenshtein().Distance(S1.AsSpan(), S2.AsSpan(), 1), Levenshtein.GetDistance(S1.AsSpan(), S2.AsSpan(), 1)); + } + + [Fact] + public void TestLongestCommonSubsequence() + { + Assert.Equal(new LongestCommonSubsequence().Distance(S1, S2), LongestCommonSubsequence.GetDistance(S1, S2)); + Assert.Equal(new LongestCommonSubsequence().Distance(S1.AsSpan(), S2.AsSpan()), LongestCommonSubsequence.GetDistance(S1.AsSpan(), S2.AsSpan())); + Assert.Equal(new LongestCommonSubsequence().Length(S1, S2), LongestCommonSubsequence.GetLength(S1, S2)); + Assert.Equal(new LongestCommonSubsequence().Length(S1, S2), LongestCommonSubsequence.GetLength(S1.AsSpan(), S2.AsSpan())); + } + + [Fact] + public void TestMetricLCS() + { + Assert.Equal(new MetricLCS().Distance(S1, S2), MetricLCS.GetDistance(S1, S2)); + Assert.Equal(new MetricLCS().Distance(S1.AsSpan(), S2.AsSpan()), MetricLCS.GetDistance(S1.AsSpan(), S2.AsSpan())); + } + + [Fact] + public void TestNGram() + { + Assert.Equal(new NGram().Distance(S1, S2), NGram.GetDistance(S1, S2)); + Assert.Equal(new NGram(3).Distance(S1, S2), NGram.GetDistance(S1, S2, n: 3)); + } + + [Fact] + public void TestNormalizedLevenshtein() + { + Assert.Equal(new NormalizedLevenshtein().Distance(S1, S2), NormalizedLevenshtein.GetDistance(S1, S2)); + Assert.Equal(new NormalizedLevenshtein().Similarity(S1, S2), NormalizedLevenshtein.GetSimilarity(S1, S2)); + Assert.Equal(new NormalizedLevenshtein().Distance(S1.AsSpan(), S2.AsSpan()), NormalizedLevenshtein.GetDistance(S1.AsSpan(), S2.AsSpan())); + Assert.Equal(new NormalizedLevenshtein().Similarity(S1.AsSpan(), S2.AsSpan()), NormalizedLevenshtein.GetSimilarity(S1.AsSpan(), S2.AsSpan())); + } + + [Fact] + public void TestOptimalStringAlignment() + { + Assert.Equal(new OptimalStringAlignment().Distance(S1, S2), OptimalStringAlignment.GetDistance(S1, S2)); + Assert.Equal(new OptimalStringAlignment().Distance(S1.AsSpan(), S2.AsSpan()), OptimalStringAlignment.GetDistance(S1.AsSpan(), S2.AsSpan())); + } + + [Fact] + public void TestQGram() + { + Assert.Equal(new QGram().Distance(S1, S2), QGram.GetDistance(S1, S2)); + Assert.Equal(new QGram(2).Distance(S1, S2), QGram.GetDistance(S1, S2, k: 2)); + + var profile1 = ShingleBased.GetProfile(S1, ShingleBased.DEFAULT_K); + var profile2 = ShingleBased.GetProfile(S2, ShingleBased.DEFAULT_K); + + Assert.Equal(new QGram().Distance(profile1, profile2), QGram.GetDistance(profile1, profile2)); + } + + [Fact] + public void TestRatcliffObershelp() + { + Assert.Equal(new RatcliffObershelp().Similarity(S1, S2), RatcliffObershelp.GetSimilarity(S1, S2)); + Assert.Equal(new RatcliffObershelp().Distance(S1, S2), RatcliffObershelp.GetDistance(S1, S2)); + } + + [Fact] + public void TestSift4() + { + Assert.Equal(new Sift4().Distance(S1, S2), Sift4.GetDistance(S1, S2)); + Assert.Equal(new Sift4 { MaxOffset = 5 }.Distance(S1, S2), Sift4.GetDistance(S1, S2, maxOffset: 5)); + } + + [Fact] + public void TestSorensenDice() + { + Assert.Equal(new SorensenDice().Similarity(S1, S2), SorensenDice.GetSimilarity(S1, S2)); + Assert.Equal(new SorensenDice().Distance(S1, S2), SorensenDice.GetDistance(S1, S2)); + Assert.Equal(new SorensenDice(2).Similarity(S1, S2), SorensenDice.GetSimilarity(S1, S2, k: 2)); + Assert.Equal(new SorensenDice(2).Distance(S1, S2), SorensenDice.GetDistance(S1, S2, k: 2)); + } + + [Fact] + public void TestWeightedLevenshtein() + { + var charSub = new ExampleCharSub(); + var insDel = new ExampleInsDel(); + + Assert.Equal(new WeightedLevenshtein(charSub).Distance(S1, S2), + WeightedLevenshtein.GetDistance(S1, S2, charSub)); + Assert.Equal(new WeightedLevenshtein(charSub).Distance(S1, S2, 1.0), + WeightedLevenshtein.GetDistance(S1, S2, charSub, limit: 1.0)); + Assert.Equal(new WeightedLevenshtein(charSub, insDel).Distance(S1, S2), + WeightedLevenshtein.GetDistance(S1, S2, charSub, insDel)); + Assert.Equal(new WeightedLevenshtein(charSub, insDel).Distance(S1, S2, 1.0), + WeightedLevenshtein.GetDistance(S1, S2, charSub, insDel, 1.0)); + } + + private class ExampleCharSub : ICharacterSubstitution + { + public double Cost(char c1, char c2) => c1 == 't' && c2 == 'r' ? 0.5 : 1.0; + } + + private class ExampleInsDel : ICharacterInsDel + { + public double DeletionCost(char c) => c == 'i' ? 0.8 : 1.0; + + public double InsertionCost(char c) => c == 'i' ? 0.5 : 1.0; + } + } +}