Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions benchmarks/F23.StringSimilarity.Benchmarks/Benchmarks.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using BenchmarkDotNet.Attributes;
using F23.StringSimilarity.Experimental;

namespace F23.StringSimilarity.Benchmarks;

Expand Down Expand Up @@ -89,6 +90,13 @@ public void RatcliffObershelp()
_ = ratcliffObershelp.Distance("hello", "world");
}

[Benchmark]
public void Sift4()
{
var sift4 = new Sift4();
_ = sift4.Distance("hello", "world");
}

[Benchmark]
public void SorensenDice()
{
Expand All @@ -103,6 +111,53 @@ public void WeightedLevenshtein()
_ = weightedLevenshtein.Distance("hello", "world");
}

#if STATIC_METHODS
[Benchmark]
public void CosineStatic() => _ = F23.StringSimilarity.Cosine.GetDistance("hello", "world");

[Benchmark]
public void DamerauStatic() => _ = F23.StringSimilarity.Damerau.GetDistance("hello", "world");

[Benchmark]
public void JaccardStatic() => _ = F23.StringSimilarity.Jaccard.GetDistance("hello", "world");

[Benchmark]
public void JaroWinklerStatic() => _ = F23.StringSimilarity.JaroWinkler.GetDistance("hello", "world");

[Benchmark]
public void LevenshteinStatic() => _ = F23.StringSimilarity.Levenshtein.GetDistance("hello", "world");

[Benchmark]
public void LongestCommonSubsequenceStatic() => _ = F23.StringSimilarity.LongestCommonSubsequence.GetDistance("hello", "world");

[Benchmark]
public void MetricLCSStatic() => _ = F23.StringSimilarity.MetricLCS.GetDistance("hello", "world");

[Benchmark]
public void NGramStatic() => _ = F23.StringSimilarity.NGram.GetDistance("hello", "world");

[Benchmark]
public void NormalizedLevenshteinStatic() => _ = F23.StringSimilarity.NormalizedLevenshtein.GetDistance("hello", "world");

[Benchmark]
public void OptimalStringAlignmentStatic() => _ = F23.StringSimilarity.OptimalStringAlignment.GetDistance("hello", "world");

[Benchmark]
public void QGramStatic() => _ = F23.StringSimilarity.QGram.GetDistance("hello", "world");

[Benchmark]
public void RatcliffObershelpStatic() => _ = F23.StringSimilarity.RatcliffObershelp.GetDistance("hello", "world");

[Benchmark]
public void Sift4Static() => _ = F23.StringSimilarity.Experimental.Sift4.GetDistance("hello", "world");

[Benchmark]
public void SorensenDiceStatic() => _ = F23.StringSimilarity.SorensenDice.GetDistance("hello", "world");

[Benchmark]
public void WeightedLevenshteinStatic() => _ = F23.StringSimilarity.WeightedLevenshtein.GetDistance("hello", "world", new ExampleCharSub());
#endif

private class ExampleCharSub : ICharacterSubstitution
{
public double Cost(char c1, char c2)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<!-- The static methods are only available in F23.StringSimilarity 8.0.0 and later,
so the benchmarks that use them are conditionally compiled. Remove this constant
when benchmarking against an older version of the library. -->
<DefineConstants>$(DefineConstants);STATIC_METHODS</DefineConstants>
</PropertyGroup>

<ItemGroup>
Expand Down
36 changes: 31 additions & 5 deletions src/F23.StringSimilarity/Cosine.cs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,17 @@ public Cosine() { }
/// <returns>The cosine similarity in the range [0, 1]</returns>
/// <exception cref="T:System.ArgumentNullException">If s1 or s2 is null.</exception>
public double Similarity(string s1, string s2)
=> GetSimilarity(s1, s2, k);

/// <summary>
/// Compute the cosine similarity between strings.
/// </summary>
/// <param name="s1">The first string to compare.</param>
/// <param name="s2">The second string to compare.</param>
/// <param name="k">The length of the k-shingles (sequences of k characters) to compare.</param>
/// <returns>The cosine similarity in the range [0, 1]</returns>
/// <exception cref="T:System.ArgumentNullException">If s1 or s2 is null.</exception>
public static double GetSimilarity(string s1, string s2, int k = DEFAULT_K)
{
if (s1 == null)
{
Expand All @@ -79,8 +90,8 @@ public double Similarity(string s1, string s2)
return 0;
}

var profile1 = GetProfile(s1);
var profile2 = GetProfile(s2);
var profile1 = GetProfile(s1, k);
var profile2 = GetProfile(s2, k);

return DotProduct(profile1, profile2) / (Norm(profile1) * Norm(profile2));
}
Expand Down Expand Up @@ -134,15 +145,30 @@ private static double DotProduct(IDictionary<string, int> profile1,
/// <returns>1.0 - the cosine similarity in the range [0, 1]</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public double Distance(string s1, string s2)
=> 1.0 - Similarity(s1, s2);
=> GetDistance(s1, s2, k);

/// <summary>
///
/// Returns 1.0 - similarity.
/// </summary>
/// <param name="s1">The first string to compare.</param>
/// <param name="s2">The second string to compare.</param>
/// <param name="k">The length of the k-shingles (sequences of k characters) to compare.</param>
/// <returns>1.0 - the cosine similarity in the range [0, 1]</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public static double GetDistance(string s1, string s2, int k = DEFAULT_K)
=> 1.0 - GetSimilarity(s1, s2, k);

/// <summary>
///
/// </summary>
/// <param name="profile1"></param>
/// <param name="profile2"></param>
/// <returns></returns>
public double Similarity(IDictionary<string, int> profile1, IDictionary<string, int> profile2)
=> GetSimilarity(profile1, profile2);

/// <inheritdoc cref="Similarity(IDictionary{string, int}, IDictionary{string, int})"/>
public static double GetSimilarity(IDictionary<string, int> profile1, IDictionary<string, int> profile2)
=> DotProduct(profile1, profile2)
/ (Norm(profile1) * Norm(profile2));
}
Expand Down
11 changes: 10 additions & 1 deletion src/F23.StringSimilarity/Damerau.cs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,11 @@ public class Damerau : IMetricStringDistance, IMetricSpanDistance
/// <returns>The computed distance.</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public double Distance(string s1, string s2)
=> Distance(s1.AsSpan(), s2.AsSpan());
=> GetDistance(s1, s2);

/// <inheritdoc cref="Distance(string, string)"/>
public static double GetDistance(string s1, string s2)
=> GetDistance(s1.AsSpan(), s2.AsSpan());

/// <summary>
/// Calculates the Damerau-Levenshtein distance between two sequences.
Expand All @@ -74,6 +78,11 @@ public double Distance(string s1, string s2)
/// <exception cref="ArgumentNullException">Thrown if <paramref name="s1"/> or <paramref name="s2"/> is <see langword="null"/>.</exception>
public double Distance<T>(ReadOnlySpan<T> s1, ReadOnlySpan<T> s2)
where T : IEquatable<T>
=> GetDistance(s1, s2);

/// <inheritdoc cref="Distance{T}(ReadOnlySpan{T}, ReadOnlySpan{T})"/>
public static double GetDistance<T>(ReadOnlySpan<T> s1, ReadOnlySpan<T> s2)
where T : IEquatable<T>
{
if (s1 == null)
{
Expand Down
21 changes: 19 additions & 2 deletions src/F23.StringSimilarity/Experimental/Sift4.cs
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ namespace F23.StringSimilarity.Experimental
/// </summary>
public class Sift4 : IStringDistance
{
private const int DEFAULT_MAX_OFFSET = 10;
/// <summary>
/// The default maximum distance to search for character transposition.
/// </summary>
public const int DEFAULT_MAX_OFFSET = 10;

/// <summary>
/// Gets or sets the maximum distance to search for character transposition.
Expand Down Expand Up @@ -77,6 +80,20 @@ internal Offset(int c1, int c2, bool trans)
/// <param name="s2"></param>
/// <returns></returns>
public double Distance(string s1, string s2)
=> GetDistance(s1, s2, MaxOffset);

/// <summary>
/// Sift4 - a general purpose string distance algorithm inspired by JaroWinkler
/// and Longest Common Subsequence.
/// Original JavaScript algorithm by siderite, java port by Nathan Fischer 2016.
/// https://siderite.dev/blog/super-fast-and-accurate-string-distance.html
/// https://blackdoor.github.io/blog/sift4-java/
/// </summary>
/// <param name="s1"></param>
/// <param name="s2"></param>
/// <param name="maxOffset">The maximum distance to search for character transposition.</param>
/// <returns></returns>
public static double GetDistance(string s1, string s2, int maxOffset = DEFAULT_MAX_OFFSET)
{
if (string.IsNullOrEmpty(s1))
{
Expand Down Expand Up @@ -169,7 +186,7 @@ public double Distance(string s1, string s2)
// (they get incremented at the end of the loop)
// so that we can have only one code block handling matches
for (int i = 0;
i < MaxOffset && (c1 + i < l1 || c2 + i < l2);
i < maxOffset && (c1 + i < l1 || c2 + i < l2);
i++)
{
if ((c1 + i < l1) && (s1[c1 + i] == s2[c2]))
Expand Down
28 changes: 25 additions & 3 deletions src/F23.StringSimilarity/Jaccard.cs
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,17 @@ public Jaccard() { }
/// <returns>The Jaccard index in the range [0, 1]</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public double Similarity(string s1, string s2)
=> GetSimilarity(s1, s2, k);

/// <summary>
/// Compute jaccard index: |A inter B| / |A union B|.
/// </summary>
/// <param name="s1">The first string to compare.</param>
/// <param name="s2">The second string to compare.</param>
/// <param name="k">The length of the k-shingles (sequences of k characters) to compare.</param>
/// <returns>The Jaccard index in the range [0, 1]</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public static double GetSimilarity(string s1, string s2, int k = DEFAULT_K)
{
if (s1 == null)
{
Expand All @@ -81,8 +92,8 @@ public double Similarity(string s1, string s2)
return 1;
}

var profile1 = GetProfile(s1);
var profile2 = GetProfile(s2);
var profile1 = GetProfile(s1, k);
var profile2 = GetProfile(s2, k);

// SSNET Specific: use LINQ for more optimal distinct count
var unionCount = profile1.Keys.Concat(profile2.Keys).Distinct().Count();
Expand All @@ -102,6 +113,17 @@ public double Similarity(string s1, string s2)
/// <returns>1 - the Jaccard similarity.</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public double Distance(string s1, string s2)
=> 1.0 - Similarity(s1, s2);
=> GetDistance(s1, s2, k);

/// <summary>
/// Distance is computed as 1 - similarity.
/// </summary>
/// <param name="s1">The first string to compare.</param>
/// <param name="s2">The second string to compare.</param>
/// <param name="k">The length of the k-shingles (sequences of k characters) to compare.</param>
/// <returns>1 - the Jaccard similarity.</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public static double GetDistance(string s1, string s2, int k = DEFAULT_K)
=> 1.0 - GetSimilarity(s1, s2, k);
}
}
61 changes: 57 additions & 4 deletions src/F23.StringSimilarity/JaroWinkler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,19 @@ public JaroWinkler(double threshold)
/// <returns>The Jaro-Winkler similarity in the range [0, 1]</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public double Similarity(string s1, string s2)
=> Similarity(s1.AsSpan(), s2.AsSpan());
=> GetSimilarity(s1, s2, Threshold);

/// <summary>
/// Compute Jaro-Winkler similarity.
/// </summary>
/// <param name="s1">The first string to compare.</param>
/// <param name="s2">The second string to compare.</param>
/// <param name="threshold">The threshold used for adding the Winkler bonus. Set to a
/// negative value to get the Jaro similarity.</param>
/// <returns>The Jaro-Winkler similarity in the range [0, 1]</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public static double GetSimilarity(string s1, string s2, double threshold = DEFAULT_THRESHOLD)
=> GetSimilarity(s1.AsSpan(), s2.AsSpan(), threshold);

/// <summary>
/// Calculates the similarity between two sequences using the Jaro-Winkler distance metric.
Expand All @@ -94,6 +106,21 @@ public double Similarity(string s1, string s2)
/// <exception cref="ArgumentNullException">Thrown if <paramref name="s1"/> or <paramref name="s2"/> is null.</exception>
public double Similarity<T>(ReadOnlySpan<T> s1, ReadOnlySpan<T> s2)
where T : IEquatable<T>
=> GetSimilarity(s1, s2, Threshold);

/// <summary>
/// Calculates the similarity between two sequences using the Jaro-Winkler distance metric.
/// </summary>
/// <typeparam name="T">The type of elements in the sequences. Must implement <see cref="IEquatable{T}"/>.</typeparam>
/// <param name="s1">The first sequence to compare. Cannot be null.</param>
/// <param name="s2">The second sequence to compare. Cannot be null.</param>
/// <param name="threshold">The threshold used for adding the Winkler bonus. Set to a
/// negative value to get the Jaro similarity.</param>
/// <returns>A value between 0 and 1 representing the similarity between the two sequences, where 1 indicates identical
/// sequences and 0 indicates no similarity.</returns>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="s1"/> or <paramref name="s2"/> is null.</exception>
public static double GetSimilarity<T>(ReadOnlySpan<T> s1, ReadOnlySpan<T> s2, double threshold = DEFAULT_THRESHOLD)
where T : IEquatable<T>
{
if (s1 == null)
{
Expand All @@ -120,7 +147,7 @@ public double Similarity<T>(ReadOnlySpan<T> s1, ReadOnlySpan<T> s2)
/ THREE;
double jw = j;

if (j > Threshold)
if (j > threshold)
{
jw = j + Math.Min(JW_COEF, 1.0 / mtp[THREE]) * mtp[2] * (1 - j);
}
Expand All @@ -135,7 +162,19 @@ public double Similarity<T>(ReadOnlySpan<T> s1, ReadOnlySpan<T> s2)
/// <returns>1 - similarity</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public double Distance(string s1, string s2)
=> 1.0 - Similarity(s1, s2);
=> GetDistance(s1, s2, Threshold);

/// <summary>
/// Return 1 - similarity.
/// </summary>
/// <param name="s1">The first string to compare.</param>
/// <param name="s2">The second string to compare.</param>
/// <param name="threshold">The threshold used for adding the Winkler bonus. Set to a
/// negative value to get the Jaro distance.</param>
/// <returns>1 - similarity</returns>
/// <exception cref="ArgumentNullException">If s1 or s2 is null.</exception>
public static double GetDistance(string s1, string s2, double threshold = DEFAULT_THRESHOLD)
=> 1.0 - GetSimilarity(s1, s2, threshold);

/// <summary>
/// Calculates the distance between two sequences based on their similarity.
Expand All @@ -149,7 +188,21 @@ public double Distance(string s1, string s2)
/// 0.0 indicates identical sequences and 1.0 indicates completely dissimilar sequences.</returns>
public double Distance<T>(ReadOnlySpan<T> s1, ReadOnlySpan<T> s2)
where T : IEquatable<T>
=> 1.0 - Similarity(s1, s2);
=> GetDistance(s1, s2, Threshold);

/// <summary>
/// Calculates the distance between two sequences based on their similarity.
/// </summary>
/// <typeparam name="T">The type of elements in the sequences. Must implement <see cref="IEquatable{T}"/>.</typeparam>
/// <param name="s1">The first sequence to compare.</param>
/// <param name="s2">The second sequence to compare.</param>
/// <param name="threshold">The threshold used for adding the Winkler bonus. Set to a
/// negative value to get the Jaro distance.</param>
/// <returns>A double value representing the distance between the two sequences. The value ranges from 0.0 to 1.0, where
/// 0.0 indicates identical sequences and 1.0 indicates completely dissimilar sequences.</returns>
public static double GetDistance<T>(ReadOnlySpan<T> s1, ReadOnlySpan<T> s2, double threshold = DEFAULT_THRESHOLD)
where T : IEquatable<T>
=> 1.0 - GetSimilarity(s1, s2, threshold);

private static int[] Matches<T>(ReadOnlySpan<T> s1, ReadOnlySpan<T> s2)
where T : IEquatable<T>
Expand Down
Loading