diff --git a/.github/workflows/arm64-codegen-probe.yml b/.github/workflows/arm64-codegen-probe.yml new file mode 100644 index 0000000..431b05a --- /dev/null +++ b/.github/workflows/arm64-codegen-probe.yml @@ -0,0 +1,119 @@ +name: arm64 codegen probe + +# Diagnostic-only. VectorMath.MultiplyWidening32 falls through to the portable `x * y` on arm64 +# (Avx2/Sse2 are both unsupported there), and NEON has no 64-bit vector multiply, so the JIT has to +# synthesise one. The open question is HOW: a vectorised umull-based decomposition, or a scalar +# software fallback. dotnet/runtime#103555 fixed exactly this for x64 and says nothing about arm64. +# This dumps the real arm64 disassembly so the answer comes from the machine, not from reasoning. + +on: + push: + branches: [ 'perf/**' ] + workflow_dispatch: + +jobs: + probe: + # Two arm64 microarchitectures from different vendors: Ampere/Neoverse-N2 on Linux and Apple + # Silicon on macOS. Both emit an identical 33-instruction sequence for each kernel, so the + # xtn/umull lowering is generic arm64 rather than a Neoverse-specific choice. Worth keeping as a + # pair: the JIT reports "generic ARM64 + SVE" on the Neoverse runner and plain "generic ARM64" on + # Apple, and Vector256 stays inactive on both (the only length gate emitted is cmp #2, the + # Vector128 one) — so even an SVE-capable arm64 host exercises only the Vector128 path. + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04-arm + label: neoverse + - runner: macos-26 + label: apple-silicon + runs-on: ${{ matrix.runner }} + permissions: + contents: read + + steps: + - uses: actions/checkout@v7 + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + + # Branch on the tool rather than on exit status: "lscpu | sed ... || sysctl" silently does nothing + # useful on macOS, because the redirect applies to sed and the pipeline reports sed's status (0), + # so the fallback never runs and the chip model is never printed. + - name: Host + run: | + uname -m + if command -v lscpu >/dev/null 2>&1; then + lscpu | sed -n '1,20p' + else + sysctl -n machdep.cpu.brand_string hw.ncpu + fi + + - name: Build + run: | + dotnet restore src/Base58Encoding.slnx + dotnet build src/Base58Encoding.slnx --configuration Release --no-restore + + # DOTNET_JitDisasm works on the release runtime since .NET 8. TieredCompilation=0 skips tier-0 so + # the first call is already fully optimised. The filter must be NAMESPACE-QUALIFIED: a bare + # "VectorMath:*" matches nothing, because the JIT names the method + # "Base58Encoding.VectorMath:TensorDot". JitDisasmSummary is captured alongside so that, if the + # filter ever stops matching again, the log shows every method actually compiled. + # The codegen step below filters to *VectorMathTests* to keep the disassembly readable, so on its + # own it proves nothing about the rest of the library. The publish workflow runs the full suite on + # arm64, but only for master and PRs into it — never for perf/** branches. Run it here so an arm64 + # lane-ordering bug cannot reach master unnoticed. + - name: Test (full suite, arm64) + run: | + dotnet run --project src/Base58Encoding.Tests/Base58Encoding.Tests.csproj \ + --configuration Release --no-build + + - name: Dump arm64 codegen for VectorMath + env: + DOTNET_JitDisasm: 'Base58Encoding.VectorMath:*' + DOTNET_JitDisasmSummary: '1' + DOTNET_TieredCompilation: '0' + run: | + dotnet run --project src/Base58Encoding.Tests/Base58Encoding.Tests.csproj \ + --configuration Release --no-build -- -method "*VectorMathTests*" \ + > arm64-codegen.txt 2>&1 || true + echo "----- captured $(wc -l < arm64-codegen.txt) lines -----" + echo "----- methods the JIT compiled from VectorMath -----" + grep -E 'JIT compiled Base58Encoding\.VectorMath' arm64-codegen.txt || echo "(none - were they inlined?)" + + - name: Verdict + run: | + set -u + listings=$(grep -cE '^; Assembly listing for method' arm64-codegen.txt || true) + echo "assembly listings captured: $listings" + if [ "$listings" -eq 0 ]; then + echo "::error::No disassembly captured - the JitDisasm filter matched nothing, so the census below would be meaningless." + exit 1 + fi + + echo + echo "=== method headers ===" + grep -E '^; Assembly listing for method|^; Emitting|Total bytes of code' arm64-codegen.txt + + echo + echo "=== instruction census (what the 64-bit multiply lowered to) ===" + for m in umull umull2 uzp1 uzp2 mul shl ushr usra add bl blr; do + printf '%-8s %s\n' "$m" "$(grep -cE "^[[:space:]]+$m( |$)" arm64-codegen.txt || true)" + done + + echo + echo "=== calls out of the kernels (scalar software fallback would show here) ===" + grep -nE '^[[:space:]]+(bl|blr)( |$)' arm64-codegen.txt || echo "no calls - not scalarising" + + echo + echo "=== every instruction in the two kernels ===" + sed -n '/^; Assembly listing for method Base58Encoding.VectorMath/,/^; Total bytes of code/p' arm64-codegen.txt + + - name: Upload full dump + if: always() + uses: actions/upload-artifact@v7 + with: + name: arm64-codegen-${{ matrix.label }} + path: arm64-codegen.txt diff --git a/src/Base58Encoding.Benchmarks/BoundsCheckComparisonBenchmark.cs b/src/Base58Encoding.Benchmarks/BoundsCheckComparisonBenchmark.cs index cc3e2f7..bd715af 100644 --- a/src/Base58Encoding.Benchmarks/BoundsCheckComparisonBenchmark.cs +++ b/src/Base58Encoding.Benchmarks/BoundsCheckComparisonBenchmark.cs @@ -9,7 +9,6 @@ namespace Base58Encoding.Benchmarks; -[SimpleJob(RuntimeMoniker.Net90)] [SimpleJob(RuntimeMoniker.Net10_0)] [DisassemblyDiagnoser(exportCombinedDisassemblyReport: true)] [HideColumns("Job", "Error", "StdDev", "Median", "RatioSD")] diff --git a/src/Base58Encoding.Benchmarks/CountLeadingCharactersBenchmark.cs b/src/Base58Encoding.Benchmarks/CountLeadingCharactersBenchmark.cs index 4087db0..a78933d 100644 --- a/src/Base58Encoding.Benchmarks/CountLeadingCharactersBenchmark.cs +++ b/src/Base58Encoding.Benchmarks/CountLeadingCharactersBenchmark.cs @@ -6,7 +6,6 @@ namespace Base58Encoding.Benchmarks; -[SimpleJob(RuntimeMoniker.Net90)] [SimpleJob(RuntimeMoniker.Net10_0)] [HideColumns("Job", "Error", "StdDev", "Median", "RatioSD")] [MemoryDiagnoser] diff --git a/src/Base58Encoding.Benchmarks/CountLeadingZerosBenchmark.cs b/src/Base58Encoding.Benchmarks/CountLeadingZerosBenchmark.cs index e25df86..3df2b46 100644 --- a/src/Base58Encoding.Benchmarks/CountLeadingZerosBenchmark.cs +++ b/src/Base58Encoding.Benchmarks/CountLeadingZerosBenchmark.cs @@ -8,7 +8,6 @@ namespace Base58Encoding.Benchmarks; -[SimpleJob(RuntimeMoniker.Net90)] [SimpleJob(RuntimeMoniker.Net10_0)] [HideColumns("Job", "Error", "StdDev", "Median", "RatioSD")] [DisassemblyDiagnoser] diff --git a/src/Base58Encoding.Tests/Base58DecodeFast.cs b/src/Base58Encoding.Tests/Base58DecodeFast.cs index 674fb0c..39b1ef9 100644 --- a/src/Base58Encoding.Tests/Base58DecodeFast.cs +++ b/src/Base58Encoding.Tests/Base58DecodeFast.cs @@ -92,6 +92,80 @@ public void Decode32ast_WithInvalidInput_ReturnsNull(string input) Assert.Throws(() => Base58.DecodeBitcoin32Fast(input)); } + // The fast paths are chosen purely by encoded LENGTH, but the base58 length ranges are wider than + // the byte counts they map to: 88 characters hold up to 58^88-1 (~2^515.5) while 64 bytes hold + // 2^512-1, and 44 characters hold ~2^257.8 while 32 bytes hold 2^256-1. So a perfectly well-formed + // string of fast-path length can still encode a value that does not fit, and the fast decoders must + // reject it (via the `binary[0] > 0xFFFFFFFF` carry check) so the caller falls back to the generic + // decoder. Without that check Decode would silently return the wrong byte count for roughly 93% of + // 88-character and 72% of 44-character inputs. + + // An over-large input can be rejected by either of two guards, depending on the value: + // * the carry guard, `binary[0] > 0xFFFFFFFF` + // * the leading-zero cross-check, `outputLeadingZeros != inputLeadingOnes` + // 2^512 exactly trips the SECOND one — its top limb is 2^32, whose low 32 bits are zero, so every + // limb looks like a leading zero byte. To exercise the carry guard the top limb's low 32 bits must + // be large, hence the 0xFF in the second byte below. + [Fact] + public void Decode64Fast_RejectsValueTooLargeFor64Bytes() + { + var largestValid = new byte[64]; + Array.Fill(largestValid, (byte)0xFF); // 2^512 - 1, the largest 64-byte value + + var tooLarge = new byte[65]; + tooLarge[0] = 0x01; + tooLarge[1] = 0xFF; // 2^512 + 0xFF * 2^504 — needs 65 bytes, still encodes to 88 chars + + string validEncoded = Base58.Bitcoin.Encode(largestValid); + string tooLargeEncoded = Base58.Bitcoin.Encode(tooLarge); + Assert.Equal(88, validEncoded.Length); + Assert.Equal(88, tooLargeEncoded.Length); // same length, different byte count + + Assert.Equal(largestValid, Base58.DecodeBitcoin64Fast(validEncoded)); + Assert.Null(Base58.DecodeBitcoin64Fast(tooLargeEncoded)); + + // The public API must fall back to the generic decoder and return all 65 bytes. + Assert.Equal(tooLarge, Base58.Bitcoin.Decode(tooLargeEncoded)); + } + + [Fact] + public void Decode32Fast_RejectsValueTooLargeFor32Bytes() + { + var largestValid = new byte[32]; + Array.Fill(largestValid, (byte)0xFF); // 2^256 - 1 + + var tooLarge = new byte[33]; + tooLarge[0] = 0x01; + tooLarge[1] = 0xFF; // 2^256 + 0xFF * 2^248 + + string validEncoded = Base58.Bitcoin.Encode(largestValid); + string tooLargeEncoded = Base58.Bitcoin.Encode(tooLarge); + Assert.Equal(44, validEncoded.Length); + Assert.Equal(44, tooLargeEncoded.Length); + + Assert.Equal(largestValid, Base58.DecodeBitcoin32Fast(validEncoded)); + Assert.Null(Base58.DecodeBitcoin32Fast(tooLargeEncoded)); + + Assert.Equal(tooLarge, Base58.Bitcoin.Decode(tooLargeEncoded)); + } + + // The extreme case: every digit at its maximum. Asserts the decoded VALUE, not just the length, so + // a fallback that returned the right size but wrong bytes would still fail. + [Theory] + [InlineData(88, 65)] // 58^88 - 1 needs 65 bytes + [InlineData(44, 33)] // 58^44 - 1 needs 33 bytes + public void Decode_AllMaxDigitsAtFastPathLength_FallsBackToGeneric(int chars, int expectedBytes) + { + string maxAtLength = new('z', chars); // 'z' is digit 57, so this is 58^chars - 1 + + byte[] decoded = Base58.Bitcoin.Decode(maxAtLength); + + Assert.Equal(expectedBytes, decoded.Length); + Assert.Equal( + System.Numerics.BigInteger.Pow(58, chars) - 1, + new System.Numerics.BigInteger(decoded, isUnsigned: true, isBigEndian: true)); + } + [Fact] public void Decode32Fast_WithLeadingOnes_HandlesCorrectly() { diff --git a/src/Base58Encoding.Tests/BigEndianTests.cs b/src/Base58Encoding.Tests/BigEndianTests.cs index abaaf58..885b7f1 100644 --- a/src/Base58Encoding.Tests/BigEndianTests.cs +++ b/src/Base58Encoding.Tests/BigEndianTests.cs @@ -12,8 +12,9 @@ public BigEndianTests(ITestOutputHelper output) } /// - /// Alternatively can be run manually to verify compatibility on big-endian systems using below command - /// docker run -it --rm --platform linux/s390x -v ProjectPath/src:/src registry.access.redhat.com/dotnet/sdk:10.0 sh -c "cd /src/Base58Encoding.Tests && dotnet run" + /// Alternatively can be run manually to verify compatibility on big-endian systems using below command. + /// Mount the NuGet cache too: restore over the network sometimes hangs under s390x emulation. + /// docker run -it --rm --platform linux/s390x -v ProjectPath/src:/src -v HomeDir/.nuget/packages:/nuget -e NUGET_PACKAGES=/nuget registry.access.redhat.com/dotnet/sdk:10.0 sh -c "cd /src/Base58Encoding.Tests && dotnet run" /// [Fact(Skip = "For local usage only")] public void Base58Encoding_Works_On_BigEndian_ViaProcess() diff --git a/src/Base58Encoding.Tests/SimpleBaseFuzzTests.cs b/src/Base58Encoding.Tests/SimpleBaseFuzzTests.cs index 77b0e8e..7fbc4e6 100644 --- a/src/Base58Encoding.Tests/SimpleBaseFuzzTests.cs +++ b/src/Base58Encoding.Tests/SimpleBaseFuzzTests.cs @@ -11,11 +11,9 @@ namespace Base58Encoding.Tests; // to run just one (e.g. -method "*Bitcoin_32And64*"). // // Ground truth is a BigInteger oracle (the literal definition of Base58), so the fuzz validates our -// code without trusting any third party. We also cross-check our ENCODER against SimpleBase's, but -// intentionally do NOT assert SimpleBase.Decode(ours): SimpleBase's decoder (5.6.0 and 5.6.2) drops -// the most-significant byte on some larger inputs (verified: our encoding matches the oracle and the -// Python base58 library, both of which decode it correctly). Reported: ssg/SimpleBase#83 -// (https://github.com/ssg/SimpleBase/issues/83). +// code without trusting any third party. We also cross-check our encoder against SimpleBase's, but +// not SimpleBase.Decode(ours): its 5.6.2 decoder drops the most-significant byte on some lengths +// (ssg/SimpleBase#83, fixed in 5.6.3 — which we cannot take yet, see Directory.Packages.props). public class SimpleBaseFuzzTests { private const string Alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; @@ -74,7 +72,7 @@ public void EncodeDecode_MatchesOracleAndSimpleBase_UnderRandomInput() _output.WriteLine($"Fuzz OK: {iterations:N0} iterations in {sw.Elapsed.TotalSeconds:F0}s, max input {maxLen} bytes, zero mismatches."); } - // Focused fuzz on the Bitcoin 32- and 64-byte fast paths (TryDecodeBitcoin{32,64}Fast and the + // Focused fuzz on the Bitcoin 32- and 64-byte fast paths (DecodeBitcoin{32,64}Fast and the // SIMD encode). Only 32/64-byte inputs; the MSB is kept non-zero most of the time so the encoding // lands in the fast-path length window (43-44 / 87-88 chars) and Decode takes the fast path. // Exercises the string and byte-span overloads of both Encode and Decode against the oracle. @@ -104,7 +102,7 @@ public void Bitcoin_32And64_FastPaths_MatchOracle_UnderRandomInput() } else if (data[0] == 0) { - // Keep it in the fast-path length window so Decode hits TryDecodeBitcoin{32,64}Fast. + // Keep it in the fast-path length window so Decode hits DecodeBitcoin{32,64}Fast. data[0] = 1; } diff --git a/src/Base58Encoding.Tests/VectorInstructionSetTests.cs b/src/Base58Encoding.Tests/VectorInstructionSetTests.cs index f34c241..d28c3ba 100644 --- a/src/Base58Encoding.Tests/VectorInstructionSetTests.cs +++ b/src/Base58Encoding.Tests/VectorInstructionSetTests.cs @@ -17,10 +17,12 @@ namespace Base58Encoding.Tests; // DOTNET_EnableAVX2=0 disables AVX2 (and AVX-512), leaving SSE -> forces the Vector128 path // DOTNET_EnableHWIntrinsic=0 disables all hardware intrinsics -> forces the scalar path // -// Explicit because it spawns processes -- opt-in, not part of every run. The child runs with the -// default (explicit off), so this test does not re-spawn itself. +// Runs by default: these are the only tests covering the Vector128 and scalar paths. ChildMarker is +// what stops the child spawning its own child. public class VectorInstructionSetTests { + private const string ChildMarker = "BASE58_VECTOR_CHILD"; + private readonly ITestOutputHelper _output; public VectorInstructionSetTests(ITestOutputHelper output) @@ -28,11 +30,13 @@ public VectorInstructionSetTests(ITestOutputHelper output) _output = output; } - [Theory(Explicit = true)] + [Theory] [InlineData("DOTNET_EnableAVX2", "0")] // disable AVX2 -> forces the Vector128 path [InlineData("DOTNET_EnableHWIntrinsic", "0")] // disable all hardware intrinsics -> forces the scalar path public void AllTests_Pass_WithVectorInstructionSetDisabled(string environmentVariable, string value) { + Assert.SkipWhen(Environment.GetEnvironmentVariable(ChildMarker) == "1", "already the child run"); + #if DEBUG const string configuration = "Debug"; #else @@ -55,6 +59,7 @@ public void AllTests_Pass_WithVectorInstructionSetDisabled(string environmentVar startInfo.ArgumentList.Add("--no-build"); startInfo.ArgumentList.Add("--no-restore"); startInfo.Environment[environmentVariable] = value; + startInfo.Environment[ChildMarker] = "1"; using var process = Process.Start(startInfo)!; string stdout = process.StandardOutput.ReadToEnd(); diff --git a/src/Base58Encoding.Tests/VectorMathTests.cs b/src/Base58Encoding.Tests/VectorMathTests.cs new file mode 100644 index 0000000..6ee03eb --- /dev/null +++ b/src/Base58Encoding.Tests/VectorMathTests.cs @@ -0,0 +1,170 @@ +namespace Base58Encoding.Tests; + +// Direct tests for the two vector kernels in VectorMath. +// +// The Bitcoin fast paths only ever call these with four lengths — TensorMultiplyAdd with 8 and 17, +// TensorDot with 9 and 18 — so going through Encode/Decode leaves most of the branch matrix +// unexercised: no tail of length 3, no 0/1/3-iteration vector loops, and on AVX2 hardware the +// `else if (Vector128 ...)` arm is unreachable entirely (it needs 2 <= len < 4). These tests sweep +// length 0..20 against a plain scalar reference so every combination is covered. +// +// Each test also runs on the Vector128 and scalar paths, because VectorInstructionSetTests re-runs +// the whole suite in child processes with DOTNET_EnableAVX2=0 and DOTNET_EnableHWIntrinsic=0. +public class VectorMathTests +{ + // Both kernels require every operand to fit in 32 bits — see MultiplyWidening32. This is the + // exclusive upper bound the fast paths guarantee. + private const long OperandLimit = 1L << 32; + + private static ulong DotReference(ReadOnlySpan x, ReadOnlySpan y) + { + ulong sum = 0UL; + for (int i = 0; i < x.Length; i++) + { + sum += x[i] * y[i]; // wraps mod 2^64, exactly like the kernel is documented to + } + + return sum; + } + + private static void MultiplyAddReference(ReadOnlySpan row, ulong scale, Span acc) + { + for (int i = 0; i < row.Length; i++) + { + acc[i] += row[i] * scale; + } + } + + private static ulong[] RandomOperands(Random rng, int length) + { + var values = new ulong[length]; + for (int i = 0; i < length; i++) + { + values[i] = (ulong)rng.NextInt64(0, OperandLimit); + } + + return values; + } + + // Sweeps every length that changes the branch structure: 0/1 (scalar only), 2/3 (the narrow arm), + // 4..20 (vector loop with every possible tail 0..3). + [Fact] + public void TensorDot_MatchesScalarReference_ForEveryLength() + { + var rng = new Random(58); + + for (int length = 0; length <= 20; length++) + { + for (int iteration = 0; iteration < 200; iteration++) + { + ulong[] x = RandomOperands(rng, length); + ulong[] y = RandomOperands(rng, length); + + Assert.Equal(DotReference(x, y), VectorMath.TensorDot(x, y)); + } + } + } + + [Fact] + public void TensorMultiplyAdd_MatchesScalarReference_ForEveryLength() + { + var rng = new Random(85); + + for (int length = 0; length <= 20; length++) + { + for (int iteration = 0; iteration < 200; iteration++) + { + ulong[] row = RandomOperands(rng, length); + ulong scale = (ulong)rng.NextInt64(0, OperandLimit); + + // Seed the accumulator with non-zero values: the kernel must ADD into it, not overwrite. + ulong[] seed = RandomOperands(rng, length); + ulong[] expected = (ulong[])seed.Clone(); + ulong[] actual = (ulong[])seed.Clone(); + + MultiplyAddReference(row, scale, expected); + VectorMath.TensorMultiplyAdd(row, scale, actual); + + Assert.Equal(expected, actual); + } + } + } + + // The accumulator is only ever read and added to, never multiplied, so it is free to exceed 32 bits + // — production relies on that, since limbs grow well past 2^32 before the reduce pass. The sweep + // above only seeds it with values under 2^32, so this is the one place that covers it. + [Fact] + public void TensorMultiplyAdd_AccumulatorMayExceed32Bits() + { + var row = new ulong[9]; + var acc = new ulong[9]; + var expected = new ulong[9]; + Array.Fill(row, 656356767UL); // 58^5 - 1, the largest table-side operand + Array.Fill(acc, ulong.MaxValue - 7); // accumulator already far above 2^32 + Array.Fill(expected, ulong.MaxValue - 7); + + MultiplyAddReference(row, uint.MaxValue, expected); + VectorMath.TensorMultiplyAdd(row, uint.MaxValue, acc); + + Assert.Equal(expected, acc); + } + + // Boundary operands, which random sampling over [0, 2^32) would essentially never hit. Includes the + // exact worst case the decode kernel can produce: max intermediate limb x max DecodeTable64 entry. + // The uint.MaxValue row also covers 2^64 wrap-around — one term is 18446744065119617025, so from + // length 2 upward the sum overflows, and the vector paths (which accumulate into 2 or 4 partials and + // reduce at the end, so in a different order than scalar) must still agree bit-for-bit. + [Theory] + [InlineData(0UL, 0UL)] + [InlineData(0UL, 4294967295UL)] + [InlineData(1UL, 1UL)] + [InlineData(4294967295UL, 4294967295UL)] // both at the 32-bit ceiling; wraps from length 2 up + [InlineData(656356767UL, 4264082837UL)] // production worst case + [InlineData(656356767UL, 656356767UL)] + public void Kernels_HandleBoundaryOperands(ulong a, ulong b) + { + foreach (int length in new[] { 1, 2, 3, 4, 8, 9, 17, 18, 20 }) + { + var x = new ulong[length]; + var y = new ulong[length]; + Array.Fill(x, a); + Array.Fill(y, b); + + Assert.Equal(DotReference(x, y), VectorMath.TensorDot(x, y)); + + var expected = new ulong[length]; + var actual = new ulong[length]; + MultiplyAddReference(x, b, expected); + VectorMath.TensorMultiplyAdd(x, b, actual); + Assert.Equal(expected, actual); + } + } + + // The precondition MultiplyWidening32 depends on, asserted at its source for both operand sides. + // If a future table or a wider intermediate base ever broke this, the vector paths would silently + // truncate — so check it against the shipped tables rather than trusting the comment. + [Fact] + public void MultiplyWidening32Precondition_HoldsForAllProductionOperands() + { + // Table side. + foreach ((string name, ulong[] table) in new (string, ulong[])[] + { + (nameof(Base58BitcoinTables.EncodeTable32RowMajor), Base58BitcoinTables.EncodeTable32RowMajor), + (nameof(Base58BitcoinTables.EncodeTable64RowMajor), Base58BitcoinTables.EncodeTable64RowMajor), + (nameof(Base58BitcoinTables.DecodeTable32), Base58BitcoinTables.DecodeTable32), + (nameof(Base58BitcoinTables.DecodeTable64), Base58BitcoinTables.DecodeTable64), + }) + { + for (int i = 0; i < table.Length; i++) + { + Assert.True(table[i] < (ulong)OperandLimit, + $"{name}[{i}] = {table[i]} exceeds 2^32; MultiplyWidening32 would truncate it."); + } + } + + // Limb side: an intermediate limb is at most 58^5 - 1, which is below 2^32. + ulong maxLimb = (57UL * 11316496UL) + (57UL * 195112UL) + (57UL * 3364UL) + (57UL * 58UL) + 57UL; + Assert.Equal(Base58BitcoinTables.R1Div - 1UL, maxLimb); + Assert.True(maxLimb < (ulong)OperandLimit); + } +} diff --git a/src/Base58Encoding/Base58.Decode.cs b/src/Base58Encoding/Base58.Decode.cs index 1782aff..549e663 100644 --- a/src/Base58Encoding/Base58.Decode.cs +++ b/src/Base58Encoding/Base58.Decode.cs @@ -2,8 +2,6 @@ using System.Buffers.Binary; using System.Numerics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; namespace Base58Encoding; @@ -28,7 +26,7 @@ public byte[] Decode(scoped ReadOnlySpan encoded) if (encoded.Length is >= 43 and <= 44) { Span buf = stackalloc byte[32]; - if (TryDecodeBitcoin32Fast(encoded, buf) == 32) + if (DecodeBitcoin32Fast(encoded, buf)) { return buf.ToArray(); } @@ -36,7 +34,7 @@ public byte[] Decode(scoped ReadOnlySpan encoded) else if (encoded.Length is >= 87 and <= 88) { Span buf = stackalloc byte[64]; - if (TryDecodeBitcoin64Fast(encoded, buf) == 64) + if (DecodeBitcoin64Fast(encoded, buf)) { return buf.ToArray(); } @@ -84,25 +82,23 @@ public int Decode(scoped ReadOnlySpan encoded, scoped Span destinati return DecodeCore(encoded, destination); } - private int DecodeCore(ReadOnlySpan encoded, Span destination) + private static int DecodeCore(ReadOnlySpan encoded, Span destination) where TChar : unmanaged, IBinaryInteger { if (typeof(TAlphabet) == typeof(BitcoinAlphabet)) { if (encoded.Length is >= 43 and <= 44) { - int r = TryDecodeBitcoin32Fast(encoded, destination); - if (r >= 0) + if (DecodeBitcoin32Fast(encoded, destination)) { - return r; + return 32; } } else if (encoded.Length is >= 87 and <= 88) { - int r = TryDecodeBitcoin64Fast(encoded, destination); - if (r >= 0) + if (DecodeBitcoin64Fast(encoded, destination)) { - return r; + return 64; } } } @@ -111,7 +107,7 @@ private int DecodeCore(ReadOnlySpan encoded, Span destinatio } [SkipLocalsInit] - private int DecodeGenericCore(ReadOnlySpan encoded, Span destination) + private static int DecodeGenericCore(ReadOnlySpan encoded, Span destination) where TChar : unmanaged, IBinaryInteger { TChar firstChar = TChar.CreateTruncating(TAlphabet.FirstCharacter); @@ -136,7 +132,7 @@ private int DecodeGenericCore(ReadOnlySpan encoded, Span des return DecodeGenericCoreLarge(encoded, leadingOnes, scratchSize, destination); } - private int DecodeGenericCoreLarge(ReadOnlySpan encoded, int leadingOnes, int scratchSize, Span destination) + private static int DecodeGenericCoreLarge(ReadOnlySpan encoded, int leadingOnes, int scratchSize, Span destination) where TChar : unmanaged, IBinaryInteger { byte[] rented = ArrayPool.Shared.Rent(scratchSize); @@ -160,7 +156,7 @@ private int DecodeGenericCoreLarge(ReadOnlySpan encoded, int leadi } [SkipLocalsInit] - private byte[] DecodeGenericToArray(ReadOnlySpan encoded) + private static byte[] DecodeGenericToArray(ReadOnlySpan encoded) where TChar : unmanaged, IBinaryInteger { TChar firstChar = TChar.CreateTruncating(TAlphabet.FirstCharacter); @@ -185,7 +181,7 @@ private byte[] DecodeGenericToArray(ReadOnlySpan encoded) return DecodeGenericToArrayLarge(encoded, leadingOnes, scratchSize); } - private byte[] DecodeGenericToArrayLarge(ReadOnlySpan encoded, int leadingOnes, int scratchSize) + private static byte[] DecodeGenericToArrayLarge(ReadOnlySpan encoded, int leadingOnes, int scratchSize) where TChar : unmanaged, IBinaryInteger { byte[] rented = ArrayPool.Shared.Rent(scratchSize); @@ -202,7 +198,7 @@ private byte[] DecodeGenericToArrayLarge(ReadOnlySpan encoded, int } } - private int ComputeGenericDecode(ReadOnlySpan encoded, int leadingOnes, Span digits) + private static int ComputeGenericDecode(ReadOnlySpan encoded, int leadingOnes, Span digits) where TChar : unmanaged, IBinaryInteger { int decodedLength = 1; @@ -254,12 +250,12 @@ private static void EmitGenericDecode(Span destination, int leadingOnes, S } /// - /// Returns bytes written (32) on success, or -1 if the encoded input doesn't - /// represent exactly 32 bytes (caller should fall back to generic decode). - /// Throws on invalid character or insufficient destination when fast path matches. + /// Writes exactly 32 bytes and returns true on success, or false if the encoded input does not + /// represent exactly 32 bytes, in which case the caller falls back to the generic decode. + /// Throws on an invalid character, or on insufficient destination once the fast path commits. /// [SkipLocalsInit] - internal static int TryDecodeBitcoin32Fast(ReadOnlySpan encoded, Span destination) + internal static bool DecodeBitcoin32Fast(ReadOnlySpan encoded, Span destination) where TChar : unmanaged, IBinaryInteger { int charCount = encoded.Length; @@ -270,23 +266,18 @@ internal static int TryDecodeBitcoin32Fast(ReadOnlySpan encoded, S // Prepend zeros to make exactly Raw58Sz32 characters int prepend0 = Base58BitcoinTables.Raw58Sz32 - charCount; - for (int j = 0; j < Base58BitcoinTables.Raw58Sz32; j++) + rawBase58[..prepend0].Clear(); + + for (int i = 0; i < charCount; i++) { - if (j < prepend0) + int c = int.CreateTruncating(encoded[i]); + // Validate + convert using Bitcoin decode table + if ((uint)c >= 128 || bitcoinDecodeTable[c] == 255) { - rawBase58[j] = 0; + ThrowHelper.ThrowInvalidCharacter((char)c); } - else - { - int c = int.CreateTruncating(encoded[j - prepend0]); - // Validate + convert using Bitcoin decode table - if ((uint)c >= 128 || bitcoinDecodeTable[c] == 255) - { - ThrowHelper.ThrowInvalidCharacter((char)c); - } - rawBase58[j] = bitcoinDecodeTable[c]; - } + rawBase58[prepend0 + i] = bitcoinDecodeTable[c]; } // Convert to intermediate format (base 58^5) @@ -306,7 +297,7 @@ internal static int TryDecodeBitcoin32Fast(ReadOnlySpan encoded, S for (int j = 0; j < Base58BitcoinTables.BinarySz32; j++) { - binary[j] = TensorDot(intermediate, Base58BitcoinTables.DecodeTable32.AsSpan(j * Base58BitcoinTables.IntermediateSz32, Base58BitcoinTables.IntermediateSz32)); + binary[j] = VectorMath.TensorDot(intermediate, Base58BitcoinTables.DecodeTable32.AsSpan(j * Base58BitcoinTables.IntermediateSz32, Base58BitcoinTables.IntermediateSz32)); } // Reduce each term to less than 2^32 @@ -319,7 +310,7 @@ internal static int TryDecodeBitcoin32Fast(ReadOnlySpan encoded, S // Check if the result is too large for 32 bytes if (binary[0] > 0xFFFFFFFFUL) { - return -1; + return false; } // Count leading zero bytes in the output directly from binary[] without materializing it. @@ -343,7 +334,7 @@ internal static int TryDecodeBitcoin32Fast(ReadOnlySpan encoded, S if (outputLeadingZeros != inputLeadingOnes) { - return -1; + return false; } if (destination.Length < 32) @@ -359,11 +350,11 @@ internal static int TryDecodeBitcoin32Fast(ReadOnlySpan encoded, S BinaryPrimitives.WriteUInt32BigEndian(destination.Slice(offset, sizeof(uint)), value); } - return 32; + return true; } [SkipLocalsInit] - internal static int TryDecodeBitcoin64Fast(ReadOnlySpan encoded, Span destination) + internal static bool DecodeBitcoin64Fast(ReadOnlySpan encoded, Span destination) where TChar : unmanaged, IBinaryInteger { int charCount = encoded.Length; @@ -374,23 +365,18 @@ internal static int TryDecodeBitcoin64Fast(ReadOnlySpan encoded, S // Prepend zeros to make exactly Raw58Sz64 characters int prepend0 = Base58BitcoinTables.Raw58Sz64 - charCount; - for (int j = 0; j < Base58BitcoinTables.Raw58Sz64; j++) + rawBase58[..prepend0].Clear(); + + for (int i = 0; i < charCount; i++) { - if (j < prepend0) + int c = int.CreateTruncating(encoded[i]); + // Validate + convert using Bitcoin decode table + if ((uint)c >= 128 || bitcoinDecodeTable[c] == 255) { - rawBase58[j] = 0; + ThrowHelper.ThrowInvalidCharacter((char)c); } - else - { - int c = int.CreateTruncating(encoded[j - prepend0]); - // Validate + convert using Bitcoin decode table - if ((uint)c >= 128 || bitcoinDecodeTable[c] == 255) - { - ThrowHelper.ThrowInvalidCharacter((char)c); - } - rawBase58[j] = bitcoinDecodeTable[c]; - } + rawBase58[prepend0 + i] = bitcoinDecodeTable[c]; } // Convert to intermediate format (base 58^5) @@ -410,7 +396,7 @@ internal static int TryDecodeBitcoin64Fast(ReadOnlySpan encoded, S for (int j = 0; j < Base58BitcoinTables.BinarySz64; j++) { - binary[j] = TensorDot(intermediate, Base58BitcoinTables.DecodeTable64.AsSpan(j * Base58BitcoinTables.IntermediateSz64, Base58BitcoinTables.IntermediateSz64)); + binary[j] = VectorMath.TensorDot(intermediate, Base58BitcoinTables.DecodeTable64.AsSpan(j * Base58BitcoinTables.IntermediateSz64, Base58BitcoinTables.IntermediateSz64)); } // Reduce each term to less than 2^32 @@ -423,7 +409,7 @@ internal static int TryDecodeBitcoin64Fast(ReadOnlySpan encoded, S // Check if the result is too large for 64 bytes if (binary[0] > 0xFFFFFFFFUL) { - return -1; + return false; } // Count leading zero bytes in the output directly from binary[] without materializing it. @@ -447,7 +433,7 @@ internal static int TryDecodeBitcoin64Fast(ReadOnlySpan encoded, S if (outputLeadingZeros != inputLeadingOnes) { - return -1; + return false; } if (destination.Length < 64) @@ -463,78 +449,18 @@ internal static int TryDecodeBitcoin64Fast(ReadOnlySpan encoded, S BinaryPrimitives.WriteUInt32BigEndian(destination.Slice(offset, sizeof(uint)), value); } - return 64; + return true; } internal static byte[]? DecodeBitcoin32Fast(ReadOnlySpan encoded) { Span buffer = stackalloc byte[32]; - int r = TryDecodeBitcoin32Fast(encoded, buffer); - return r < 0 ? null : buffer.ToArray(); + return DecodeBitcoin32Fast(encoded, buffer) ? buffer.ToArray() : null; } internal static byte[]? DecodeBitcoin64Fast(ReadOnlySpan encoded) { Span buffer = stackalloc byte[64]; - int r = TryDecodeBitcoin64Fast(encoded, buffer); - return r < 0 ? null : buffer.ToArray(); - } - - // Vectorized dot product of two equal-length ulong spans: sum(x[i] * y[i]). - // A focused, dependency-free stand-in for TensorPrimitives.Dot, tuned for the small - // fixed-length decode columns (IntermediateSz32/64). Widest available width first, then a - // scalar tail / fallback. The ulong multiply-accumulate wraps identically to the scalar loop, - // so results are bit-for-bit the same. - // - // A fully bounds-checked (safe) rewrite is possible on .NET 11+ using the consume-and-advance - // idiom — guard every span and advance by re-slicing: - // while (x.Length >= Vector256.Count && y.Length >= Vector256.Count) - // { - // acc += Vector256.Create(x) * Vector256.Create(y); - // x = x.Slice(Vector256.Count); - // y = y.Slice(Vector256.Count); - // } - // On .NET 11 it JITs bounds-check-free and reaches parity on arm64, but on x64 the JIT still - // emits a redundant second length guard per iteration (the spans are equal-length, but it can't - // prove it), so it runs ~13-33% slower at the short lengths this kernel uses (9/18). On .NET 10 - // it is slower on every architecture. Staying on LoadUnsafe until the x64 check is elided. - // Benchmark (our exact kernels, by @EgorBo): https://github.com/EgorBot/Benchmarks/issues/401 - private static ulong TensorDot(ReadOnlySpan x, ReadOnlySpan y) - { - ref ulong xr = ref MemoryMarshal.GetReference(x); - ref ulong yr = ref MemoryMarshal.GetReference(y); - int len = x.Length; - int i = 0; - ulong sum = 0UL; - - if (Vector256.IsHardwareAccelerated && len >= Vector256.Count) - { - Vector256 acc = Vector256.Zero; - int upper = len - Vector256.Count; - for (; i <= upper; i += Vector256.Count) - { - acc += Vector256.LoadUnsafe(ref xr, (nuint)i) * Vector256.LoadUnsafe(ref yr, (nuint)i); - } - - sum += Vector256.Sum(acc); - } - else if (Vector128.IsHardwareAccelerated && len >= Vector128.Count) - { - Vector128 acc = Vector128.Zero; - int upper = len - Vector128.Count; - for (; i <= upper; i += Vector128.Count) - { - acc += Vector128.LoadUnsafe(ref xr, (nuint)i) * Vector128.LoadUnsafe(ref yr, (nuint)i); - } - - sum += Vector128.Sum(acc); - } - - for (; i < len; i++) - { - sum += Unsafe.Add(ref xr, i) * Unsafe.Add(ref yr, i); - } - - return sum; + return DecodeBitcoin64Fast(encoded, buffer) ? buffer.ToArray() : null; } } diff --git a/src/Base58Encoding/Base58.Encode.cs b/src/Base58Encoding/Base58.Encode.cs index 9cfe3fb..02e9fe8 100644 --- a/src/Base58Encoding/Base58.Encode.cs +++ b/src/Base58Encoding/Base58.Encode.cs @@ -3,8 +3,6 @@ using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; namespace Base58Encoding; @@ -64,7 +62,7 @@ public int Encode(scoped ReadOnlySpan data, scoped Span destination) } [SkipLocalsInit] - private string EncodeGenericToString(ReadOnlySpan data) + private static string EncodeGenericToString(ReadOnlySpan data) { int leadingZeros = Base58.CountLeadingZeros(data); @@ -87,7 +85,7 @@ private string EncodeGenericToString(ReadOnlySpan data) return EncodeGenericToStringLarge(inputSpan, leadingZeros, size); } - private string EncodeGenericToStringLarge(ReadOnlySpan inputSpan, int leadingZeros, int size) + private static string EncodeGenericToStringLarge(ReadOnlySpan inputSpan, int leadingZeros, int size) { byte[] rented = ArrayPool.Shared.Rent(size); try @@ -103,7 +101,7 @@ private string EncodeGenericToStringLarge(ReadOnlySpan inputSpan, int lead } [SkipLocalsInit] - private int EncodeGenericToBytes(ReadOnlySpan data, Span destination) + private static int EncodeGenericToBytes(ReadOnlySpan data, Span destination) { int leadingZeros = Base58.CountLeadingZeros(data); @@ -139,7 +137,7 @@ private int EncodeGenericToBytes(ReadOnlySpan data, Span destination return EncodeGenericToBytesLarge(inputSpan, leadingZeros, size, destination); } - private int EncodeGenericToBytesLarge(ReadOnlySpan inputSpan, int leadingZeros, int size, Span destination) + private static int EncodeGenericToBytesLarge(ReadOnlySpan inputSpan, int leadingZeros, int size, Span destination) { byte[] rented = ArrayPool.Shared.Rent(size); try @@ -245,6 +243,11 @@ private static int EncodeBitcoin32FastToBytes(ReadOnlySpan data, Span data, Span rawBase58) { + // Span params hide their length. Re-slicing to the fixed sizes folds away ~50 bounds checks + // below: 30% less code, perf-neutral. + data = data[..(Base58BitcoinTables.BinarySz32 * sizeof(uint))]; + rawBase58 = rawBase58[..Base58BitcoinTables.Raw58Sz32]; + // Convert 32 bytes to 8 uint32 limbs (big-endian) Span binary = stackalloc uint[Base58BitcoinTables.BinarySz32]; for (int i = 0; i < Base58BitcoinTables.BinarySz32; i++) @@ -262,7 +265,7 @@ private static int ComputeBitcoin32FastRaw(ReadOnlySpan data, Span r Span acc = intermediate.Slice(1, rowLen); for (int i = 0; i < Base58BitcoinTables.BinarySz32; i++) { - TensorMultiplyAdd(Base58BitcoinTables.EncodeTable32RowMajor.AsSpan(i * rowLen, rowLen), binary[i], acc); + VectorMath.TensorMultiplyAdd(Base58BitcoinTables.EncodeTable32RowMajor.AsSpan(i * rowLen, rowLen), binary[i], acc); } // Reduce each term to be less than 58^5 @@ -353,6 +356,10 @@ private static int EncodeBitcoin64FastToBytes(ReadOnlySpan data, Span data, Span rawBase58) { + // See ComputeBitcoin32FastRaw. + data = data[..(Base58BitcoinTables.BinarySz64 * sizeof(uint))]; + rawBase58 = rawBase58[..Base58BitcoinTables.Raw58Sz64]; + // Convert 64 bytes to 16 uint32 limbs (big-endian) Span binary = stackalloc uint[Base58BitcoinTables.BinarySz64]; for (int i = 0; i < Base58BitcoinTables.BinarySz64; i++) @@ -371,7 +378,7 @@ private static int ComputeBitcoin64FastRaw(ReadOnlySpan data, Span r Span acc = intermediate.Slice(1, rowLen); for (int i = 0; i < 8; i++) { - TensorMultiplyAdd(Base58BitcoinTables.EncodeTable64RowMajor.AsSpan(i * rowLen, rowLen), binary[i], acc); + VectorMath.TensorMultiplyAdd(Base58BitcoinTables.EncodeTable64RowMajor.AsSpan(i * rowLen, rowLen), binary[i], acc); } // Mini-reduction to prevent overflow (like Firedancer) @@ -380,7 +387,7 @@ private static int ComputeBitcoin64FastRaw(ReadOnlySpan data, Span r for (int i = 8; i < Base58BitcoinTables.BinarySz64; i++) { - TensorMultiplyAdd(Base58BitcoinTables.EncodeTable64RowMajor.AsSpan(i * rowLen, rowLen), binary[i], acc); + VectorMath.TensorMultiplyAdd(Base58BitcoinTables.EncodeTable64RowMajor.AsSpan(i * rowLen, rowLen), binary[i], acc); } // Reduce each term to be less than 58^5 @@ -415,52 +422,6 @@ private static int ComputeBitcoin64FastRaw(ReadOnlySpan data, Span r return rawLeadingZeros; } - // acc[k] += row[k] * scale over the whole row: multiply the row by a scalar and add into the - // accumulator. The encode counterpart of decode's TensorDot; mirrors TensorPrimitives.MultiplyAdd - // (System.Numerics.Tensors) as a tiny dependency-free version tuned for the fixed-length encode - // rows. Widest available vector width first, then a scalar tail that also serves as the fallback - // when no width is hardware-accelerated. Wrapping ulong multiply-add in source-limb order, so the - // result is bit-identical to the scalar loop. - // - // Kept on LoadUnsafe for the same reason as decode's TensorDot — see the safe-rewrite note there - // (safe span ops keep a bounds check on .NET 10, and a redundant per-iteration length guard on - // x64 through .NET 11; parity only on arm64 / large inputs). - private static void TensorMultiplyAdd(ReadOnlySpan row, ulong scale, Span acc) - { - ref ulong rr = ref MemoryMarshal.GetReference(row); - ref ulong ar = ref MemoryMarshal.GetReference(acc); - int len = row.Length; - int i = 0; - - if (Vector256.IsHardwareAccelerated && len >= Vector256.Count) - { - Vector256 s = Vector256.Create(scale); - int upper = len - Vector256.Count; - for (; i <= upper; i += Vector256.Count) - { - Vector256 a = Vector256.LoadUnsafe(ref ar, (nuint)i); - Vector256 r = Vector256.LoadUnsafe(ref rr, (nuint)i); - (a + (r * s)).StoreUnsafe(ref ar, (nuint)i); - } - } - else if (Vector128.IsHardwareAccelerated && len >= Vector128.Count) - { - Vector128 s = Vector128.Create(scale); - int upper = len - Vector128.Count; - for (; i <= upper; i += Vector128.Count) - { - Vector128 a = Vector128.LoadUnsafe(ref ar, (nuint)i); - Vector128 r = Vector128.LoadUnsafe(ref rr, (nuint)i); - (a + (r * s)).StoreUnsafe(ref ar, (nuint)i); - } - } - - for (; i < len; i++) - { - Unsafe.Add(ref ar, i) += Unsafe.Add(ref rr, i) * scale; - } - } - private readonly ref struct EncodeState where T : struct, IBase58Alphabet { diff --git a/src/Base58Encoding/VectorMath.cs b/src/Base58Encoding/VectorMath.cs new file mode 100644 index 0000000..b727fcc --- /dev/null +++ b/src/Base58Encoding/VectorMath.cs @@ -0,0 +1,183 @@ +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; + +namespace Base58Encoding; + +// Dot product and scaled add (axpy) for the 32/64-byte Bitcoin fast paths, plus the widening multiply +// they share. The only file in the library that touches MemoryMarshal, Unsafe or hardware intrinsics, +// so that surface stays auditable in one place. Not nested in Base58: nothing here depends +// on the alphabet, and a nested type would be JIT-compiled once per instantiation. +internal static class VectorMath +{ + /// + /// Multiplies the low 32 bits of each 64-bit lane into a full 64-bit product. + /// Both operands must fit in 32 bits; the assert enforces it. + /// + /// + /// AVX2 and NEON have no 64x64 multiply, so a portable x * y costs eight instructions. Every + /// operand here is under 2^32 — limbs and encode entries are < 58^5, decode entries < 2^32, + /// binary limbs are uint32 — so vpmuludq gives the identical answer in one. The JIT cannot + /// substitute it: that needs proof the operands are narrow, and they come from a runtime-built table + /// and a span. Purely a speed change; x * y was already correct. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Vector256 MultiplyWidening32(Vector256 x, Vector256 y) + { + Debug.Assert( + (x & Vector256.Create(0xFFFFFFFF_00000000UL)) == Vector256.Zero && + (y & Vector256.Create(0xFFFFFFFF_00000000UL)) == Vector256.Zero, + "MultiplyWidening32 requires both operands < 2^32; a wider value would be silently truncated."); + + // No arm64 branch here, unlike the Vector128 overload. Vector256.IsHardwareAccelerated is + // never true on arm64: NEON registers are 128-bit, and SVE is vector-length agnostic so .NET + // exposes it through its own API rather than mapping Vector256 onto it — the Neoverse-N2 + // probe reports sve2 in its CPU flags and the JIT still emits only the Vector128 length gate. + // Both callers gate on that property, so arm64 never reaches this width. Note the type itself + // is perfectly usable there; unguarded it would give identical results, just emulated as two + // 128-bit halves. Accelerated or not, an AdvSimd branch here would never execute. + return Avx2.IsSupported ? Avx2.Multiply(x.AsUInt32(), y.AsUInt32()) : x * y; + } + + /// + /// + /// arm64 hits the same wall as x64 for the opposite reason: NEON's MUL has no 64-bit form and + /// UMULL takes 32-bit inputs, so the JIT cannot use it for a general 64x64 multiply either. + /// Its fallback extracts each lane to a general-purpose register, uses the scalar mul, and + /// reinserts — 8 instructions for 2 lanes, crossing the NEON/GPR domain four times (verified on a + /// Neoverse-N2 via the arm64 codegen probe workflow). Because both operands fit in 32 bits, two + /// xtn and one umull do the same work in the vector domain with no round trip. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Vector128 MultiplyWidening32(Vector128 x, Vector128 y) + { + Debug.Assert( + (x & Vector128.Create(0xFFFFFFFF_00000000UL)) == Vector128.Zero && + (y & Vector128.Create(0xFFFFFFFF_00000000UL)) == Vector128.Zero, + "MultiplyWidening32 requires both operands < 2^32; a wider value would be silently truncated."); + + if (Sse2.IsSupported) + { + return Sse2.Multiply(x.AsUInt32(), y.AsUInt32()); + } + + if (AdvSimd.IsSupported) + { + // xtn narrows each 64-bit lane to its low 32 bits, packing a vector into a Vector64; + // lossless here because both operands are under 2^32. umull then widens lane-wise back to + // 64-bit products. Three instructions, all in the vector domain. + return AdvSimd.MultiplyWideningLower( + AdvSimd.ExtractNarrowingLower(x), + AdvSimd.ExtractNarrowingLower(y)); + } + + return x * y; + } + + // Vectorized dot product of two equal-length ulong spans: sum(x[i] * y[i]). + // A focused, dependency-free stand-in for TensorPrimitives.Dot, tuned for the small + // fixed-length decode columns (IntermediateSz32/64). Widest available width first, then a + // scalar tail / fallback. The ulong multiply-accumulate wraps identically to the scalar loop, + // so results are bit-for-bit the same. + // + // A fully bounds-checked (safe) rewrite is possible on .NET 11+ using the consume-and-advance + // idiom — guard every span and advance by re-slicing: + // while (x.Length >= Vector256.Count && y.Length >= Vector256.Count) + // { + // acc += Vector256.Create(x) * Vector256.Create(y); + // x = x.Slice(Vector256.Count); + // y = y.Slice(Vector256.Count); + // } + // On .NET 11 it JITs bounds-check-free and reaches parity on arm64, but on x64 the JIT still + // emits a redundant second length guard per iteration (the spans are equal-length, but it can't + // prove it), so it runs ~13-33% slower at the short lengths this kernel uses (9/18). On .NET 10 + // it is slower on every architecture. Staying on LoadUnsafe until the x64 check is elided. + // Upstream tracking: https://github.com/dotnet/runtime/issues/127506 (removing unsafe from the + // vectorization guidance). The x64 gap and the 13-33% figures are from the benchmark run in + // https://github.com/EgorBot/Benchmarks/issues/401 (Zen 5 / Turin x64 vs Apple M1 arm64). + internal static ulong TensorDot(ReadOnlySpan x, ReadOnlySpan y) + { + ref ulong xr = ref MemoryMarshal.GetReference(x); + ref ulong yr = ref MemoryMarshal.GetReference(y); + int len = x.Length; + int i = 0; + ulong sum = 0UL; + + if (Vector256.IsHardwareAccelerated && len >= Vector256.Count) + { + Vector256 acc = Vector256.Zero; + int upper = len - Vector256.Count; + for (; i <= upper; i += Vector256.Count) + { + acc += MultiplyWidening32(Vector256.LoadUnsafe(ref xr, (nuint)i), Vector256.LoadUnsafe(ref yr, (nuint)i)); + } + + sum += Vector256.Sum(acc); + } + else if (Vector128.IsHardwareAccelerated && len >= Vector128.Count) + { + Vector128 acc = Vector128.Zero; + int upper = len - Vector128.Count; + for (; i <= upper; i += Vector128.Count) + { + acc += MultiplyWidening32(Vector128.LoadUnsafe(ref xr, (nuint)i), Vector128.LoadUnsafe(ref yr, (nuint)i)); + } + + sum += Vector128.Sum(acc); + } + + for (; i < len; i++) + { + sum += Unsafe.Add(ref xr, i) * Unsafe.Add(ref yr, i); + } + + return sum; + } + + // acc[k] += row[k] * scale over the whole row: multiply the row by a scalar and add into the + // accumulator. The encode counterpart of TensorDot; mirrors TensorPrimitives.MultiplyAdd + // (System.Numerics.Tensors) as a tiny dependency-free version tuned for the fixed-length encode + // rows. Widest available vector width first, then a scalar tail that also serves as the fallback + // when no width is hardware-accelerated. Wrapping ulong multiply-add in source-limb order, so the + // result is bit-identical to the scalar loop. + // + // Kept on LoadUnsafe for the same reason as TensorDot — see the safe-rewrite note above. + internal static void TensorMultiplyAdd(ReadOnlySpan row, ulong scale, Span acc) + { + ref ulong rr = ref MemoryMarshal.GetReference(row); + ref ulong ar = ref MemoryMarshal.GetReference(acc); + int len = row.Length; + int i = 0; + + if (Vector256.IsHardwareAccelerated && len >= Vector256.Count) + { + Vector256 s = Vector256.Create(scale); + int upper = len - Vector256.Count; + for (; i <= upper; i += Vector256.Count) + { + Vector256 a = Vector256.LoadUnsafe(ref ar, (nuint)i); + Vector256 r = Vector256.LoadUnsafe(ref rr, (nuint)i); + (a + MultiplyWidening32(r, s)).StoreUnsafe(ref ar, (nuint)i); + } + } + else if (Vector128.IsHardwareAccelerated && len >= Vector128.Count) + { + Vector128 s = Vector128.Create(scale); + int upper = len - Vector128.Count; + for (; i <= upper; i += Vector128.Count) + { + Vector128 a = Vector128.LoadUnsafe(ref ar, (nuint)i); + Vector128 r = Vector128.LoadUnsafe(ref rr, (nuint)i); + (a + MultiplyWidening32(r, s)).StoreUnsafe(ref ar, (nuint)i); + } + } + + for (; i < len; i++) + { + Unsafe.Add(ref ar, i) += Unsafe.Add(ref rr, i) * scale; + } + } +} diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index f1c207b..1e9d92d 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -5,6 +5,7 @@ +