From 5fd4e4627eb7194a29bcddda09b2f12b4f734a78 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 14 Oct 2026 00:50:50 +0000 Subject: [PATCH 1/4] Add failing repro: NVQ bitwise oracle test flips with vector width and JIT state --- .../jvector/vector/KernelTestProbe.java | 82 +++++++++++++++++++ .../TestNVQEncodingBitsAcrossJitStates.java | 61 ++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/KernelTestProbe.java create mode 100644 jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestNVQEncodingBitsAcrossJitStates.java diff --git a/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/KernelTestProbe.java b/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/KernelTestProbe.java new file mode 100644 index 000000000..c20e53519 --- /dev/null +++ b/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/KernelTestProbe.java @@ -0,0 +1,82 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.jbellis.jvector.vector; + +import org.opentest4j.TestAbortedException; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/// Runs one of this package's kernel tests in a fresh JVM with extra JVM options, so a test can +/// be checked under a host shape this machine doesn't have (e.g. `-XX:MaxVectorSize=16` gives a +/// 128-bit preferred species, as on ARM NEON) and with a fresh JIT state each time. +/// +/// The child prints `PASS`, `SKIP` (the test aborted on an assumption) or `FAIL `. +final class KernelTestProbe { + private KernelTestProbe() {} + + /// Result of one child run: its outcome line and full output. + record Outcome(String result, String output) { + boolean passedOrSkipped() { return result.equals("PASS") || result.startsWith("SKIP"); } + } + + static Outcome run(String test, String... jvmOptions) throws Exception { + var command = new ArrayList(); + command.add(Path.of(System.getProperty("java.home"), "bin", "java").toString()); + command.add("--add-modules=jdk.incubator.vector"); + command.addAll(Arrays.asList(jvmOptions)); + command.add("-cp"); + command.add(System.getProperty("surefire.test.class.path", System.getProperty("java.class.path"))); + command.add(Child.class.getName()); + command.add(test); + var process = new ProcessBuilder(command).redirectErrorStream(true).start(); + if (!process.waitFor(120, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new AssertionError("probe timed out: " + test + " " + Arrays.toString(jvmOptions)); + } + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + String result = output.lines().filter(l -> l.startsWith("PASS") || l.startsWith("SKIP") || l.startsWith("FAIL")) + .reduce((first, second) -> second).orElse("FAIL no result: " + output); + return new Outcome(result, output); + } + + /// Child entry point: runs the named test method and reports its outcome on stdout. + static final class Child { + public static void main(String[] args) { + try { + switch (args[0]) { + case "nvqEncodingBits" -> new TestNVQEncodingAllocation().preservesFrozenEncodingAndLossBits(); + case "dot256ExactWidth" -> new TestDotProduct256().exactWidthPreservesOriginalReductionBits(); + default -> throw new IllegalArgumentException("unknown test " + args[0]); + } + System.out.println("PASS"); + } catch (TestAbortedException e) { + System.out.println("SKIP " + e.getMessage()); + } catch (Throwable t) { + System.out.println("FAIL " + t); + } + } + } + + static List describe(List outcomes) { + return outcomes.stream().map(Outcome::result).toList(); + } +} diff --git a/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestNVQEncodingBitsAcrossJitStates.java b/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestNVQEncodingBitsAcrossJitStates.java new file mode 100644 index 000000000..7123ac83c --- /dev/null +++ b/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestNVQEncodingBitsAcrossJitStates.java @@ -0,0 +1,61 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.jbellis.jvector.vector; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// Repro for #755: `TestNVQEncodingAllocation` fails intermittently, by 1-2 ULP, depending on +/// the host's vector width and on JIT timing. +/// +/// ## The bug +/// +/// `TestNVQEncodingAllocation.preservesFrozenEncodingAndLossBits` compares the reductions of +/// `nvqLoss`, `nvqUniformLoss`, `nvqSquareL2Distance8bit` and `nvqCosine8bit` bit for bit against a +/// `Reference` subclass. Both sides end in `FloatVector.reduceLanes(ADD)`, whose order of +/// additions the Vector API leaves unspecified: the Java fallback (interpreter, C1) and the C2 +/// intrinsic can round differently. The test calls the two sides alternately in one JVM, so which +/// of them is compiled, and by which tier, varies from run to run. On this x86 host the default +/// 512-bit species happens to agree across tiers, but with a 256-bit or 128-bit preferred species +/// the two sides disagree in most runs, while both agree under `-Xint`, under `-Xcomp`, and under +/// C1 alone. That is the 1-2 ULP `nvqLoss` failure the PR's own CI shows on avx512f (JDK 24), the +/// ARM NEON runner, and the bench build. The kernels are not wrong; the bitwise oracle is not a +/// stable requirement across JIT states. +/// +/// ## The test +/// +/// Runs the PR's test in six fresh JVMs, three with `-XX:MaxVectorSize=32` (a 256-bit preferred +/// species, as on AVX2 hosts) and three with `-XX:MaxVectorSize=16` (128-bit, as on NEON), and +/// requires every run to pass. At `d903d206` individual runs fail about 60-80% of the time at these +/// widths, so all six passing by chance is very unlikely. +class TestNVQEncodingBitsAcrossJitStates { + @Test + void passesInEveryFreshJvmAtEveryVectorWidth() throws Exception { + var outcomes = new ArrayList(); + for (String width : new String[] {"-XX:MaxVectorSize=32", "-XX:MaxVectorSize=16"}) { + for (int run = 0; run < 3; run++) { + outcomes.add(KernelTestProbe.run("nvqEncodingBits", width)); + } + } + assertTrue(outcomes.stream().allMatch(KernelTestProbe.Outcome::passedOrSkipped), + "TestNVQEncodingAllocation outcomes (3 runs at 256-bit, then 3 at 128-bit): " + + KernelTestProbe.describe(outcomes)); + } +} From e25e8ba2e9c0a7f502f442deb3d973cb247badc6 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 14 Oct 2026 00:50:58 +0000 Subject: [PATCH 2/4] Add failing repro: exact-width dot256 test fails on 128-bit hosts --- .../TestDotProduct256OnNarrowHosts.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestDotProduct256OnNarrowHosts.java diff --git a/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestDotProduct256OnNarrowHosts.java b/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestDotProduct256OnNarrowHosts.java new file mode 100644 index 000000000..e1cda3221 --- /dev/null +++ b/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestDotProduct256OnNarrowHosts.java @@ -0,0 +1,48 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.jbellis.jvector.vector; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/// Repro for #755: `TestDotProduct256.exactWidthPreservesOriginalReductionBits` fails on every +/// host whose preferred species is 128 bits, such as the ARM NEON CI runner. +/// +/// ## The bug +/// +/// The test's expected value is one 8-lane `SPECIES_256` multiply followed by `reduceLanes(ADD)`, +/// and it asserts that `dotProduct(a, offset, b, offset + 2, 8)` returns those exact bits. That +/// holds only where a length-8 dot product is routed to `dot256` (512-bit preferred species) or to +/// an 8-lane preferred kernel (256-bit). With a 4-lane preferred species, `dotProduct` sends length +/// 8 to `dotProductPreferred`, which adds two 4-lane products lane-wise before reducing, a +/// different association. That is the PR's ARM CI failure, +/// `expected: <-1088322672> but was: <-1088322673>`. The kernel is fine (main routes the same +/// way); the test asserts a property the 128-bit dispatch never had. +/// +/// ## The test +/// +/// Runs the PR's test in a fresh JVM with `-XX:MaxVectorSize=16`, which gives this x86 host a +/// 128-bit preferred species, and requires it to pass or to skip. At `d903d206` it fails with the +/// same bits as on ARM, on every run. +class TestDotProduct256OnNarrowHosts { + @Test + void exactWidthTestHoldsOnA128BitHost() throws Exception { + var outcome = KernelTestProbe.run("dot256ExactWidth", "-XX:MaxVectorSize=16"); + assertTrue(outcome.passedOrSkipped(), "TestDotProduct256 with a 128-bit preferred species: " + outcome.result()); + } +} From 0ea4ef0a7ee311e01df12304b159321a49df9cb2 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Wed, 14 Oct 2026 00:57:38 +0000 Subject: [PATCH 3/4] Compare NVQ reductions within tolerance instead of bit for bit --- .../jvector/vector/KernelTestProbe.java | 2 +- .../vector/TestNVQEncodingAllocation.java | 21 ++++++++++++++----- .../TestNVQEncodingBitsAcrossJitStates.java | 3 ++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/KernelTestProbe.java b/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/KernelTestProbe.java index c20e53519..42df95b98 100644 --- a/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/KernelTestProbe.java +++ b/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/KernelTestProbe.java @@ -63,7 +63,7 @@ static final class Child { public static void main(String[] args) { try { switch (args[0]) { - case "nvqEncodingBits" -> new TestNVQEncodingAllocation().preservesFrozenEncodingAndLossBits(); + case "nvqEncodingBits" -> new TestNVQEncodingAllocation().preservesFrozenEncodingBitsAndLossValues(); case "dot256ExactWidth" -> new TestDotProduct256().exactWidthPreservesOriginalReductionBits(); default -> throw new IllegalArgumentException("unknown test " + args[0]); } diff --git a/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestNVQEncodingAllocation.java b/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestNVQEncodingAllocation.java index a9142b3e7..a01f1ca6d 100644 --- a/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestNVQEncodingAllocation.java +++ b/jvector-twenty/src/test/java/io/github/jbellis/jvector/vector/TestNVQEncodingAllocation.java @@ -22,7 +22,13 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; class TestNVQEncodingAllocation { - @Test void preservesFrozenEncodingAndLossBits() { + /// Encoded bytes must match the frozen reference exactly. The four reductions (`nvqLoss`, + /// `nvqUniformLoss`, `nvqSquareL2Distance8bit`, `nvqCosine8bit`) are compared within a small + /// relative tolerance instead of bit for bit: they end in `FloatVector.reduceLanes(ADD)`, whose + /// addition order the Vector API leaves unspecified, so the interpreter/C1 fallback and the C2 + /// intrinsic can differ by an ULP or two, and which tier runs each side varies with JIT timing + /// and the host's vector width (see `TestNVQEncodingBitsAcrossJitStates`). + @Test void preservesFrozenEncodingBitsAndLossValues() { var current=new PanamaVectorUtilSupport();var old=new Reference();var random=new Random(9117); for(int dim:new int[]{0,1,7,8,9,15,16,17,31,32,33,127,128,129,256,768,1536}) { float[] data=new float[dim];for(int i=0;i