diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index f4783ed..8a298db 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -130,7 +130,9 @@ public `Generator`/`TestCase`/`Generators`/`Hegel`/`Stateful` surface stays in ` the printing reporter prints it as `note: …`). A failure whose origin has no capture is an internal error. `Settings.reproduceFailure` drives the same loop over `hegel_run_start_blob` (the engine replays until a replay fails, under its budget): PASSED → `Runner.STALE_BLOB` - `HegelException`, ERROR → "blob is not valid" `HegelException`. There is no Java-side flaky + `HegelException`, ERROR → "blob replay ended in an error" `HegelException`; a blob the + engine cannot decode fails `hegel_run_start_blob` itself (0.44.1+) and surfaces as the binding's + `HegelException` with the engine's "could not be decoded" message. There is no Java-side flaky detection any more: the engine handles nondeterminism per `nondeterminismStrictness` (`NondeterminismStrictness.QUIET`/`WARN` confirm-by-replay and report with a caveat; `ERROR` aborts the run, surfacing as the engine's own `Flaky test detected` message). The runner returns a `RunReport` (status, client-side `RunStatistics` counted diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..ae8b928 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,22 @@ +RELEASE_TYPE: patch + +This patch upgrades the bundled libhegel engine from 0.44.0 to 0.45.0. + +`Generators.fromRegex` now generates patterns that contain the NUL character the way Python's `re` +reads them: `fromRegex("a\0b")` generates `"a\0b"`. Before, the pattern crossed into the engine as a +NUL-terminated string, so the NUL was replaced by U+FFFD on the way and the generated strings did not +match the pattern. The engine now takes the pattern as a length-delimited buffer, and the bindings +pass it that way. + +The engine also fixes several ways `fromRegex` could generate a string that does not match its +pattern as Python's `re` reads it, or reject a string it could have generated: under `(?i)`, +characters are compared by Python's case-folding rules, so `(?i)[^k]` no longer generates the Kelvin +sign; under `(?a)`, case-insensitivity folds only ASCII letters and explicit characters outside ASCII +are kept rather than dropped; and a repetition or alternation whose body the alphabet cannot supply +no longer trips the filter-too-much health check. + +A reproduce blob that the engine cannot decode (corrupt, or from an incompatible engine version) is +now rejected before any run starts: `Settings.reproduceFailure` and `@HegelTest(reproduceFailure = +...)` throw a `HegelException` whose message says the blob "could not be decoded", instead of starting +a run that ends in an error. Likewise, a run started while `ANTITHESIS_OUTPUT_DIR` names a directory +that does not exist fails up front with the engine's diagnostic. diff --git a/hegel-jna/src/main/java/dev/hegel/JnaLibhegel.java b/hegel-jna/src/main/java/dev/hegel/JnaLibhegel.java index a3cd87f..e6d641e 100644 --- a/hegel-jna/src/main/java/dev/hegel/JnaLibhegel.java +++ b/hegel-jna/src/main/java/dev/hegel/JnaLibhegel.java @@ -182,7 +182,12 @@ int hegel_string_generator_text( PointerByReference out); int hegel_string_generator_regex( - Pointer ctx, String pattern, byte fullmatch, Pointer alphabet, PointerByReference out); + Pointer ctx, + Pointer pattern, + long patternLen, + byte fullmatch, + Pointer alphabet, + PointerByReference out); int hegel_string_generator_email(Pointer ctx, PointerByReference out); @@ -811,8 +816,15 @@ private static Pointer cstrArray(List strings) { @Override public int stringGeneratorRegex(String pattern, boolean fullmatch, long alphabet, long[] out) { + byte[] bytes = utf8OrNull(pattern); PointerByReference outRef = new PointerByReference(); - int code = lib.hegel_string_generator_regex(ctx(), pattern, cbool(fullmatch), pointer(alphabet), outRef); + int code = lib.hegel_string_generator_regex( + ctx(), + bytesOrNull(bytes), + bytes == null ? 0L : bytes.length, + cbool(fullmatch), + pointer(alphabet), + outRef); out[0] = address(outRef.getValue()); return code; } diff --git a/hegel-jna/src/test/java/dev/hegel/JnaLibhegelTest.java b/hegel-jna/src/test/java/dev/hegel/JnaLibhegelTest.java index b7a81bc..98ae940 100644 --- a/hegel-jna/src/test/java/dev/hegel/JnaLibhegelTest.java +++ b/hegel-jna/src/test/java/dev/hegel/JnaLibhegelTest.java @@ -227,22 +227,45 @@ void undecodableBlobWithDefaultOutputIsRejected() { } @Test - void blobRunWithDefaultOutputStartsAndReportsTheBadBlob() { + void blobRunWithDefaultOutputReplaysTheBlob() { JnaLibhegel lib = real(); + RunReport report = Hegel.run( + tc -> { + tc.draw(Generators.integers(), "x"); + throw new AssertionError("boom"); + }, + new Settings().database(Database.disabled()).seed(1), + Reporter.silent()); + String blob = report.failures().get(0).reproduceBlob().orElseThrow(); long s = newSettings(lib); - // A null output callback leaves the run's output on stderr; the garbage blob surfaces as - // the run's error once it is pumped dry. - long run = lib.runStartBlob(s, "not-a-blob!!!", null); + lib.settingsDatabase(s, ""); + // A null output callback leaves the run's output on stderr. Marking every replay valid + // pumps the run dry without a failure, which is the engine's "stale blob" verdict. + long run = lib.runStartBlob(s, blob, null); assertNotEquals(0, run); - assertEquals(0, lib.nextTestCase(run)); + long tc; + while ((tc = lib.nextTestCase(run)) != 0) { + assertEquals(Abi.OK, lib.markComplete(tc, Abi.STATUS_VALID, null)); + lib.testCaseFree(tc); + } long result = lib.runResult(run); - assertEquals(Abi.RUN_STATUS_ERROR, lib.runResultStatus(result)); - assertNotNull(lib.runResultError(result)); + assertEquals(Abi.RUN_STATUS_PASSED, lib.runResultStatus(result)); lib.runResultFree(result); lib.runFree(run); lib.settingsFree(s); } + @Test + void undecodableBlobRunIsRejectedBeforeItStarts() { + JnaLibhegel lib = real(); + long s = newSettings(lib); + // Since libhegel 0.44.1 hegel_run_start_blob rejects a blob it cannot decode with + // E_INVALID_ARG instead of starting a run that ends in an error. + HegelException e = assertThrows(HegelException.class, () -> lib.runStartBlob(s, "not-a-blob!!!", null)); + assertTrue(e.getMessage().contains("could not be decoded"), e.getMessage()); + lib.settingsFree(s); + } + @Test void regexGeneratorAcceptsATextAlphabet() { JnaLibhegel lib = real(); @@ -255,4 +278,16 @@ void regexGeneratorAcceptsATextAlphabet() { lib.stringGeneratorFree(regex[0]); lib.stringGeneratorFree(alphabet[0]); } + + @Test + void regexPatternCrossesAsALengthDelimitedBuffer() { + // A NUL inside the pattern must survive the call (0.45 takes pointer + byte length), and + // a null pattern is the engine's E_INVALID_ARG rather than a crash. + JnaLibhegel lib = real(); + long[] regex = new long[1]; + assertEquals(Abi.OK, lib.stringGeneratorRegex("a\0b", true, 0, regex)); + assertTrue(regex[0] != 0); + lib.stringGeneratorFree(regex[0]); + assertEquals(Abi.E_INVALID_ARG, lib.stringGeneratorRegex(null, true, 0, new long[1])); + } } diff --git a/hegel-lowlevel/src/main/java/dev/hegel/lowlevel/Libhegel.java b/hegel-lowlevel/src/main/java/dev/hegel/lowlevel/Libhegel.java index 113528d..ddae3f9 100644 --- a/hegel-lowlevel/src/main/java/dev/hegel/lowlevel/Libhegel.java +++ b/hegel-lowlevel/src/main/java/dev/hegel/lowlevel/Libhegel.java @@ -165,9 +165,11 @@ static Libhegel load(Path library, Iterable backends) /** * {@code hegel_run_start_blob}: start a run that replays a reproduce blob until a replay fails * (under the engine's bounded budget) instead of exploring. Driven exactly like a run from - * {@link #runStart}: a reproducing replay is the run's failure, a run with no failures means - * the blob is stale, and an undecodable blob surfaces as the run's error from {@link - * #runResultError}. {@code output} has the same contract as in {@link #runStart}. + * {@link #runStart}: a reproducing replay is the run's failure and a run with no failures + * means the blob is stale. A blob the engine cannot decode (corrupt, or from an incompatible + * engine version) fails the call itself with {@code HEGEL_E_INVALID_ARG}, which the binding + * throws as a {@link LibhegelException} carrying the engine's message; no run is started. + * {@code output} has the same contract as in {@link #runStart}. */ long runStartBlob(long settings, String blob, Consumer output); @@ -274,6 +276,10 @@ int stringGeneratorText( String excludeCharacters, long[] out); + /** + * {@code hegel_string_generator_regex}. The pattern crosses as a length-delimited UTF-8 buffer, + * so it may contain the NUL character, which Python's {@code re} accepts. + */ int stringGeneratorRegex(String pattern, boolean fullmatch, long alphabet, long[] out); int stringGeneratorEmail(long[] out); diff --git a/hegel/src/main/java/dev/hegel/RealLibhegel.java b/hegel/src/main/java/dev/hegel/RealLibhegel.java index ad99803..08645cd 100644 --- a/hegel/src/main/java/dev/hegel/RealLibhegel.java +++ b/hegel/src/main/java/dev/hegel/RealLibhegel.java @@ -373,7 +373,7 @@ private static MethodHandle findEmitLine() { linker, lookup, "hegel_string_generator_regex", - FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS, JAVA_BOOLEAN, ADDRESS, ADDRESS)); + FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS, JAVA_LONG, JAVA_BOOLEAN, ADDRESS, ADDRESS)); this.stringGeneratorEmail = h(linker, lookup, "hegel_string_generator_email", FunctionDescriptor.of(JAVA_INT, ADDRESS, ADDRESS)); this.stringGeneratorUrl = @@ -1030,8 +1030,17 @@ private static MemorySegment cstrArray(Arena arena, List strings) { @Override public int stringGeneratorRegex(String pattern, boolean fullmatch, long alphabet, long[] out) { try (Arena arena = Arena.ofConfined()) { + // The pattern crosses as a length-delimited UTF-8 buffer (libhegel 0.45), so it may + // contain NUL, which Python's `re` accepts. + byte[] bytes = utf8OrNull(pattern); MemorySegment outSeg = arena.allocate(ADDRESS); - int code = rc(stringGeneratorRegex, cstr(arena, pattern), fullmatch, segment(alphabet), outSeg); + int code = rc( + stringGeneratorRegex, + bytesOrNull(arena, bytes), + bytes == null ? 0L : (long) bytes.length, + fullmatch, + segment(alphabet), + outSeg); out[0] = outSeg.get(ADDRESS, 0).address(); return code; } diff --git a/hegel/src/test/java/dev/hegel/RealLibhegelTest.java b/hegel/src/test/java/dev/hegel/RealLibhegelTest.java index 52c84a1..4492b00 100644 --- a/hegel/src/test/java/dev/hegel/RealLibhegelTest.java +++ b/hegel/src/test/java/dev/hegel/RealLibhegelTest.java @@ -224,22 +224,45 @@ void undecodableBlobWithDefaultOutputIsRejected() { } @Test - void blobRunWithDefaultOutputStartsAndReportsTheBadBlob() { + void blobRunWithDefaultOutputReplaysTheBlob() { RealLibhegel lib = real(); + RunReport report = Hegel.run( + tc -> { + tc.draw(Generators.integers(), "x"); + throw new AssertionError("boom"); + }, + new Settings().database(Database.disabled()).seed(1), + Reporter.silent()); + String blob = report.failures().get(0).reproduceBlob().orElseThrow(); long s = newSettings(lib); - // A null output callback leaves the run's output on stderr; the garbage blob surfaces as - // the run's error once it is pumped dry. - long run = lib.runStartBlob(s, "not-a-blob!!!", null); + lib.settingsDatabase(s, ""); + // A null output callback leaves the run's output on stderr. Marking every replay valid + // pumps the run dry without a failure, which is the engine's "stale blob" verdict. + long run = lib.runStartBlob(s, blob, null); assertNotEquals(0, run); - assertEquals(0, lib.nextTestCase(run)); + long tc; + while ((tc = lib.nextTestCase(run)) != 0) { + assertEquals(Abi.OK, lib.markComplete(tc, Abi.STATUS_VALID, null)); + lib.testCaseFree(tc); + } long result = lib.runResult(run); - assertEquals(Abi.RUN_STATUS_ERROR, lib.runResultStatus(result)); - assertNotNull(lib.runResultError(result)); + assertEquals(Abi.RUN_STATUS_PASSED, lib.runResultStatus(result)); lib.runResultFree(result); lib.runFree(run); lib.settingsFree(s); } + @Test + void undecodableBlobRunIsRejectedBeforeItStarts() { + RealLibhegel lib = real(); + long s = newSettings(lib); + // Since libhegel 0.44.1 hegel_run_start_blob rejects a blob it cannot decode with + // E_INVALID_ARG instead of starting a run that ends in an error. + HegelException e = assertThrows(HegelException.class, () -> lib.runStartBlob(s, "not-a-blob!!!", null)); + assertTrue(e.getMessage().contains("could not be decoded"), e.getMessage()); + lib.settingsFree(s); + } + @Test void regexGeneratorAcceptsATextAlphabet() { RealLibhegel lib = real(); @@ -252,4 +275,16 @@ void regexGeneratorAcceptsATextAlphabet() { lib.stringGeneratorFree(regex[0]); lib.stringGeneratorFree(alphabet[0]); } + + @Test + void regexPatternCrossesAsALengthDelimitedBuffer() { + // A NUL inside the pattern must survive the call (0.45 takes pointer + byte length), and + // a null pattern is the engine's E_INVALID_ARG rather than a crash. + RealLibhegel lib = real(); + long[] regex = new long[1]; + assertEquals(Abi.OK, lib.stringGeneratorRegex("a\0b", true, 0, regex)); + assertTrue(regex[0] != 0); + lib.stringGeneratorFree(regex[0]); + assertEquals(Abi.E_INVALID_ARG, lib.stringGeneratorRegex(null, true, 0, new long[1])); + } } diff --git a/pom.xml b/pom.xml index b579f29..4334525 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,7 @@ - 0.44.0 + 0.45.0 false diff --git a/shared/src/main/java/dev/hegel/Runner.java b/shared/src/main/java/dev/hegel/Runner.java index 4dba0f8..7d5a0bb 100644 --- a/shared/src/main/java/dev/hegel/Runner.java +++ b/shared/src/main/java/dev/hegel/Runner.java @@ -166,11 +166,11 @@ private static RunReport finish( return new RunReport(RunStatus.PASSED, counts.snapshot(), null, List.of()); case Abi.RUN_STATUS_ERROR: // The run produced no verdict on the property: a failed health check, a - // nondeterminism abort under ERROR strictness, an engine panic — or, on a blob - // replay, a blob the engine could not decode. + // nondeterminism abort under ERROR strictness, an engine panic. (A blob the engine + // cannot decode never gets this far: hegel_run_start_blob rejects it.) String error = nullToEmpty(lib.runResultError(result)); if (replayingBlob) { - throw new HegelException("reproduceFailure: the supplied blob is not valid: " + error); + throw new HegelException("reproduceFailure: the blob replay ended in an error: " + error); } return new RunReport(RunStatus.ERROR, counts.snapshot(), error, List.of()); default: diff --git a/shared/src/test/java/dev/hegel/ConformanceTest.java b/shared/src/test/java/dev/hegel/ConformanceTest.java index 2ed23f4..97ab607 100644 --- a/shared/src/test/java/dev/hegel/ConformanceTest.java +++ b/shared/src/test/java/dev/hegel/ConformanceTest.java @@ -213,6 +213,13 @@ void regexDefaultsToFullmatch() { findAny(fromRegex("[0-9]{3}").fullmatch(false), s -> !s.matches("[0-9]{3}")); } + @Test + void regexPatternsMayContainNul() { + // The pattern crosses the ABI as a length-delimited buffer (libhegel 0.45), so a NUL in + // it is generated, not replaced by U+FFFD. + assertAllExamples(fromRegex("a\0b"), s -> s.equals("a\0b")); + } + @Test void temporalGeneratorsProduceJavaTimeTypes() { // Typed (not strings): the engine's offset-free output parses into java.time values, so a draw diff --git a/shared/src/test/java/dev/hegel/ReproduceFailureTest.java b/shared/src/test/java/dev/hegel/ReproduceFailureTest.java index 9b9549e..90259c8 100644 --- a/shared/src/test/java/dev/hegel/ReproduceFailureTest.java +++ b/shared/src/test/java/dev/hegel/ReproduceFailureTest.java @@ -82,7 +82,7 @@ void corruptBlobsAreRejected() { HegelException.class, () -> Hegel.test( FAILING, new Settings().database(Database.disabled()).reproduceFailure("!!!"))); - assertTrue(e.getMessage().contains("not valid"), e.getMessage()); + assertTrue(e.getMessage().contains("could not be decoded"), e.getMessage()); } @Test diff --git a/shared/src/test/java/dev/hegel/RunnerTest.java b/shared/src/test/java/dev/hegel/RunnerTest.java index bc85704..7c85bbc 100644 --- a/shared/src/test/java/dev/hegel/RunnerTest.java +++ b/shared/src/test/java/dev/hegel/RunnerTest.java @@ -320,15 +320,14 @@ void reproduceFailureReportsAStaleBlob() { } @Test - void reproduceFailureRejectsAnInvalidBlob() { - // An undecodable blob surfaces as the blob run's error. + void reproduceFailureSurfacesAnErroredReplay() { FakeLibhegel fake = new FakeLibhegel(); fake.runStatus = Abi.RUN_STATUS_ERROR; - fake.runError = "corrupt"; + fake.runError = "engine panicked"; HegelException e = assertThrows(HegelException.class, () -> run(fake, new Settings().reproduceFailure("???"), tc -> {})); - assertTrue(e.getMessage().contains("not valid"), e.getMessage()); - assertTrue(e.getMessage().contains("corrupt"), e.getMessage()); + assertTrue(e.getMessage().contains("ended in an error"), e.getMessage()); + assertTrue(e.getMessage().contains("engine panicked"), e.getMessage()); } @Test