diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 0000000..ada95e1 --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,64 @@ +name: Nightly fuzz + +# Separate from ci.yml on purpose: ci.yml is reused via workflow_call +# by release.yml specifically so the release gate and the PR gate never +# drift apart (see the comment at the top of ci.yml). Folding a +# schedule trigger into it would fire that whole ladder on a cron too, +# not just this exploratory job. +on: + schedule: + # Arbitrary; 03:00 UTC lands after most contributors' working day + # across common timezones, clear of any push/PR traffic. + - cron: "0 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + # tests/test_fuzz.py's own "netprotocols" profile is deliberately + # derandomize=True at 200 examples, so CI has run the *same* 200 + # inputs on every push since it was written. This job runs the whole + # suite under the "nightly" profile instead (10,000 examples, a real + # random seed each time — see tests/test_fuzz.py's module docstring), + # so it explores new ground every run rather than replaying the same + # counterexample search forever. Pinned to Python 3.12 only, unlike + # the "test" job's full matrix: this is an exploratory job, not a + # compatibility gate, and 3x the interpreters would 3x its cost for + # no extra coverage of what it is actually looking for. + fuzz: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.12" + # Keyed by run id with a prefix restore-key: a fixed key would + # restore forever but (actions/cache never overwrites an existing + # key) never actually save a new one, so nothing would + # accumulate. This restores the most recent run's cache and saves + # today's as a new entry, so Hypothesis's example database grows + # night over night instead of starting cold every time. + - name: Restore accumulated Hypothesis examples + uses: actions/cache@v4 + with: + path: .hypothesis + key: hypothesis-nightly-${{ github.run_id }} + restore-keys: | + hypothesis-nightly- + - name: Run the full suite under the nightly fuzz profile + env: + HYPOTHESIS_PROFILE: nightly + run: uv run --frozen pytest + # A failed scheduled run already notifies repo watchers by + # default (GitHub's standard behavior) — that red run *is* the + # report; nothing here files an issue on top of it. This just + # makes the minimal counterexample recoverable without needing + # repo write access to re-run the job locally. + - name: Upload the failing example database + if: failure() + uses: actions/upload-artifact@v4 + with: + name: hypothesis-nightly-failure-${{ github.run_id }} + path: .hypothesis + retention-days: 30 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a1591a..4bd6197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,6 +125,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 addresses, and disagrees with *itself* between Python 3.11 and 3.12). IPv4 addressing is unaffected; behavior for every existing platform is unchanged, exception type included (#99). +- **Nightly fuzzing with a moving seed.** `tests/test_fuzz.py`'s + `"netprotocols"` Hypothesis profile is deterministic on purpose (200 + examples, `derandomize=True`) so a pull request is reproducible — but + that also means every build since it was written has run the *same* + 200 inputs. A new `"nightly"` profile (10,000 examples, a real random + seed each run) runs the whole suite on a new schedule, + `.github/workflows/fuzz.yml` (03:00 UTC daily, plus manual dispatch), + kept out of `ci.yml` deliberately — `ci.yml` is reused by + `release.yml` via `workflow_call`, and a `schedule:` trigger there + would fire the whole PR/release gate on a cron, not just this + exploratory job. `.hypothesis/`'s example database accumulates + across nightly runs via `actions/cache` (keyed by run id with a + prefix restore-key, since a fixed key would restore forever but + never actually save a new entry) and uploads as a workflow artifact + on failure, so a counterexample is recoverable without repo write + access; a failed scheduled run's own red build is the notification + (GitHub already does this by default), so nothing here files an + issue on top of it. Reproduce locally: + `HYPOTHESIS_PROFILE=nightly uv run pytest`. Also adds a targeted + strategy building well-formed TCP SYN options (MSS, window scale, + SACK-Permitted, SACK, timestamps) — the real-capture corpus never + caught a SYN, so unlike NOP/Timestamps these otherwise depend on + `max_examples` alone stumbling into a well-formed TLV by chance + (#98). ## [2.0.0] - 2026-09-04 diff --git a/docs/CLAIMS.md b/docs/CLAIMS.md index 6fef8dc..26b5ab9 100644 --- a/docs/CLAIMS.md +++ b/docs/CLAIMS.md @@ -431,25 +431,25 @@ VLAN, GRE and DHCP topologies. Every frame's checksums verify internally (`scripts/check_fixtures.py`). ### 5.2 "Property-based fuzzing of every decoder" -**Status: PARTLY TRUE — precision required** - -What is true: Hypothesis fuzzing asserts that decoding never raises -outside `ProtocolError`, that chain walks terminate, that layers -recompose, that five TLV/name accessors never hang, and — as of #97 — -that `bytes(decode(x)) == x` is Hypothesis-generated for **all 18** -protocols, up from 4. `tests/strategies.py` holds one reusable strategy -per protocol; `tests/test_fuzz.py::TestGeneralizedRoundTrips` -parametrizes the property over all 14 that were missing it. Reproduce: -`uv run pytest tests/test_fuzz.py::TestGeneralizedRoundTrips -v`. +**Status: VERIFIED** -What is **not** yet true: CI still runs a fixed 200 examples with -`derandomize=True` (#98) — the same 200 inputs on every build since the -profile was written, for every property including this one. +Hypothesis fuzzing asserts that decoding never raises outside +`ProtocolError`, that chain walks terminate, that layers recompose, +that five TLV/name accessors never hang, and that `bytes(decode(x)) == +x` is Hypothesis-generated for **all 18** protocols, up from 4. +`tests/strategies.py` holds one reusable strategy per protocol; +`tests/test_fuzz.py::TestGeneralizedRoundTrips` parametrizes the +property over all 14 that were missing it (#97). Reproduce: +`uv run pytest tests/test_fuzz.py::TestGeneralizedRoundTrips -v`. -Until #98 lands, say "property-based fuzzing of the decode path, -including a universally-generated round-trip property", not "fuzzed -with a fresh seed on every run" — the corpus evidence in 5.1 does not -have this limitation and is strong on its own. +CI runs two Hypothesis profiles (#98): every push/PR runs a fixed 200 +examples with `derandomize=True`, so a landed PR's result is +reproducible; a scheduled `.github/workflows/fuzz.yml` additionally +runs the *entire* suite nightly under a `"nightly"` profile — 10,000 +examples, a real random seed each run — so fuzzing explores new +ground every night instead of replaying the same 200 inputs forever. +Reproduce a nightly-style run locally: +`HYPOTHESIS_PROFILE=nightly uv run pytest`. ### 5.3 "99% test coverage" **Status: VERIFIED** diff --git a/tests/test_fuzz.py b/tests/test_fuzz.py index b262c50..0738d6a 100644 --- a/tests/test_fuzz.py +++ b/tests/test_fuzz.py @@ -6,12 +6,24 @@ same rule. Strategies mix pure random bytes with mutations of real corpus frames (much better at reaching deep decode branches). -The hypothesis profile is deterministic (``derandomize=True``) so an -unrelated pull request can never trip a freshly discovered -counterexample; bump ``max_examples`` locally to explore. +Two Hypothesis profiles are registered: + +- ``"netprotocols"`` (the default, loaded unless overridden): 200 + examples, ``derandomize=True`` so an unrelated pull request can + never trip a freshly discovered counterexample. +- ``"nightly"``: 10,000 examples, a real random seed each run + (``derandomize=False``) — the profile ``.github/workflows/fuzz.yml`` + runs on a schedule, deliberately trading determinism for a moving + seed that explores new ground on every run instead of replaying the + same 200 inputs forever (#98). + +Select a profile with ``HYPOTHESIS_PROFILE`` (read once, at import +time) — e.g. ``HYPOTHESIS_PROFILE=nightly pytest tests/test_fuzz.py`` +reproduces a nightly run locally. """ import contextlib +import os import pytest from hypothesis import given, settings @@ -39,6 +51,7 @@ IPv6Routing, Packet, ProtocolError, + TCPOption, ) from strategies import ROUND_TRIP_STRATEGIES from test_corpus import walk @@ -46,7 +59,10 @@ settings.register_profile( "netprotocols", max_examples=200, deadline=None, derandomize=True ) -settings.load_profile("netprotocols") +settings.register_profile( + "nightly", max_examples=10_000, deadline=None, derandomize=False +) +settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "netprotocols")) ALL_PROTOCOLS = ( Ethernet, @@ -184,6 +200,101 @@ def test_ipv6_fragment( assert IPv6Fragment.decode(bytes(header)) == header +# TCP option kinds a real SYN carries (RFC 9293 §3.2; RFC 7323 for +# Window Scale and Timestamps; RFC 2018 for SACK). The real-capture +# corpus never caught a SYN (tests/fixtures/MANIFEST.md), so unlike +# NOP/Timestamps (kinds 1, 8 — exercised via other captured traffic), +# MSS/window-scale/SACK are otherwise fuzzed only as arbitrary bytes +# inside ``fuzz_input`` above, which essentially never lands on a +# well-formed TLV by chance. This strategy builds one on purpose. +_KIND_NOP = 1 +_KIND_MSS = 2 +_KIND_WINDOW_SCALE = 3 +_KIND_SACK_PERMITTED = 4 +_KIND_SACK = 5 +_KIND_TIMESTAMPS = 8 + + +@st.composite +def tcp_syn_options(draw: st.DrawFn) -> tuple[bytes, list[TCPOption]]: + """Well-formed SYN-shaped option bytes (MSS, window scale, + SACK-Permitted, timestamps, then a 1-2 block SACK, NOP-padded to a + multiple of 4), paired with the :class:`TCPOption` list + ``TCP.decode`` should produce from them.""" + mss = draw(st.integers(min_value=0, max_value=0xFFFF)) + shift = draw(st.integers(min_value=0, max_value=0xFF)) + tsval = draw(st.integers(min_value=0, max_value=0xFFFFFFFF)) + tsecr = draw(st.integers(min_value=0, max_value=0xFFFFFFFF)) + # Capped at 2 blocks (not RFC 2018's 4): MSS + window scale + + # SACK-Permitted + timestamps already spend 19 of the 40 bytes TCP's + # 4-bit data offset allows for options, leaving room for at most 2 + # 8-byte SACK blocks plus up-to-3 bytes of NOP padding. + blocks = draw( + st.lists( + st.tuples( + st.integers(min_value=0, max_value=0xFFFFFFFF), + st.integers(min_value=0, max_value=0xFFFFFFFF), + ), + min_size=1, + max_size=2, + ) + ) + sack_data = b"".join( + left.to_bytes(4, "big") + right.to_bytes(4, "big") + for left, right in blocks + ) + + raw = ( + bytes([_KIND_MSS, 4]) + + mss.to_bytes(2, "big") + + bytes([_KIND_WINDOW_SCALE, 3, shift]) + + bytes([_KIND_SACK_PERMITTED, 2]) + + bytes([_KIND_TIMESTAMPS, 10]) + + tsval.to_bytes(4, "big") + + tsecr.to_bytes(4, "big") + + bytes([_KIND_SACK, 2 + len(sack_data)]) + + sack_data + ) + padding = (-len(raw)) % 4 + raw += bytes([_KIND_NOP]) * padding + + expected = [ + TCPOption(kind=_KIND_MSS, data=mss.to_bytes(2, "big")), + TCPOption(kind=_KIND_WINDOW_SCALE, data=bytes([shift])), + TCPOption(kind=_KIND_SACK_PERMITTED), + TCPOption( + kind=_KIND_TIMESTAMPS, + data=tsval.to_bytes(4, "big") + tsecr.to_bytes(4, "big"), + ), + TCPOption(kind=_KIND_SACK, data=sack_data), + *([TCPOption(kind=_KIND_NOP)] * padding), + ] + return raw, expected + + +class TestTCPSynOptionsRoundTrip: + @given(drawn=tcp_syn_options()) + def test_synthesized_options_round_trip_and_decode( + self, drawn: tuple[bytes, list[TCPOption]] + ) -> None: + raw, expected_options = drawn + header = TCP( + src_port=1234, + dst_port=443, + seq=0, + ack=0, + data_offset=5 + len(raw) // 4, + reserved=0, + flags=0b0_0000_0010, # SYN + window=0xFFFF, + checksum=0, + urgent_pointer=0, + options=raw, + ) + assert TCP.decode(bytes(header)) == header + assert list(header.parsed_options) == expected_options + + class TestGeneralizedRoundTrips: """``bytes(decode(x)) == x`` for every protocol, not just the four above — one property, parametrized over the strategies in