Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions .github/workflows/unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ jobs:
allow-prereleases: ${{ matrix.experimental }}
cache: pip

# The Scapy extra is there for examples/samples/pcap.py and legacy.py,
# which build their captures with scapy; examples/samples/pcapng.py needs
# The Scapy extra is there for examples/generators/pcap.py and legacy.py,
# which build their captures with scapy; examples/generators/pcapng.py needs
# nothing beyond the standard library and pcapkit itself, which it uses to
# parse each fixture back.
- name: Install package, test and generator dependencies
Expand All @@ -100,12 +100,12 @@ jobs:
# Keep a failure here legible as what it is -- the fixtures could not
# be built, so no test has run yet and nothing is wrong with the code
# under test.
if ! python examples/samples/make_samples.py 2>&1 | tee "$RUNNER_TEMP/make-samples.log"; then
echo "::error title=Sample fixture generation failed::examples/samples/make_samples.py could not rebuild examples/sample/. This is a fixture-generation failure, not a test failure -- the test suite has not run."
if ! python examples/generators/make_samples.py 2>&1 | tee "$RUNNER_TEMP/make-samples.log"; then
echo "::error title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite has not run."
exit 1
fi

# examples/samples/pcapng.py fetches two captures from the Wireshark
# examples/generators/pcapng.py fetches two captures from the Wireshark
# repository and synthesises stand-ins when the download fails. The
# stand-ins parse, and tests/protocols/test_pcapng_regression.py only
# asserts that extraction succeeds, so a degraded run goes green and
Expand Down Expand Up @@ -143,8 +143,8 @@ jobs:
- name: Regenerate sample captures
shell: bash
run: |
if ! python examples/samples/make_samples.py 2>&1 | tee "$RUNNER_TEMP/make-samples.log"; then
echo "::error title=Sample fixture generation failed::examples/samples/make_samples.py could not rebuild examples/sample/. This is a fixture-generation failure, not a test failure -- the test suite has not run."
if ! python examples/generators/make_samples.py 2>&1 | tee "$RUNNER_TEMP/make-samples.log"; then
echo "::error title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite has not run."
exit 1
fi

Expand Down
16 changes: 8 additions & 8 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,18 @@
pcapkit-*-tempdir/
pcapkit-temp.html

examples/sample/*
!examples/sample/out.json
!examples/sample/out.plist
!examples/sample/out.txt
!examples/sample/in.pcap
!examples/sample/dhcp.pcapng
!examples/sample/pcapng.txt
examples/captures/*
!examples/captures/out.json
!examples/captures/out.plist
!examples/captures/out.txt
!examples/captures/in.pcap
!examples/captures/dhcp.pcapng
!examples/captures/pcapng.txt

src
!requirements.txt

examples/sample/test
examples/captures/test
test/pcapkit
test/dictdumper
deprecated/
Expand Down
1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ prune .eggs
prune .github
prune .venv
prune docs
prune examples/captures
Comment thread
JarryShaw marked this conversation as resolved.
prune examples/sample
prune temp
prune test
Expand Down
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,10 @@ pipenv:
vendor:
pipenv run pcapkit-vendor

# Sample captures under examples/sample/ are not tracked (see .gitignore);
# Sample captures under examples/captures/ are not tracked (see .gitignore);
# regenerate the ones the runtime, regression and integration tests read.
samples:
pipenv run python examples/samples/make_samples.py
pipenv run python examples/generators/make_samples.py

# Mirrors the selection run by .github/workflows/unit-tests.yml, i.e. the tests
# that need no sample captures beyond the committed ones.
Expand All @@ -99,7 +99,7 @@ docs-autobuild:
isort:
pipenv run isort -l100 -ppcapkit --skip-glob '**/__init__.py' pcapkit $(wildcard temp/sort.py)
pipenv run isort -l100 -ppcapkit pcapkit/{const,vendor}/*/*.py
pipenv run isort -l100 -ppcapkit util/*.py examples/samples/*.py
pipenv run isort -l100 -ppcapkit util/*.py examples/generators/*.py

vermin:
mkdir -p temp
Expand Down
6 changes: 3 additions & 3 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -187,16 +187,16 @@ tracked in the repository:

The runtime, regression and integration tests additionally read sample captures
that are **not** tracked (see ``.gitignore``);
``examples/samples/make_samples.py`` reconstructs them into ``examples/sample/``,
``examples/generators/make_samples.py`` reconstructs them into ``examples/captures/``,
and ``make test-all`` regenerates them before running the whole suite:

.. code-block:: shell

make samples # write examples/sample/*.pcap and *.pcapng
make samples # write examples/captures/*.pcap and *.pcapng
make test-all # regenerate the fixtures, then run every test

The same fixtures back the demonstration scripts in
``examples/legacy_smoke/``, which read them as ``../sample/…``.
``examples/legacy_smoke/``, which read them as ``../captures/…``.

Continuous integration runs the ``make test`` selection, since the fixtures are
not in the repository. ``tshark`` is only required to exercise the PyShark
Expand Down
2 changes: 1 addition & 1 deletion docs/source/pcapkit/foundation/reassembly/tcp.rst
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ Data Models
:members:
:show-inheritance:

.. autoclass:: pcapkit.foundation.reassembly.data.tcp.HoleDiscriptor
.. autoclass:: pcapkit.foundation.reassembly.data.tcp.HoleDescriptor
:members:
:show-inheritance:

Expand Down
2 changes: 1 addition & 1 deletion docs/source/pep.rst
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ Test Cases
(84 modules), bundled with the distribution, and it runs in CI against Python
3.10 through 3.14 (see ``.github/workflows/unit-tests.yml``). The sample
captures the runtime, regression and integration tiers read are not tracked in
git, so ``examples/samples/make_samples.py`` (``make samples``) rebuilds them
git, so ``examples/generators/make_samples.py`` (``make samples``) rebuilds them
from source.

What remains wanted is coverage rather than infrastructure: the protocols and
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
10 changes: 5 additions & 5 deletions examples/samples/legacy.py → examples/generators/legacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"""Generate the extra sample captures the legacy smoke scripts read.

The demonstration scripts under ``examples/legacy_smoke/`` predate the test
suite and read their captures out of ``examples/sample/`` by relative path.
suite and read their captures out of ``examples/captures/`` by relative path.
Most of them read fixtures the sibling generators in this directory already
write, but two read captures that exist nowhere in the repository, so those two
scripts cannot run at all on a fresh checkout. This module writes them, with
Expand Down Expand Up @@ -107,7 +107,7 @@
#: Repository root, i.e. the grandparent of the directory holding this file.
ROOT = pathlib.Path(__file__).resolve().parents[2]
#: Default destination directory for the generated captures.
SAMPLE = ROOT / 'examples' / 'sample'
SAMPLE = ROOT / 'examples' / 'captures'

#: Capture start time, fixed so that regenerating gives identical files.
EPOCH = 1500000000.0
Expand Down Expand Up @@ -605,7 +605,7 @@ def rst(self, to_server: 'bool' = True) -> 'Packet':


###############################################################################
# examples/sample/test.pcap
# examples/captures/test.pcap
###############################################################################

#: Client of both connections in ``test.pcap``, and of ``http6.cap``.
Expand Down Expand Up @@ -725,7 +725,7 @@ def _write_test(dest: 'pathlib.Path') -> 'pathlib.Path':


###############################################################################
# examples/sample/http6.cap
# examples/captures/http6.cap
###############################################################################

#: The IPv6 web server of ``http6.cap``, and the name it answers to.
Expand Down Expand Up @@ -807,7 +807,7 @@ def generate(dest: 'pathlib.Path | None' = None) -> 'list[pathlib.Path]':
"""Write the legacy-smoke sample fixtures.

Args:
dest: Destination directory; ``examples/sample/`` under the repository
dest: Destination directory; ``examples/captures/`` under the repository
root, if not given. Created if it does not exist.

Returns:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
# -*- coding: utf-8 -*-
"""Regenerate the sample captures used by the test suite and the examples.

The captures under ``examples/sample/`` are not tracked in git (see
The captures under ``examples/captures/`` are not tracked in git (see
``.gitignore``), yet the runtime, regression and integration tests read them and
pin their contents, as do the demonstration scripts in
``examples/legacy_smoke/``. This script rebuilds the whole set, so a fresh clone
can run ``pytest`` without any ignore flags:

.. code-block:: shell

python examples/samples/make_samples.py # or: make samples
python examples/generators/make_samples.py # or: make samples

The fixtures themselves come from the sibling modules in this directory, each of
which may also be run on its own:
Expand Down Expand Up @@ -43,7 +43,7 @@
#: Directory holding this script and its sibling generator modules.
HERE = pathlib.Path(__file__).resolve().parent
#: Destination directory for every generated capture.
DEST = ROOT / 'examples' / 'sample'
DEST = ROOT / 'examples' / 'captures'
#: Generator modules, in the order they are run.
GENERATORS = ('pcap', 'pcapng', 'legacy')

Expand Down Expand Up @@ -73,7 +73,7 @@ def load(name: 'str') -> 'ModuleType':


def main() -> 'int':
"""Write every sample capture into ``examples/sample/``."""
"""Write every sample capture into ``examples/captures/``."""
DEST.mkdir(parents=True, exist_ok=True)

written = [] # type: list[pathlib.Path]
Expand Down
18 changes: 9 additions & 9 deletions examples/samples/pcap.py → examples/generators/pcap.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""Generate the ``.pcap`` sample captures used by the test suite.

The tests under ``tests/`` extract capture files from ``examples/sample/``, but
The tests under ``tests/`` extract capture files from ``examples/captures/``, but
``.gitignore`` keeps that directory empty except for a handful of committed
files, so a fresh checkout cannot run them. This module rebuilds the missing
``.pcap`` fixtures with :mod:`scapy`, deterministically and without network
Expand Down Expand Up @@ -110,7 +110,7 @@
#: Repository root, i.e. the grandparent of the directory holding this file.
ROOT = pathlib.Path(__file__).resolve().parents[2]
#: Default destination directory for the generated captures.
SAMPLE = ROOT / 'examples' / 'sample'
SAMPLE = ROOT / 'examples' / 'captures'

#: Capture start time, fixed so that regenerating gives identical files.
EPOCH = 1500000000.0
Expand Down Expand Up @@ -462,7 +462,7 @@ def fin(self, to_server: 'bool' = True, *, options: 'Optional[Options]' = None,


###############################################################################
# examples/sample/arp.pcap
# examples/captures/arp.pcap
###############################################################################


Expand Down Expand Up @@ -497,7 +497,7 @@ def _write_arp(dest: 'pathlib.Path') -> 'pathlib.Path':


###############################################################################
# examples/sample/ipv4.pcap
# examples/captures/ipv4.pcap
###############################################################################


Expand Down Expand Up @@ -534,7 +534,7 @@ def build(index: 'int', word: 'int') -> 'Packet':


###############################################################################
# examples/sample/ipv6.pcap
# examples/captures/ipv6.pcap
###############################################################################


Expand Down Expand Up @@ -604,7 +604,7 @@ def advertise() -> 'Packet':


###############################################################################
# examples/sample/tcp.pcap
# examples/captures/tcp.pcap
###############################################################################


Expand Down Expand Up @@ -661,7 +661,7 @@ def _write_tcp(dest: 'pathlib.Path') -> 'pathlib.Path':


###############################################################################
# examples/sample/stream.pcap
# examples/captures/stream.pcap
###############################################################################

#: Service types a Bonjour client looks for when hunting media receivers. The
Expand Down Expand Up @@ -717,7 +717,7 @@ def query(word: 'int') -> 'Packet':


###############################################################################
# examples/sample/http.pcap
# examples/captures/http.pcap
###############################################################################

#: Hosts the generated page load talks to, and the address each resolved to.
Expand Down Expand Up @@ -1117,7 +1117,7 @@ def generate(dest: 'pathlib.Path | None' = None) -> 'list[pathlib.Path]':
"""Write the ``.pcap`` sample fixtures.

Args:
dest: Destination directory; ``examples/sample/`` under the repository
dest: Destination directory; ``examples/captures/`` under the repository
root, if not given. Created if it does not exist.

Returns:
Expand Down
24 changes: 12 additions & 12 deletions examples/samples/pcapng.py → examples/generators/pcapng.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"""Provision the ``.pcapng`` sample fixtures used by the test suite.

The test suite (``tests/protocols/test_pcapng_regression.py``) reads five
PCAP-NG captures out of ``examples/sample/`` that :file:`.gitignore` deliberately keeps
PCAP-NG captures out of ``examples/captures/`` that :file:`.gitignore` deliberately keeps
out of the repository. This module puts them back on any machine, without a
checkout of anything private, by two routes:

Expand All @@ -13,7 +13,7 @@
console output and the return value say so rather than pretending otherwise.
* **Synthesised** -- written byte by byte by :class:`_Blocks` below, with no
network access whatsoever, from packet payloads carried in the committed
``examples/sample/dhcp.pcapng`` fixture. Generation is deterministic: the same input
``examples/captures/dhcp.pcapng`` fixture. Generation is deterministic: the same input
tree always produces the same bytes, so a second run is a no-op.

Which fixture comes from where:
Expand All @@ -33,15 +33,15 @@

Both downloads come from the Wireshark source tree's ``test/captures``
directory (https://gitlab.com/wireshark/wireshark, mirrored on GitHub), which
is distributed under the GNU GPL v2 or later. They are fetched into ``examples/sample/``,
is distributed under the GNU GPL v2 or later. They are fetched into ``examples/captures/``,
which :file:`.gitignore` excludes, so this project never redistributes them --
each machine fetches its own copy. ``many_interfaces.pcapng`` does not exist
upstream under that name; upstream ships a three-file ring-buffer set, and the
first member (``many_interfaces.pcapng.1``) is a complete, self-contained
PCAP-NG file, so that is what is fetched and stored under the name the test
expects.

The committed ``examples/sample/dhcp.pcapng`` is byte-identical to upstream's
The committed ``examples/captures/dhcp.pcapng`` is byte-identical to upstream's
``test/captures/dhcp.pcapng`` (SHA-256 ``e47f667c...5a1666``), which is where
the synthesised DHCP payloads come from.

Expand Down Expand Up @@ -117,8 +117,8 @@

.. code-block:: shell

python examples/samples/pcapng.py # write into examples/sample
python examples/samples/pcapng.py /tmp/fix # write somewhere else
python examples/generators/pcapng.py # write into examples/captures
python examples/generators/pcapng.py /tmp/fix # write somewhere else

"""

Expand All @@ -141,7 +141,7 @@
#: Repository root, i.e. the grandparent of the directory holding this script.
ROOT = pathlib.Path(__file__).resolve().parents[2]
#: Default destination directory for the fixtures.
SAMPLE_DIR = ROOT / 'examples' / 'sample'
SAMPLE_DIR = ROOT / 'examples' / 'captures'
#: Committed fixture the synthesised DHCP payloads are lifted from.
DHCP_SOURCE = SAMPLE_DIR / 'dhcp.pcapng'

Expand Down Expand Up @@ -524,7 +524,7 @@ def build_dhcp(endian: 'str') -> 'bytes':
(OPT_COMMENT, f'DHCP exchange in a {writer.name} section'.encode('utf-8')),
(SHB_HARDWARE, b'synthetic'),
(SHB_OS, b'synthetic capture host'),
(SHB_USERAPPL, b'examples/samples/pcapng.py'),
(SHB_USERAPPL, b'examples/generators/pcapng.py'),
])]
blocks.append(writer.idb(LINKTYPE_ETHERNET, 0x0004_0000, [
(IF_NAME, b'eth0'),
Expand Down Expand Up @@ -559,7 +559,7 @@ def build_test() -> 'bytes':
(OPT_COMMENT, b'auxiliary PCAP-NG block types, section 1 of 2'),
(SHB_HARDWARE, b'synthetic'),
(SHB_OS, b'synthetic capture host'),
(SHB_USERAPPL, b'examples/samples/pcapng.py'),
(SHB_USERAPPL, b'examples/generators/pcapng.py'),
])]
blocks.append(writer.idb(LINKTYPE_ETHERNET, 0x0004_0000,
_interface_profile(writer, 'eth0', 'primary Ethernet interface',
Expand Down Expand Up @@ -602,7 +602,7 @@ def build_test() -> 'bytes':
# Second section, with its own interface table.
blocks.append(writer.shb([
(OPT_COMMENT, b'auxiliary PCAP-NG block types, section 2 of 2'),
(SHB_USERAPPL, b'examples/samples/pcapng.py'),
(SHB_USERAPPL, b'examples/generators/pcapng.py'),
]))
blocks.append(writer.idb(LINKTYPE_ETHERNET, 0x0004_0000, [
(IF_NAME, b'eth1'),
Expand Down Expand Up @@ -650,7 +650,7 @@ def build_many_interfaces() -> 'bytes':
(OPT_COMMENT, f'{len(interfaces)} interfaces in one section'.encode('utf-8')),
(SHB_HARDWARE, b'synthetic'),
(SHB_OS, b'synthetic capture host'),
(SHB_USERAPPL, b'examples/samples/pcapng.py'),
(SHB_USERAPPL, b'examples/generators/pcapng.py'),
])]
for index, (name, description, linktype) in enumerate(interfaces):
blocks.append(writer.idb(linktype, 0x0004_0000, [
Expand Down Expand Up @@ -703,7 +703,7 @@ def build_profile() -> 'bytes':
(OPT_COMMENT, b'interface profile and statistics'),
(SHB_HARDWARE, b'synthetic'),
(SHB_OS, b'synthetic capture host'),
(SHB_USERAPPL, b'examples/samples/pcapng.py'),
(SHB_USERAPPL, b'examples/generators/pcapng.py'),
])]
blocks.append(writer.idb(LINKTYPE_ETHERNET, 0x0004_0000,
_interface_profile(writer, 'eth0',
Expand Down
Loading