diff --git a/pcapkit/toolkit/pyshark.py b/pcapkit/toolkit/pyshark.py index e28b230393..a7a1ba37d0 100644 --- a/pcapkit/toolkit/pyshark.py +++ b/pcapkit/toolkit/pyshark.py @@ -145,7 +145,7 @@ 108: Enum_LinkType.MOST, # most 109: Enum_LinkType.CAN20B, # can20b 111: Enum_LinkType.X2E_SERIAL, # x2e-serial - 112: Enum_LinkType.IPMB_LINUX, # i2c-linux + 112: Enum_LinkType.I2C_LINUX, # i2c-linux 113: Enum_LinkType.IEEE802_15_4_NONASK_PHY, # wpan-nonask-phy 115: Enum_LinkType.USB_LINUX_MMAPPED, # usb-linux-mmap 121: Enum_LinkType.FC_2, # fc2 diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index 45fe792438..185a2f9942 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -237,6 +237,17 @@ 'asks whether /proc/self/fd is a directory, i.e. whether this kernel ' 'exposes the procfs file-descriptor listing -- nothing pip installs' ), + # tests/project/test_pyshark_encap_map.py (#851/#853). Asks whether the + # tshark/editcap binaries are on PATH -- a Wireshark install, not a Python + # distribution any pyproject.toml extra could ever provide. Declared here + # rather than left as an unprefixed shutil.which check: the latter was + # tried first and made the AST scan blind to two classes skipping on every + # CI leg, which is the opposite of what "no unaccounted HAS_* gate" was + # meant to prevent. + 'HAS_WIRESHARK': ( + 'asks whether the tshark/editcap binaries are on PATH -- a Wireshark ' + 'install, not a Python distribution any extra could provide' + ), } #: Top-level import names two (or more) genuinely *mutually exclusive* diff --git a/tests/project/test_pyshark_encap_map.py b/tests/project/test_pyshark_encap_map.py new file mode 100644 index 0000000000..5b47d723c7 --- /dev/null +++ b/tests/project/test_pyshark_encap_map.py @@ -0,0 +1,653 @@ +# -*- coding: utf-8 -*- +"""Tests for :file:`util/pyshark_encap_map.py`, the pyshark table generator. + +GitHub issue #851. ``ENCAP_TYPE_TO_LINKTYPE`` and ``FILTER_NAME_TO_LINKTYPE`` in +:mod:`pcapkit.toolkit.pyshark` were measured, not transcribed, by sweeping every +encapsulation :program:`editcap` accepts through :program:`editcap`/:program:`tshark` +4.6.9. This generator is that sweep, made runnable; these tests pin the two traps +that cost real time when the tables were first produced (a PDML ``showname`` +holding a ``/``, and the bogus token in ``editcap -T``'s own help banner), the +formatting that has to be byte-stable for a re-run to change nothing, and the +two exclusion rules (an unmapped DLT, an ambiguous filter name) generically -- +against the real :class:`~pcapkit.const.reg.linktype.LinkType` registry, whose +``_missing_`` mints a placeholder member for *any* unknown value rather than +raising, which is itself worth pinning (#575). + +Most of this is unit-tier: it exercises the parsing and formatting helpers +against fabricated PDML/help-text fragments and a fabricated measurement list, +never shelling out to :program:`editcap`/:program:`tshark`. Two classes do +shell out, both gated on :data:`HAS_WIRESHARK` -- a real ``HAS_*`` flag, +deliberately, rather than a same-effect :func:`shutil.which` check under an +unprefixed name. The latter was this file's first cut, on the theory that a +name ``tests/_dependency_gates.py``'s AST scan does not recognise cannot need +an entry there -- true, but the wrong lesson: it also means the scan cannot +see the gate at all, so a real skip on every CI leg (Wireshark is not +installed on most of them) went unreported as anything. ``HAS_WIRESHARK`` is +declared in :data:`~tests._dependency_gates.NON_DISTRIBUTION_FLAGS`, next to +the existing ``HAS_PROC_FD`` precedent for "a flag that asks about something +pip cannot install at all" -- see that module for the declaration, and +:func:`tests.test_tier_guard.DependencyGateCoverageTests +.test_every_gated_flag_is_classified` for the check that would fail if it were +missing. + +* :class:`RealSweepTests` asserts the exact 4.6.9 sweep arithmetic (226 + accepted, 157 writable, 69 refused) and byte-identical reproduction of the + committed tables. It additionally requires :data:`MEASURED_WIRESHARK_VERSION` + itself, because those counts are properties of *that* Wireshark release, not + of the sweep method -- Ubuntu noble's packaged 4.2.2, what CI's runners + install, accepts 224 encapsulations, not 226, and asserting 226 there is a + test bug, not a generator bug (found the hard way: PR #853's first CI run). +* :class:`VersionIndependentInvariantTests` runs the same real sweep on + *whatever* Wireshark is on ``PATH`` and checks properties that hold + regardless of how many encapsulations that copy accepts: no + ``frame.encap_type`` key resolves to two different DLTs, no DLT resolves to + two different keys, and every key or filter name the local sweep *did* + measure agrees with the committed table rather than merely being present or + absent from it. This is what still exercises the real binaries on the CI + legs :class:`RealSweepTests` skips. + +""" + +from __future__ import annotations + +import importlib.util +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +from pcapkit.const.reg.linktype import LinkType as Enum_LinkType + +ROOT = pathlib.Path(__file__).resolve().parents[2] + + +def _load_generator(): + """Load :file:`util/pyshark_encap_map.py` as a module. + + ``util/`` is a directory of scripts rather than a package, so there is no + import path to it -- the same reason :file:`test_changelog_md.py` loads + :file:`util/changelog_md.py` this way. + + """ + path = ROOT / 'util' / 'pyshark_encap_map.py' + spec = importlib.util.spec_from_file_location('pyshark_encap_map', path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +encap_map = _load_generator() + +#: The real :program:`editcap`/:program:`tshark` binaries, resolved once and +#: kept as the resolved paths -- what :func:`~pyshark_encap_map.list_encap_types` +#: and :func:`~pyshark_encap_map.measure_encap` actually need to invoke them. +_EDITCAP = shutil.which('editcap') +_TSHARK = shutil.which('tshark') + +#: Whether Wireshark is on ``PATH`` at all. Named with the ``HAS_`` prefix +#: deliberately: ``tests/_dependency_gates.py``'s AST scan keys on exactly that +#: prefix inside a ``skipUnless(...)`` call, and it needs to *see* this gate +#: darkening two classes on every CI leg -- see the module docstring for why +#: an unprefixed same-effect name was the wrong fix. Declared in +#: :data:`~tests._dependency_gates.NON_DISTRIBUTION_FLAGS` rather than +#: :data:`~tests._dependency_gates.MODULE_PROVIDERS`, since a Wireshark binary +#: is not something any ``pyproject.toml`` extra could ever install. +HAS_WIRESHARK = _EDITCAP is not None and _TSHARK is not None + +#: The one Wireshark release ``ENCAP_TYPE_TO_LINKTYPE`` and +#: ``FILTER_NAME_TO_LINKTYPE`` were measured against, and the only one +#: :class:`RealSweepTests` trusts to reproduce their exact counts. Named here, +#: singly, so a future reader greps one constant rather than three magic +#: numbers scattered across assertions. +MEASURED_WIRESHARK_VERSION = '4.6.9' + +#: ``tshark --version``'s first line reads +#: ``"TShark (Wireshark) 4.6.9 (Git commit ...)."`` -- Ubuntu noble's packaged +#: build reads ``"TShark (Wireshark) 4.2.2 (Git commit ...)."`` in exactly the +#: same shape, just a different number -- so an ``X.Y.Z`` anywhere in the +#: output is enough. +_VERSION_PATTERN = re.compile(r'(\d+\.\d+\.\d+)') + + +def detect_tshark_version(tshark: 'str | None') -> 'str | None': + """The ``X.Y.Z`` :program:`tshark` reports itself as, or :obj:`None`. + + Args: + tshark: Path or bare name of the :program:`tshark` binary, or + :obj:`None` if it was not found at all. + + Returns: + The version string, or :obj:`None` if *tshark* is :obj:`None` or its + ``--version`` output does not contain a recognisable ``X.Y.Z``. + + """ + if tshark is None: + return None + completed = subprocess.run([tshark, '--version'], capture_output=True, + text=True, check=False) + match = _VERSION_PATTERN.search(completed.stdout or completed.stderr) + return match.group(1) if match else None + + +#: Resolved once, the same way :data:`_EDITCAP`/:data:`_TSHARK` are. Not itself +#: a ``HAS_*`` flag: it is a version *string*, not a boolean gate, so it drives +#: an equality check inside a :func:`unittest.skipUnless` rather than being the +#: condition directly -- :data:`HAS_WIRESHARK` is what the dependency-gate scan +#: needs to see, and it already is one. +_TSHARK_VERSION = detect_tshark_version(_TSHARK) + + +class ParseParenthesisedNumberTests(unittest.TestCase): + """:func:`~pyshark_encap_map.parse_parenthesised_number`'s own robustness. + + **What this does not pin, stated plainly because it was previously + overclaimed here:** #851/#850's real defect -- 10 of 157 rows lost, + including ``null`` -- was in how the original ad hoc script extracted the + *name* portion of a PDML ``showname`` such as ``"Encapsulation type: + NULL/Loopback (15)"``. This generator never does that extraction at all; + a key comes from ``frame.encap_type``'s number directly, and a filter + name comes from the PDML ```` attribute, never parsed out + of prose. Reverting :data:`~pyshark_encap_map._PAREN_NUMBER` to #850's + original naive shape, ``r'(\\w+) \\((\\d+)\\)'`` used with + :func:`re.search`, leaves every test below still passing: ``.search`` is + unanchored, so ``\\w+`` matches ``Loopback`` and steps straight over the + ``/`` -- confirmed by trying it. + + **What this does pin:** the number-extraction is anchored to the *end* of + the string rather than to a run of name-shaped characters immediately + before it, which is a genuinely different -- and stricter -- property. + :meth:`test_a_name_with_no_word_boundary_before_the_number_still_matches` + is the case that tells the two approaches apart: a naive + ``\\w+ \\(\\d+\\)`` search requires *some* word character to sit directly + against the space before the parenthesis, and a name ending in anything + else -- a trailing ``/`` with nothing after it, say -- leaves the naive + pattern with nothing in the whole string to match at all, so + ``re.search`` returns :obj:`None` and the naive code (whatever it did + with a failed match) drops the row. This function does not have that + hole, because it never looks at the name. + + """ + + def test_a_slash_in_the_name_does_not_stop_the_match(self) -> None: + # The real showname tshark 4.6.9 renders for ``editcap -T null``. + # Included for realism, not as proof of the fix -- see the class + # docstring: the naive regex also passes this particular case. + self.assertEqual( + encap_map.parse_parenthesised_number('Encapsulation type: NULL/Loopback (15)'), + 15, + ) + + def test_a_name_with_no_word_boundary_before_the_number_still_matches(self) -> None: + # The case that actually distinguishes this function from #850's + # ``r'(\w+) \((\d+)\)'``: nothing but ``/`` sits between the name and + # the number, so a search anchored on a word run immediately before + # the space finds nothing anywhere in the string and returns None, + # while the end-anchored pattern here does not care what precedes it. + naive = re.compile(r'(\w+) \((\d+)\)') + showname = 'Encapsulation type: NULL/ (15)' + self.assertIsNone(naive.search(showname), 'test premise: the naive regex must fail here') + + self.assertEqual(encap_map.parse_parenthesised_number(showname), 15) + + def test_a_plain_name_still_matches(self) -> None: + self.assertEqual( + encap_map.parse_parenthesised_number('Encapsulation type: Ethernet (1)'), + 1, + ) + + def test_several_slashes_still_match(self) -> None: + self.assertEqual( + encap_map.parse_parenthesised_number('Encapsulation type: A/B/C (99)'), + 99, + ) + + def test_no_trailing_number_is_an_error(self) -> None: + with self.assertRaises(ValueError): + encap_map.parse_parenthesised_number('Encapsulation type: Ethernet') + + +class ListEncapTypesTests(unittest.TestCase): + """The second trap: ``editcap -T ''``'s own banner contributes a bogus token.""" + + #: Shaped like the real ``editcap -T ''`` output: an error line with no + #: leading whitespace (the trap), a blank line, a banner line that itself + #: reads ``editcap: ...`` with no leading whitespace either, then the + #: indented `` - `` listing the real parser wants. + LISTING = ( + 'editcap: "" isn\'t a valid encapsulation type\n' + '\n' + 'editcap: The available encapsulation types for the "-T" flag are:\n' + ' alp - ATSC Link-Layer Protocol (A/330) packets\n' + ' ap1394 - Apple IP-over-IEEE 1394\n' + ' fddi-swapped - FDDI with bit-swapped MAC addresses\n' + ) + + def test_banner_lines_are_not_mistaken_for_tokens(self) -> None: + # A split on the first run of whitespace reads the banner's own + # ``editcap:`` as a token; only the indented shape excludes it. + names = encap_map._ENCAP_LINE + matches = [names.match(line) for line in self.LISTING.splitlines()] + found = [match.group(1) for match in matches if match] + + self.assertNotIn('editcap:', found) + self.assertEqual(found, ['alp', 'ap1394', 'fddi-swapped']) + + def test_list_encap_types_reads_a_fabricated_binary(self) -> None: + # A fake "editcap" -- a tiny script that prints the fixture listing to + # stderr and exits non-zero, exactly as the real ``-T ''`` invocation + # does -- so the subprocess plumbing is exercised without needing the + # real binary at all. + fake = make_tmp_dir(self) / 'fake-editcap' + fake.write_text( + '#!/bin/sh\n' + f'printf %s {shell_quote(self.LISTING)} >&2\n' + 'exit 1\n', + encoding='utf-8', + ) + fake.chmod(0o755) + + names = encap_map.list_encap_types(str(fake)) + + self.assertEqual(names, ['alp', 'ap1394', 'fddi-swapped']) + + def test_a_listing_with_no_tokens_is_an_error(self) -> None: + fake = make_tmp_dir(self) / 'fake-editcap' + fake.write_text('#!/bin/sh\nprintf %s "nothing indented here" >&2\nexit 1\n', + encoding='utf-8') + fake.chmod(0o755) + + with self.assertRaises(encap_map.EncapSweepError): + encap_map.list_encap_types(str(fake)) + + +def shell_quote(text: str) -> str: + """A single-quoted, shell-safe literal for *text* -- no embedded quotes here.""" + return "'" + text.replace("'", "'\\''") + "'" + + +def make_tmp_dir(case: unittest.TestCase) -> pathlib.Path: + """A scratch directory that cleans itself up when *case* finishes. + + :meth:`unittest.TestCase.enterContext` would do this in one line, but it + is Python 3.11+ and this repository supports back to + ``requires-python = ">=3.6, <4"`` (:file:`pyproject.toml`) -- CI's own + oldest tested leg is ``Python 3.10``, where ``enterContext`` does not + exist at all (``AttributeError``). ``TemporaryDirectory.cleanup`` plus + :meth:`~unittest.TestCase.addCleanup` is the same "clean up after the + test regardless of outcome" guarantee, built from APIs both present since + 3.6. + + """ + tmp = tempfile.TemporaryDirectory() + case.addCleanup(tmp.cleanup) + return pathlib.Path(tmp.name) + + +class RenderDictTests(unittest.TestCase): + """The formatting has to be byte-stable, or a re-run is not idempotent.""" + + def test_comment_column_is_two_past_the_longest_entry(self) -> None: + entries = { + 1: (Enum_LinkType.ETHERNET, ['ether']), + 2: (Enum_LinkType.IEEE802_5, ['tr']), + } + block = encap_map.render_dict('ENCAP_TYPE_TO_LINKTYPE', entries, 'int') + # Entry lines only -- the closing "} # type: ..." line carries a "#" + # of its own, at a column that has nothing to do with entry alignment. + entry_lines = [line for line in block.split('\n') if line.startswith(' ')] + + codes = [line[:line.index('#')].rstrip() for line in entry_lines] + columns = {line.index('#') for line in entry_lines} + + self.assertEqual(len(columns), 1, f'comments are not aligned: {entry_lines}') + self.assertEqual(next(iter(columns)), max(len(code) for code in codes) + 2) + + def test_multiple_sources_are_comma_joined_in_order(self) -> None: + entries = {6: (Enum_LinkType.FDDI, ['fddi', 'fddi-nettl', 'fddi-swapped'])} + block = encap_map.render_dict('ENCAP_TYPE_TO_LINKTYPE', entries, 'int') + + self.assertIn('# fddi,fddi-nettl,fddi-swapped', block) + + def test_string_keys_are_quoted_like_the_committed_table(self) -> None: + entries = {'eth': (Enum_LinkType.ETHERNET, ['ether'])} + block = encap_map.render_dict('FILTER_NAME_TO_LINKTYPE', entries, 'str') + + self.assertIn(" 'eth': Enum_LinkType.ETHERNET,", block) + self.assertTrue(block.rstrip('\n').endswith('} # type: dict[str, Enum_LinkType]')) + + def test_empty_table_still_closes_correctly(self) -> None: + block = encap_map.render_dict('EMPTY', {}, 'int') + + self.assertEqual(block, 'EMPTY = {\n} # type: dict[int, Enum_LinkType]\n') + + +class ReplaceBlockTests(unittest.TestCase): + """Only the two dict bodies move; everything else in the file is untouched.""" + + TEXT = ( + 'BEFORE = 1\n' + '\n' + 'ENCAP_TYPE_TO_LINKTYPE = {\n' + ' 1: Enum_LinkType.ETHERNET, # ether\n' + '} # type: dict[int, Enum_LinkType]\n' + '\n' + 'FILTER_NAME_TO_LINKTYPE = {\n' + " 'eth': Enum_LinkType.ETHERNET, # ether\n" + '} # type: dict[str, Enum_LinkType]\n' + '\n' + 'AFTER = 2\n' + ) + + def test_surrounding_text_survives_the_swap(self) -> None: + replacement = 'ENCAP_TYPE_TO_LINKTYPE = {\n 2: Enum_LinkType.SLIP, # slip\n} # type: dict[int, Enum_LinkType]\n' + + updated = encap_map._replace_block(self.TEXT, 'ENCAP_TYPE_TO_LINKTYPE', replacement) + + self.assertIn('BEFORE = 1', updated) + self.assertIn('AFTER = 2', updated) + self.assertIn('Enum_LinkType.SLIP', updated) + # ETHERNET is gone from the ENCAP_TYPE_TO_LINKTYPE block specifically -- + # it legitimately survives in FILTER_NAME_TO_LINKTYPE, untouched by this + # replacement, which is the point of the assertion just above. + encap_block = updated[updated.index('ENCAP_TYPE_TO_LINKTYPE'): + updated.index('FILTER_NAME_TO_LINKTYPE')] + self.assertNotIn('Enum_LinkType.ETHERNET', encap_block) + + def test_a_missing_block_is_an_error(self) -> None: + with self.assertRaises(encap_map.EncapSweepError): + encap_map._replace_block('NOTHING_HERE = 1\n', 'ENCAP_TYPE_TO_LINKTYPE', 'x') + + def test_rewrite_is_idempotent(self) -> None: + entries = {1: (Enum_LinkType.ETHERNET, ['ether'])} + once = encap_map.rewrite(self.TEXT, entries, {}) + twice = encap_map.rewrite(once, entries, {}) + + self.assertEqual(once, twice) + + +class BuildTablesExclusionTests(unittest.TestCase): + """The two exclusion rules, against the real, mutable ``LinkType`` registry. + + ``LinkType``'s own ``_missing_`` mints a placeholder ``Unassigned_N`` member + for *any* value in range rather than raising (#575) -- so a naive + ``try: linktype(dlt) except ValueError`` never fires, and would silently + invent a table entry for a DLT the registry never actually named. These + exercise :func:`~pyshark_encap_map.build_tables` against a DLT (121) picked + to be absent from the registry *right now*, which is what makes the + assertion meaningful rather than assumed. + + """ + + def setUp(self) -> None: + self.assertNotIn( + 121, [member.value for member in Enum_LinkType], + 'DLT 121 has gained a LinkType member; this test needs a value the ' + 'registry does not define yet to prove build_tables does not mint one', + ) + + def test_an_unmapped_dlt_is_left_out_and_noted(self) -> None: + # frame.encap_type 32, editcap -T hhdlc, DLT 121 -- the one entry #850 + # deliberately left out of ENCAP_TYPE_TO_LINKTYPE. + measurements = [encap_map.Measurement(name='hhdlc', dlt=121, encap_type=32, + root_name='fake-field-wrapper')] + + encap_entries, filter_entries, notes = encap_map.build_tables( + measurements, Enum_LinkType) + + self.assertEqual(encap_entries, {}) + self.assertEqual(filter_entries, {}) + self.assertTrue(any('32' in note and '121' in note for note in notes), notes) + + # And the lookup must not have minted Unassigned_121 as a side effect. + self.assertNotIn(121, [member.value for member in Enum_LinkType]) + + def test_an_ambiguous_filter_name_is_left_out_and_noted(self) -> None: + measurements = [ + encap_map.Measurement(name='null', dlt=0, encap_type=15, root_name='null'), + encap_map.Measurement(name='loop', dlt=108, encap_type=15, root_name='null'), + ] + + encap_entries, filter_entries, notes = encap_map.build_tables( + measurements, Enum_LinkType) + + self.assertNotIn('null', filter_entries) + self.assertTrue(any('null' in note and 'ambiguous' in note for note in notes), notes) + + def test_pseudo_protocol_roots_are_never_candidates(self) -> None: + measurements = [ + encap_map.Measurement(name='hhdlc', dlt=1, encap_type=99, + root_name='fake-field-wrapper'), + ] + + _, filter_entries, _ = encap_map.build_tables(measurements, Enum_LinkType) + + self.assertEqual(filter_entries, {}) + + def test_an_unambiguous_name_is_kept_with_its_sources_in_order(self) -> None: + # DLT 10 is FDDI's; frame.encap_type 6 is what tshark 4.6.9 reports for + # it -- the two numbers are unrelated registries and the table's own + # comment (#850) is explicit that a WTAP_ENCAP_* number is not a DLT. + measurements = [ + encap_map.Measurement(name='fddi', dlt=10, encap_type=6, root_name='fddi'), + encap_map.Measurement(name='fddi-nettl', dlt=10, encap_type=6, root_name='fddi'), + ] + + encap_entries, filter_entries, _ = encap_map.build_tables(measurements, Enum_LinkType) + + self.assertEqual(encap_entries[6], (Enum_LinkType.FDDI, ['fddi', 'fddi-nettl'])) + self.assertEqual(filter_entries['fddi'], (Enum_LinkType.FDDI, ['fddi', 'fddi-nettl'])) + + +class DetectTsharkVersionTests(unittest.TestCase): + """The version parsing that decides whether :class:`RealSweepTests` runs. + + Unit-tier and binary-free: each case fabricates the *text* a real + ``tshark --version`` would print, rather than requiring a second Wireshark + install to prove the parser handles more than one release. The 4.2.2 case + is exactly what CI's runners install (Ubuntu noble's + ``libwireshark-data 4.2.2-1.1build3``) -- this is "the simulated older + version" that PR #853's own CI run could not be, locally. + + """ + + def test_the_local_4_6_9_style_banner_parses(self) -> None: + fake = make_tmp_dir(self) / 'fake-tshark' + fake.write_text( + '#!/bin/sh\n' + 'printf %s ' + + shell_quote('TShark (Wireshark) 4.6.9 (Git commit 2d548b197c75).\n') + + '\n', + encoding='utf-8', + ) + fake.chmod(0o755) + + self.assertEqual(detect_tshark_version(str(fake)), '4.6.9') + + def test_ubuntu_noble_4_2_2_style_banner_parses_and_mismatches(self) -> None: + # The exact banner shape apt's packaged tshark prints, per the CI log + # PR #853's cross-review quoted: libwireshark-data 4.2.2-1.1build3. + fake = make_tmp_dir(self) / 'fake-tshark' + fake.write_text( + '#!/bin/sh\n' + 'printf %s ' + + shell_quote('TShark (Wireshark) 4.2.2 (Git v4.2.2 packaged as 4.2.2-1.1build3).\n') + + '\n', + encoding='utf-8', + ) + fake.chmod(0o755) + + version = detect_tshark_version(str(fake)) + + self.assertEqual(version, '4.2.2') + self.assertNotEqual(version, MEASURED_WIRESHARK_VERSION) + + def test_no_binary_is_none(self) -> None: + self.assertIsNone(detect_tshark_version(None)) + + def test_unparseable_output_is_none(self) -> None: + fake = make_tmp_dir(self) / 'fake-tshark' + fake.write_text('#!/bin/sh\nprintf %s "no version number here"\n', encoding='utf-8') + fake.chmod(0o755) + + self.assertIsNone(detect_tshark_version(str(fake))) + + +@unittest.skipUnless(HAS_WIRESHARK, 'editcap/tshark not found on PATH') +@unittest.skipUnless( + _TSHARK_VERSION == MEASURED_WIRESHARK_VERSION, + f'tshark reports {_TSHARK_VERSION!r}, this class only trusts its exact ' + f'sweep arithmetic against {MEASURED_WIRESHARK_VERSION!r} -- the version ' + f'ENCAP_TYPE_TO_LINKTYPE and FILTER_NAME_TO_LINKTYPE were measured against ' + f'-- see VersionIndependentInvariantTests for what still runs here', +) +class RealSweepTests(unittest.TestCase): + """The full sweep, for real, against the committed source capture. + + Skipped -- via :data:`HAS_WIRESHARK` and a version check stacked on top of + it -- on any host without Wireshark installed, or whose Wireshark is not + :data:`MEASURED_WIRESHARK_VERSION`. Where it does run, it is the + executable form of #851's acceptance criteria: the sweep arithmetic + and byte-identical reproduction of the committed tables. CI's runners + install Ubuntu noble's packaged 4.2.2 (224 accepted, not 226), so this + class is expected to skip there and run only locally, on a pinned + Homebrew/self-built 4.6.9 -- see the module docstring and + :file:`util/pyshark_encap_map.py`'s own docstring for why that is the + right trade rather than a gap. + + """ + + def test_sweep_arithmetic_matches_the_committed_comment(self) -> None: + accepted = encap_map.list_encap_types(_EDITCAP) + self.assertEqual(len(accepted), encap_map.EXPECTED_ACCEPTED) + + def test_check_mode_reports_the_committed_file_as_current(self) -> None: + # examples/captures/in.pcap is git-tracked (not gitignored), so this is + # a unit-tier-legal read per tests/_tiers.py -- it is one of the + # captures committed rather than one only make_samples.py produces. + source = ROOT / 'examples' / 'captures' / 'in.pcap' + if not source.is_file(): + self.skipTest(f'{source} is absent') + + completed = subprocess.run( + [sys.executable, str(ROOT / 'util' / 'pyshark_encap_map.py'), '--check'], + capture_output=True, text=True, check=False, cwd=str(ROOT), + ) + + self.assertEqual( + completed.returncode, 0, + f'pcapkit/toolkit/pyshark.py has drifted from the sweep:\n' + f'{completed.stdout}\n{completed.stderr}', + ) + self.assertIn( + f'sweep: {encap_map.EXPECTED_ACCEPTED} accepted, ' + f'{encap_map.EXPECTED_WRITABLE} writable, {encap_map.EXPECTED_REFUSED} refused', + completed.stdout, + ) + + +@unittest.skipUnless(HAS_WIRESHARK, 'editcap/tshark not found on PATH') +class VersionIndependentInvariantTests(unittest.TestCase): + """Properties of a real sweep that hold on any Wireshark, not only 4.6.9. + + Where :class:`RealSweepTests` skips -- any Wireshark that is not + :data:`MEASURED_WIRESHARK_VERSION`, which is every CI leg today -- this + still runs the real sweep against whatever :program:`editcap`/ + :program:`tshark` *is* on ``PATH`` (224 accepted on CI's Ubuntu noble + 4.2.2, not the 226 :class:`RealSweepTests` requires) and checks structure + instead of counts: no ``frame.encap_type`` key or DLT is ambiguous within + this sweep, and every key or filter name this sweep *did* measure agrees + with what is committed. It does not require the committed table to be + complete for this Wireshark -- only that it is not wrong about what this + Wireshark actually reports, which is a property completeness on 4.6.9 + does not by itself establish. + + """ + + @classmethod + def setUpClass(cls) -> None: + source = ROOT / 'examples' / 'captures' / 'in.pcap' + if not source.is_file(): + raise unittest.SkipTest(f'{source} is absent') + + # The same value-must-already-be-registered guard build_tables() uses + # (#575): Enum_LinkType(dlt) mints a placeholder member for any value + # in range rather than raising, so "is this DLT really in the table" + # has to be answered by membership, never by calling the constructor + # first and asking forgiveness. + cls.known_dlts = frozenset(member.value for member in Enum_LinkType) + + accepted = encap_map.list_encap_types(_EDITCAP) + measurements = [] # type: list[encap_map.Measurement] + with tempfile.TemporaryDirectory(prefix='pyshark_encap_map_invariants_') as tmp: + tmp_dir = pathlib.Path(tmp) + for name in accepted: + result = encap_map.measure_encap(_EDITCAP, _TSHARK, source, name, tmp_dir) + if isinstance(result, encap_map.Measurement): + measurements.append(result) + cls.measurements = measurements + + def test_no_encap_type_key_resolves_to_two_dlts(self) -> None: + by_key = {} # type: dict[int, set[int]] + for item in self.measurements: + by_key.setdefault(item.encap_type, set()).add(item.dlt) + + ambiguous = {key: dlts for key, dlts in by_key.items() if len(dlts) > 1} + self.assertEqual(ambiguous, {}, f'this Wireshark disagrees with itself: {ambiguous}') + + def test_no_dlt_resolves_to_two_encap_type_keys(self) -> None: + by_dlt = {} # type: dict[int, set[int]] + for item in self.measurements: + by_dlt.setdefault(item.dlt, set()).add(item.encap_type) + + ambiguous = {dlt: keys for dlt, keys in by_dlt.items() if len(keys) > 1} + self.assertEqual(ambiguous, {}, f'this Wireshark disagrees with itself: {ambiguous}') + + def test_measured_encap_type_keys_agree_with_the_committed_table(self) -> None: + from pcapkit.toolkit.pyshark import ENCAP_TYPE_TO_LINKTYPE + + mismatches = [] + for item in self.measurements: + if item.encap_type not in ENCAP_TYPE_TO_LINKTYPE or item.dlt not in self.known_dlts: + continue # committed table may simply not cover this key -- not this test's job + measured = Enum_LinkType(item.dlt) + committed = ENCAP_TYPE_TO_LINKTYPE[item.encap_type] + if measured != committed: + mismatches.append((item.name, item.encap_type, item.dlt, committed)) + + self.assertEqual(mismatches, [], + f'committed ENCAP_TYPE_TO_LINKTYPE disagrees with a live ' + f'measurement: {mismatches}') + + def test_measured_filter_names_agree_with_the_committed_table(self) -> None: + from pcapkit.toolkit.pyshark import FILTER_NAME_TO_LINKTYPE + + by_root = {} # type: dict[str, set[int]] + for item in self.measurements: + if item.root_name is not None: + by_root.setdefault(item.root_name, set()).add(item.dlt) + + mismatches = [] + for name, dlts in by_root.items(): + # Ambiguous within this sweep, or the committed table simply does + # not cover this name -- either way, not what this test checks. + if name not in FILTER_NAME_TO_LINKTYPE or len(dlts) != 1: + continue + dlt, = dlts + if dlt not in self.known_dlts: + continue + measured = Enum_LinkType(dlt) + committed = FILTER_NAME_TO_LINKTYPE[name] + if measured != committed: + mismatches.append((name, dlt, committed)) + + self.assertEqual(mismatches, [], + f'committed FILTER_NAME_TO_LINKTYPE disagrees with a live ' + f'measurement: {mismatches}') + + +if __name__ == '__main__': + unittest.main() diff --git a/util/pyshark_encap_map.py b/util/pyshark_encap_map.py new file mode 100644 index 0000000000..3208f09412 --- /dev/null +++ b/util/pyshark_encap_map.py @@ -0,0 +1,694 @@ +# -*- coding: utf-8 -*- +"""Regenerate the two hand-maintained tables in :mod:`pcapkit.toolkit.pyshark`. + +GitHub issue #851. :file:`pcapkit/toolkit/pyshark.py` carries +``ENCAP_TYPE_TO_LINKTYPE`` (Wireshark's internal ``WTAP_ENCAP_*`` number -> +:class:`~pcapkit.const.reg.linktype.LinkType`) and ``FILTER_NAME_TO_LINKTYPE`` +(a PDML root protocol's *filter* name -> the same enum), added by #850 as two +literal dict blocks. Neither table was ever transcribed from a Wireshark +source file -- there is not one on this host to transcribe -- each entry is +one round trip through Wireshark's own ``wiretap/pcap-common.c``, and this +script is that round trip, made runnable instead of ad hoc. + +The method, unchanged from #850 +-------------------------------- + +For every encapsulation :program:`editcap` accepts under ``-T``: + +1. ``editcap -F pcap -T `` rewrites the source capture + as that encapsulation. Some of the 226 encapsulations refuse the rewrite + entirely -- their dissector cannot re-encode this capture's Ethernet + frames -- and are simply skipped; that is the *69 refused* the acceptance + arithmetic below asserts. +2. The **value** is read from the 4-byte little-endian ``network`` field at + offset 20 of ````'s own pcap file header -- not looked up in a table, + since the whole point is not to trust a second table transcribed by hand. +3. The **key**, for ``ENCAP_TYPE_TO_LINKTYPE``, is ``frame.encap_type`` as + :program:`tshark`'s PDML output reports it for ````'s first packet. + The **key** for ``FILTER_NAME_TO_LINKTYPE`` is the ``name`` of the first + ```` PDML emits after ``frame`` -- the outermost dissector's filter + name, which is what :mod:`pyshark` exposes as ``packet.layers[0] + .layer_name`` and what :func:`pcapkit.toolkit.pyshark.tcp_traceflow` falls + back to when a capture carries no ``frame.encap_type`` field at all. + +Two traps, both costing real time when #850's tables were first produced +-------------------------------------------------------------------------- + +**A PDML ``showname`` can hold a ``/``.** ``tshark`` renders +``frame.encap_type``'s ``showname`` as ``"Encapsulation type: NULL/Loopback +(15)"`` -- the display name itself, not just this one, can contain the +character. A regex that expects the name to be made of "word" characters up +to the parenthesised number -- ``r'(\\w+) \\((\\d+)\\)'``, say -- stops at the +``/`` and drops the row outright. That silently lost 10 of 157 rows, +including ``null`` itself, when #850's tables were first produced. +:func:`parse_parenthesised_number` is anchored to the *end* of the string +instead, so the name in front of the number can contain anything at all. + +**The ``editcap -T`` help listing carries bogus tokens.** Piping +``editcap -T ''`` (an empty argument, which is how the flag's own help text +says to list the encapsulations) through any plain whitespace-based split -- +:func:`str.split` over the whole output, with no maxsplit -- gives **1237** +tokens (measured against Wireshark 4.6.9, the version every other count here +was taken against), because ``editcap: The available encapsulation types for +the "-T" flag are:`` and every one-line description contribute their own +words on top of the 226 real tokens. The listing also opens with **two** +banner lines, not one: ``editcap: "" isn't a valid encapsulation type`` (the +error for the empty argument itself) and then ``editcap: The available +encapsulation types for the "-T" flag are:`` immediately after it -- so a +line-oriented split still has two non-entries to exclude, not the single line +an earlier draft of this docstring assumed. The true count is **226**, and +the shape that yields it is `` - ``, anchored to leading +whitespace so neither banner line -- both flush against the left margin, with +no leading whitespace at all -- can match: + +.. code-block:: shell + + sed -n 's/^[[:space:]]\\{1,\\}\\([A-Za-z0-9._-]\\{1,\\}\\) - .*/\\1/p' + +:data:`_ENCAP_LINE` is that same shape, read with :mod:`re` instead of +:program:`sed` so the count is asserted in-process rather than trusted from a +shell pipeline. + +What is deliberately left out +------------------------------ + +A DLT with no :class:`~pcapkit.const.reg.linktype.LinkType` member -- +``editcap -T hhdlc`` writes DLT 121, which the enum does not define -- is +skipped rather than invented, in both tables, the same way an unmapped +``frame.encap_type`` already raises :exc:`~pcapkit.utilities.exceptions +.MissingKeyError` rather than resolving to a near-miss DLT (#843). +``FILTER_NAME_TO_LINKTYPE`` additionally excludes a root name that resolved +to more than one distinct DLT across the sweep (ambiguous: the PDML node does +not say which arrived) and the two pseudo-protocol roots ``fake-field-wrapper`` +(:mod:`pyshark`'s ``data``, meaning no dissector recognised the frame) and +``_ws.malformed`` (the one capture :program:`tshark` could not parse). Both +tables' own module-level comments in :file:`pcapkit/toolkit/pyshark.py` -- +untouched by this script, which only rewrites the two dict *bodies* -- record +the full accounting of what was excluded and why. + +Byte-stable output +------------------- + +Re-running this script against an unchanged tree must change nothing, which +is the only thing that makes it worth having. Each dict's entries are +emitted key-ascending, one entry per line, with the trailing ``# `` +comment aligned to one column past the longest entry in *that* dict -- exactly +the formatting already committed, verified below by :func:`main`'s own diff +against the file it just wrote back. + +Usage +----- + +.. code-block:: shell + + python util/pyshark_encap_map.py # regenerate pcapkit/toolkit/pyshark.py + python util/pyshark_encap_map.py --check # exit non-zero if it has drifted + +Requires the :program:`tshark` and :program:`editcap` binaries (Wireshark +4.6.9 is what every measurement here was taken against); their absence is +reported on :data:`sys.stderr` and exits cleanly rather than raising, so a +host without Wireshark installed does not turn "nothing to regenerate" into a +traceback. + +This cannot be run in CI to verify the committed tables, only locally against +a pinned Wireshark +------------------------------------------------------------------------------ + +:data:`EXPECTED_ACCEPTED` / :data:`EXPECTED_WRITABLE` / :data:`EXPECTED_REFUSED` +are properties of Wireshark 4.6.9, not of this script's method, and this +script refuses to rewrite anything when the installed Wireshark disagrees with +them rather than silently regenerating a *different* table (#851 was explicit +that a discrepancy here is a finding, not license to overwrite what was +measured and committed) -- see :class:`EncapSweepError`. Ubuntu noble's +packaged Wireshark, what every ``apt``-based CI runner installs, is 4.2.2: its +``editcap -T`` accepts 224 encapsulations, not 226, so this script's own +assertion trips there every time, by design (found the hard way: PR #853's +first CI run, three legs, ``AssertionError: 224 != 226``). + +That is harmless to the *committed* tables at runtime -- an +``ENCAP_TYPE_TO_LINKTYPE``/``FILTER_NAME_TO_LINKTYPE`` entry a 4.2.2 install +would never produce is simply never looked up by one -- and is arguably the +right way round: the tables should be at least as rich as the newest +Wireshark a user might have, not capped at whatever a distribution happens to +package. The real consequence is narrower and worth being plain about: **this +generator can only be re-run, and its output only re-verified, on a host with +Wireshark pinned to :data:`EXPECTED_ACCEPTED`'s version (4.6.9) or one that +still sweeps to the same 226/157/69**. CI cannot do either with its packaged +Wireshark, so CI does not run this script at all; what it does still exercise +against whatever Wireshark it has -- structural invariants that hold at any +version, plus the parsing and formatting helpers against fabricated +input -- lives in ``tests/project/test_pyshark_encap_map.py``, not here. + +""" +from __future__ import annotations + +import argparse +import difflib +import pathlib +import re +import shutil +import struct +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from typing import TYPE_CHECKING, NamedTuple + +if TYPE_CHECKING: + from typing import Any, Optional, Sequence + +__all__ = [ + 'EncapSweepError', 'parse_parenthesised_number', 'list_encap_types', + 'measure_encap', 'build_tables', 'render_dict', 'rewrite', 'main', +] + +#: Repository root, taken from this file's location rather than the working +#: directory -- the same spelling :file:`util/changelog_md.py` and +#: :file:`util/bump_version.py` use, and for the same reason: the script gives +#: the same answer run from anywhere. +ROOT = pathlib.Path(__file__).resolve().parent.parent + +#: The module both generated dict blocks live in. +TARGET = ROOT / 'pcapkit' / 'toolkit' / 'pyshark.py' + +#: The capture every encapsulation is measured from. Generated by +#: :file:`examples/generators/make_samples.py`; this script does not run that +#: generator itself -- see :func:`main`. +SOURCE_CAPTURE = ROOT / 'examples' / 'captures' / 'in.pcap' + +#: Expected sweep arithmetic against :data:`SOURCE_CAPTURE`, asserted rather +#: than assumed (#851's acceptance criterion). A mismatch means either +#: Wireshark's own encapsulation list moved since 4.6.9, or the source +#: capture is not the one every comment in :data:`TARGET` was measured +#: against -- either way, a generated table nobody checked is worse than a +#: script that refuses to run. +EXPECTED_ACCEPTED = 226 +EXPECTED_WRITABLE = 157 +EXPECTED_REFUSED = 69 + +#: The `` - `` shape of ``editcap -T ''``'s listing, +#: anchored to leading whitespace so the banner line -- which has none -- +#: cannot match. See the module docstring's second trap. +_ENCAP_LINE = re.compile(r'^[ \t]+([A-Za-z0-9._-]+) - .*$') + +#: The trailing parenthesised integer of a PDML ``showname``, e.g. +#: ``"Encapsulation type: NULL/Loopback (15)"`` -> ``15``. Anchored to the +#: *end* of the string rather than to a run of name-shaped characters, so a +#: ``/`` earlier in the name -- as in that example -- cannot stop the match. +#: See the module docstring's first trap. +_PAREN_NUMBER = re.compile(r'\((\d+)\)\s*$') + +#: PDML root-protocol names that name a parser outcome rather than a +#: link-layer or payload dissector, and are therefore never eligible for +#: ``FILTER_NAME_TO_LINKTYPE`` regardless of how many DLTs they cover. +#: ``fake-field-wrapper`` is what :program:`tshark` emits when no dissector +#: recognised the frame at all -- :mod:`pyshark` reports it as ``data`` -- +#: and ``_ws.malformed`` is what it emits when it could not parse the frame +#: it did recognise. +_PSEUDO_PROTOCOLS = frozenset({'fake-field-wrapper', '_ws.malformed', 'data'}) + +#: Offset of the ``network`` field in a classic pcap file header (RFC-less, +#: but stable since libpcap's very first release): magic (4) + version (4) + +#: thiszone (4) + sigfigs (4) + snaplen (4) = 20 bytes in. +_DLT_OFFSET = 20 + + +class EncapSweepError(RuntimeError): + """The sweep did not match what every comment in :data:`TARGET` asserts. + + Raised rather than silently regenerating a different table -- per #851, + a discrepancy here is a finding about the generator (or about Wireshark + having moved), not license to overwrite what #850 measured and committed. + + """ + + +class Missing(NamedTuple): + """Why the sweep skipped one encapsulation.""" + + #: The ``editcap -T`` token. + name: str + #: Human-readable reason, for the summary :func:`main` prints. + reason: str + + +class Measurement(NamedTuple): + """One encapsulation's round trip through :program:`editcap`/:program:`tshark`.""" + + #: The ``editcap -T`` token that produced this measurement. + name: str + #: The DLT read from the rewritten file's own pcap header. + dlt: int + #: ``frame.encap_type``, read from the rewritten file's first packet. + encap_type: int + #: The first PDML ```` after ``frame``, or :obj:`None` if there + #: was none at all. + root_name: 'Optional[str]' + + +def parse_parenthesised_number(showname: str) -> int: + """Extract the trailing ``(N)`` integer from a PDML ``showname``. + + Args: + showname: A field's ``showname`` attribute, e.g. + ``"Encapsulation type: NULL/Loopback (15)"``. + + Returns: + The integer, ``15`` for that example. + + Raises: + ValueError: If *showname* does not end in a parenthesised integer. + + """ + match = _PAREN_NUMBER.search(showname) + if match is None: + raise ValueError(f'no parenthesised number at the end of {showname!r}') + return int(match.group(1)) + + +def list_encap_types(editcap: str) -> 'list[str]': + """The ``editcap -T`` tokens this binary accepts, in the order it lists them. + + Args: + editcap: Path (or bare name, resolved against ``PATH``) of the + :program:`editcap` binary. + + Returns: + Every accepted encapsulation token. 226 of them, against Wireshark + 4.6.9 -- see :data:`EXPECTED_ACCEPTED`. + + Raises: + EncapSweepError: If the listing does not parse to any tokens at all, + which means ``-T ''`` no longer lists encapsulations the way it + does in 4.6.9. + + """ + # An empty ``-T`` argument is deliberately invalid input -- it is how the + # flag's own ``--help`` text says to get the listing -- so a non-zero + # exit is the expected outcome and is not treated as a failure here. + completed = subprocess.run( + [editcap, '-T', ''], capture_output=True, text=True, check=False, + ) + text = completed.stdout + completed.stderr + + names = [match.group(1) for line in text.splitlines() + for match in (_ENCAP_LINE.match(line),) if match] + if not names: + raise EncapSweepError( + f"{editcap} -T '' produced no recognisable encapsulation listing; " + f'either the binary is not editcap, or its -T help output has ' + f'changed shape since Wireshark 4.6.9:\n{text}' + ) + return names + + +def measure_encap(editcap: str, tshark: str, source: pathlib.Path, + name: str, tmp_dir: pathlib.Path) -> 'Measurement | Missing': + """Round-trip one encapsulation through :program:`editcap` and :program:`tshark`. + + Args: + editcap: Path or bare name of the :program:`editcap` binary. + tshark: Path or bare name of the :program:`tshark` binary. + source: Capture to rewrite -- :data:`SOURCE_CAPTURE` in practice. + name: The ``editcap -T`` token to measure. + tmp_dir: Scratch directory for the rewritten capture. + + Returns: + A :class:`Measurement` if *name* could be written as pcap from + *source* and read back; a :class:`Missing` naming why not, if + :program:`editcap` refused the rewrite. + + """ + out_path = tmp_dir / f'{name}.pcap' + written = subprocess.run( + [editcap, '-F', 'pcap', '-T', name, str(source), str(out_path)], + capture_output=True, text=True, check=False, + ) + if written.returncode != 0 or not out_path.is_file(): + return Missing(name, written.stderr.strip() or 'editcap refused the rewrite') + + with open(out_path, 'rb') as file: + header = file.read(_DLT_OFFSET + 4) + dlt, = struct.unpack_from(' ( + 'tuple[dict[int, tuple[Any, list[str]]], dict[str, tuple[Any, list[str]]], list[str]]'): + """Group the sweep's measurements into the two tables' entries. + + Args: + measurements: Every successfully written-and-read encapsulation, in + the order :func:`list_encap_types` produced them -- ascending + within a shared key, which is what lets the multi-source comment + (``# fddi,fddi-nettl,fddi-swapped``) come out in the same order + every time. + linktype: The :class:`~pcapkit.const.reg.linktype.LinkType` enum. + + Returns: + ``(encap_entries, filter_entries, notes)`` -- the two tables' key -> + (member, [source names]) mappings, plus human-readable notes about + what was measured but excluded (an unmapped DLT, an ambiguous filter + name, ...), for :func:`main` to print. + + """ + notes = [] # type: list[str] + + # *linktype* is one of this project's registry enums: an unknown value + # does not raise out of the constructor, it mints a placeholder member + # via ``_missing_``/``extend_enum`` (#575) -- so ``LinkType(121)`` + # *succeeds*, returning a freshly minted ``Unassigned_121`` rather than + # telling us 121 was never a real member. The set of values genuinely + # registered has to be captured *before* any such lookup runs, or the + # very first miss would mint a member that every later check then finds + # already there. ``dlt in known_values`` is the check; ``linktype(dlt)`` + # is only ever called once *that* has already said yes, so it can never + # reach ``_missing_`` itself. + known_values = frozenset(member.value for member in linktype) + + by_encap_type = {} # type: dict[int, list[Measurement]] + by_root_name = {} # type: dict[str, list[Measurement]] + for measurement in measurements: + by_encap_type.setdefault(measurement.encap_type, []).append(measurement) + if measurement.root_name is not None: + by_root_name.setdefault(measurement.root_name, []).append(measurement) + + encap_entries = {} # type: dict[int, tuple[Any, list[str]]] + for key, group in sorted(by_encap_type.items()): + dlts = {item.dlt for item in group} + if len(dlts) != 1: + notes.append( + f'frame.encap_type {key} maps to more than one DLT within the ' + f'sweep ({sorted(dlts)}); left out of ENCAP_TYPE_TO_LINKTYPE ' + f'rather than guessing one' + ) + continue + dlt = dlts.pop() + if dlt not in known_values: + names = ','.join(item.name for item in group) + notes.append( + f'frame.encap_type {key} (editcap -T {names}, DLT {dlt}) has no ' + f'LinkType member; left out of ENCAP_TYPE_TO_LINKTYPE' + ) + continue + encap_entries[key] = (linktype(dlt), [item.name for item in group]) + + filter_entries = {} # type: dict[str, tuple[Any, list[str]]] + for name, group in sorted(by_root_name.items()): + if name in _PSEUDO_PROTOCOLS: + continue + dlts = {item.dlt for item in group} + members = {linktype(dlt) for dlt in dlts if dlt in known_values} + if len(members) != 1: + if len(dlts) > 1: + notes.append( + f'filter name {name!r} is ambiguous within the sweep ' + f'({len(dlts)} distinct DLTs); left out of ' + f'FILTER_NAME_TO_LINKTYPE' + ) + continue + filter_entries[name] = (members.pop(), [item.name for item in group]) + + return encap_entries, filter_entries, notes + + +def _key_repr(key: 'int | str') -> str: + """Render a dict key the way the committed tables spell it.""" + return repr(key) if isinstance(key, str) else str(key) + + +def render_dict(name: str, entries: 'dict[Any, tuple[Any, list[str]]]', + value_type: str) -> str: + """Render one ``NAME = { ... }`` block, byte-for-byte as it is committed. + + Args: + name: The dict's variable name, e.g. ``ENCAP_TYPE_TO_LINKTYPE``. + entries: Key -> (enum member, [source names]), key-ascending. + value_type: The key type's spelling in the trailing + ``# type: dict[...]`` comment (``int`` or ``str``). + + Returns: + The complete block, ending in a single newline. + + """ + rows = [] # type: list[tuple[str, str]] + for key, (member, sources) in entries.items(): + code = f' {_key_repr(key)}: Enum_LinkType.{member.name},' + rows.append((code, ','.join(sources))) + + width = max((len(code) for code, _ in rows), default=0) + 2 + + lines = [f'{name} = {{'] + for code, comment in rows: + lines.append(f'{code.ljust(width)}# {comment}') + lines.append(f'}} # type: dict[{value_type}, Enum_LinkType]') + return '\n'.join(lines) + '\n' + + +def _replace_block(text: str, name: str, replacement: str) -> str: + """Swap ``NAME = { ... }`` for *replacement* in *text*. + + The block is found by its opening ``NAME = {`` line and its first + closing ``}`` line after that -- sufficient here because no value in + either table is itself a mapping, so no line before the close can start + with ``}``. + + Args: + text: The file's current contents. + name: The dict's variable name. + replacement: The complete rendered block, from :func:`render_dict`. + + Returns: + *text* with that one block replaced. + + Raises: + EncapSweepError: If *name*'s block cannot be found, which means + :data:`TARGET` no longer has the shape this script rewrites. + + """ + lines = text.split('\n') + start = next((index for index, line in enumerate(lines) + if line == f'{name} = {{'), None) + if start is None: + raise EncapSweepError(f'{TARGET} has no {name!r} block to replace') + end = next((index for index in range(start + 1, len(lines)) + if lines[index].startswith('}')), None) + if end is None: + raise EncapSweepError(f'{TARGET} has no closing brace for {name!r}') + + return '\n'.join(lines[:start] + replacement.rstrip('\n').split('\n') + lines[end + 1:]) + + +def rewrite(current: str, encap_entries: 'dict[int, tuple[Any, list[str]]]', + filter_entries: 'dict[str, tuple[Any, list[str]]]') -> str: + """Return *current* with both dict blocks replaced by freshly rendered ones. + + Args: + current: :data:`TARGET`'s current contents. + encap_entries: As returned by :func:`build_tables`. + filter_entries: As returned by :func:`build_tables`. + + Returns: + The rewritten contents. + + """ + updated = _replace_block( + current, 'ENCAP_TYPE_TO_LINKTYPE', + render_dict('ENCAP_TYPE_TO_LINKTYPE', encap_entries, 'int'), + ) + return _replace_block( + updated, 'FILTER_NAME_TO_LINKTYPE', + render_dict('FILTER_NAME_TO_LINKTYPE', filter_entries, 'str'), + ) + + +def main(argv: 'Optional[Sequence[str]]' = None) -> int: + """Command line entry point. + + Args: + argv: Argument list, defaulting to :data:`sys.argv`. + + Returns: + ``0`` on success, or when :program:`tshark`/:program:`editcap` are + absent -- see the module docstring for why that is not a failure. + ``1`` if ``--check`` found :data:`TARGET` stale, or if the sweep's + arithmetic did not match :data:`EXPECTED_ACCEPTED` / + :data:`EXPECTED_WRITABLE` / :data:`EXPECTED_REFUSED`. + + """ + parser = argparse.ArgumentParser( + prog='pyshark_encap_map.py', + description=( + 'Regenerate ENCAP_TYPE_TO_LINKTYPE and FILTER_NAME_TO_LINKTYPE in ' + 'pcapkit/toolkit/pyshark.py by sweeping every encapsulation editcap ' + 'accepts.' + ), + ) + parser.add_argument('--check', action='store_true', + help='write nothing; exit non-zero if the committed ' + 'file has drifted') + parser.add_argument('--editcap', default='editcap', + help='editcap binary (default: %(default)s, resolved ' + 'against PATH)') + parser.add_argument('--tshark', default='tshark', + help='tshark binary (default: %(default)s, resolved ' + 'against PATH)') + parser.add_argument('--source', type=pathlib.Path, default=SOURCE_CAPTURE, + help='capture to sweep from (default: %(default)s)') + args = parser.parse_args(argv) + + editcap = shutil.which(args.editcap) + tshark = shutil.which(args.tshark) + if editcap is None or tshark is None: + missing = [tool for tool, path in (('editcap', editcap), ('tshark', tshark)) + if path is None] + print( + f'{", ".join(missing)} not found on PATH -- skipping. ' + f'ENCAP_TYPE_TO_LINKTYPE and FILTER_NAME_TO_LINKTYPE were measured ' + f'with Wireshark 4.6.9 and nothing here can re-measure them without ' + f'it; the committed tables in {TARGET} are left exactly as they are.', + file=sys.stderr, + ) + return 0 + + if not args.source.is_file(): + print( + f'{args.source} does not exist. It is generated -- run ' + f'"python examples/generators/make_samples.py" first, then re-run ' + f'this script.', + file=sys.stderr, + ) + return 0 + + # Imported here rather than at module level, the same way + # util/bump_version.py's current_version() imports pcapkit lazily: this + # script has to run -- and print its --check verdict -- even where + # pcapkit itself is not importable, which is exactly the case argparse + # and the earlier tshark/editcap and source-capture checks exist to + # report cleanly rather than as a bare ImportError traceback. + # + # ROOT is put at the front of sys.path, and any pip-installed editable + # finder is evicted from sys.meta_path first, so that "the LinkType this + # process sees" is unconditionally *this* checkout's + # pcapkit/const/reg/linktype.py rather than whatever an unrelated + # editable install (`pip install -e .` run against a different checkout + # entirely -- a sibling clone, another git worktree) happens to point + # at. A meta_path finder is consulted before sys.path is, so inserting + # ROOT alone is not enough to override one; measured the hard way while + # regenerating this table across a git worktree whose venv's editable + # install pointed at the main checkout, where it silently kept reading + # the main checkout's (older) linktype.py and produced a table that + # matched the *wrong* base. + sys.path.insert(0, str(ROOT)) + for _finder in list(sys.meta_path): + if 'editable' in type(_finder).__module__.lower(): + sys.meta_path.remove(_finder) + from pcapkit.const.reg.linktype import LinkType as Enum_LinkType + + _pcapkit_file = getattr(sys.modules.get('pcapkit'), '__file__', None) or '' + if not _pcapkit_file.startswith(str(ROOT)): + raise EncapSweepError( + f'pcapkit resolved to {_pcapkit_file!r}, not under {ROOT} -- an ' + f'editable install pointed at a different checkout is still ' + f'winning, and this script refuses to measure LinkType membership ' + f'against the wrong tree' + ) + + accepted = list_encap_types(editcap) + + measurements = [] # type: list[Measurement] + refused = [] # type: list[Missing] + with tempfile.TemporaryDirectory(prefix='pyshark_encap_map_') as tmp: + tmp_dir = pathlib.Path(tmp) + for name in accepted: + result = measure_encap(editcap, tshark, args.source, name, tmp_dir) + if isinstance(result, Missing): + refused.append(result) + else: + measurements.append(result) + + if len(accepted) != EXPECTED_ACCEPTED: + raise EncapSweepError( + f'editcap -T accepted {len(accepted)} encapsulations, expected ' + f'{EXPECTED_ACCEPTED}; Wireshark has moved since 4.6.9 and every ' + f'table comment measured against that version needs re-checking, ' + f'not just this script' + ) + if len(measurements) != EXPECTED_WRITABLE: + raise EncapSweepError( + f'{len(measurements)} encapsulations were writable from {args.source}, ' + f'expected {EXPECTED_WRITABLE}' + ) + if len(refused) != EXPECTED_REFUSED: + raise EncapSweepError( + f'{len(refused)} encapsulations were refused, expected {EXPECTED_REFUSED}' + ) + print(f'sweep: {len(accepted)} accepted, {len(measurements)} writable, ' + f'{len(refused)} refused (matches the committed arithmetic)') + + encap_entries, filter_entries, notes = build_tables(measurements, Enum_LinkType) + for note in notes: + print(f'note: {note}', file=sys.stderr) + + current = TARGET.read_text(encoding='utf-8') + want = rewrite(current, encap_entries, filter_entries) + + if not args.check: + if want != current: + # newline='\n' rather than the default None: every string this + # script builds already ends its lines in '\n', so on Windows the + # default would translate each into '\r\n' on the way out and + # rewrite all ~210 lines rather than the handful that actually + # changed. .gitattributes' `* text=auto` keeps CRLF from reaching + # a commit either way, so this is hardening against a noisy local + # diff, not a fix for output this repository would ever ship. + TARGET.write_text(want, encoding='utf-8', newline='\n') + print(f'wrote {TARGET}') + else: + print(f'{TARGET} already matches the sweep; nothing to write') + return 0 + + if want == current: + print(f'{TARGET} matches the sweep') + return 0 + + print(f'{TARGET} has drifted from the sweep', file=sys.stderr) + sys.stderr.writelines(difflib.unified_diff( + current.splitlines(keepends=True), + want.splitlines(keepends=True), + fromfile=f'{TARGET.name} (committed)', + tofile=f'{TARGET.name} (regenerated)', + )) + return 1 + + +if __name__ == '__main__': + sys.exit(main())