diff --git a/Pipfile b/Pipfile index 42eed5e3ac..928e4dd631 100644 --- a/Pipfile +++ b/Pipfile @@ -21,6 +21,25 @@ cryptography = "*" # ``pipenv lock``, which has to build its metadata -- on any machine without the # libpcap development files. Install it by hand to work on that engine: # ``pipenv run pip install pypcap``. +# +# ``pcap-ct`` and ``libpcap``, which drive the PCAP_CT engine, are listed -- +# the reason ``pypcap`` cannot be does not apply to them. Both are +# ``py3-none-any`` wheels, so there is nothing to compile and ``pipenv lock`` +# works on a machine with no libpcap *development* files at all. (A system +# ``libpcap.so.1`` is still needed to actually run the engine; that is a runtime +# prerequisite, not something the lock file can express.) +# +# The versions are spelled out rather than left as ``"*"`` because neither +# project has ever published a non-pre-release, and naming a pre-release in the +# specifier is the PEP 440 way to opt into one. That keeps the opt-in scoped to +# these two: ``allow_prereleases`` below stays commented out, since switching it +# on would let every other package in this file resolve to a beta as well. +# +# The marker matches the PCAP_CT extra in pyproject.toml. ``libpcap`` 1.11.0b29 +# declares ``Requires-Python: <4.0.0,>=3.10.0``, which is the binding floor +# (``pcap-ct`` itself allows 3.9). +pcap-ct = {version = ">=1.3.0b3",markers = "python_version >= '3.10'"} +libpcap = {version = ">=1.11.0b29",markers = "python_version >= '3.10'"} pypcapfile = "*" beautifulsoup4 = {extras = ["html5lib"],version = "*"} requests = {extras = ["socks"],version = "*"} diff --git a/README.rst b/README.rst index 7cf1f3fd7a..072bbef566 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ The PyPCAPKit project is an open source Python program focus on network packet parsing and analysis, which works as a comprehensive `PCAP`_ file extraction, construction and analysis library. - The whole project supports **Python 3.6** or later. + The whole project supports **Python 3.6** or later; CI covers 3.10 to 3.14, and 3.15 as an allowed-to-fail leg. ----- About @@ -79,16 +79,24 @@ fact that ``pcapkit`` is a **comprehensive** packet processing module. Additionally, ``pcapkit`` introduced alternative extraction engines to accelerate this procedure. By now ``pcapkit`` supports `Scapy`_, `DPKT`_, `PyShark`_, -`PyPCAP`_ and `PyPCAPFile`_, selected through ``engine='scapy'``, -``'dpkt'``, ``'pyshark'``, ``'pypcap'`` and ``'pypcapfile'`` respectively; -``engine='default'`` (also spelled ``'pcapkit'``) is ``pcapkit``'s own parser -and the only one with no third-party requirement. +`PyPCAP`_, `pcap-ct`_ and `PyPCAPFile`_, selected through ``engine='scapy'``, +``'dpkt'``, ``'pyshark'``, ``'pypcap'``, ``'pcap_ct'`` and ``'pypcapfile'`` +respectively; ``engine='default'`` (also spelled ``'pcapkit'``) is ``pcapkit``'s +own parser and the only one with no third-party requirement. + +`PyPCAP`_ and `pcap-ct`_ are two independent distributions of the same +``libpcap(3)`` interface, and both install a top-level ``pcap`` module, so +they are two engines rather than one. Upstream `PyPCAP`_ stops at Python 3.11; +`pcap-ct`_ covers 3.10 and newer. **Install exactly one of them** -- with both +present, ``pcap-ct`` wins the import and the other becomes unselectable, which +each engine detects and reports. Speed is not free. Every third-party engine supports **less** than the -``default`` one, and the two newest support markedly less: +``default`` one, and the newest ones support markedly less: - `PyPCAP`_ performs no protocol dissection at all, so it offers neither reassembly nor flow tracing, and reads PCAP savefiles from disk only. +- `pcap-ct`_ reads the same interface, so it has exactly the same gaps. - `PyPCAPFile`_ has no IPv6 decoder, so IPv6 reassembly is unavailable; IPv4 and TCP reassembly still work, and it too is PCAP-only. - `PyShark`_ performs no reassembly. @@ -96,6 +104,42 @@ Speed is not free. Every third-party engine supports **less** than the Each gap is announced with a warning or an exception rather than silently returning nothing. The `engine support documentation`_ tabulates them. +Every engine also answers a preflight check before it is used -- +``unsupported_reason()`` -- so asking for one that cannot run in the current +environment produces a single warning naming the actual cause (a Python version, a +missing ``tshark``, a missing ``libpcap``, the wrong ``pcap`` distribution) and a +clean fall back to ``pcapkit``'s own parser, rather than an error from inside the +third-party package. + +Engine support by Python version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Which engines can run at all, by interpreter. Verified by installing each engine +and extracting a capture on 3.10, 3.11, 3.12 and 3.14; 3.13 and 3.15 were not +available and are marked accordingly. + +============== ======== ======== ======== ======== ======== ======== +Engine 3.10 3.11 3.12 3.13 3.14 3.15 +============== ======== ======== ======== ======== ======== ======== +``pcapkit`` yes yes yes yes* yes yes* +``dpkt`` yes yes yes yes* yes yes* +``scapy`` yes yes yes yes* yes yes* +``pcap_ct`` yes yes yes yes* yes yes* +``pypcap`` yes yes no no no no +``pypcapfile`` yes yes no no no no +``pyshark`` yes† yes† yes† yes*† no no +============== ======== ======== ======== ======== ======== ======== + +``*`` inferred, not measured -- no 3.13 or 3.15 interpreter was available. +``†`` also needs Wireshark's ``tshark``, which was absent, so only the +interpreter half was verified for ``pyshark``. + +``pypcap`` and ``pypcapfile`` stop at 3.11, and ``pyshark`` at 3.13, for the +reasons under `Engine prerequisites`_. **Python 3.11 is the last version on which +every engine can run** -- and even there ``pypcap`` and ``pcap_ct`` are mutually +exclusive, since both provide the ``pcap`` module, so no single environment ever +has all seven at once. + Test Environment ~~~~~~~~~~~~~~~~ @@ -121,19 +165,51 @@ Engine Performance (ms per packet) ``dpkt`` 0.010390_056723 ``scapy`` 0.091690_233567 ``pcapkit`` 0.200390_390390 -``pyshark`` 24.682185_018351 +``pyshark`` 24.682185_018351 [3]_ ``pypcap`` *not measured* [1]_ +``pcap_ct`` *not measured* [4]_ ``pypcapfile`` *not measured* [2]_ ============== =========================== -Both figures will be filled in once the two engines can be timed on the same -host, capture and iteration count as the existing rows. +**These figures are historical, and three of the rows can no longer be +reproduced on a current Python.** The table was taken on the environment above, +whose interpreter still ran every engine. Since then ``pyshark``, ``pypcap`` and +``pypcapfile`` have each acquired a hard Python ceiling -- 3.13, 3.11 and 3.11 +respectively, for the reasons under `Engine prerequisites`_ -- so on the latest +Python only ``pcapkit``, ``dpkt``, ``scapy`` and ``pcap_ct`` can be timed at all. +A re-run on a modern interpreter would therefore not extend this table; it would +replace it with a shorter one, measured on different hardware and not comparable +row-for-row with what is here. + +The empty cells stay empty for the same reason: a figure taken on a different +host, capture or iteration count is not comparable with these, and inventing one +would be worse than admitting the gap. ------------ Installation ------------ - **Note** -- ``pcapkit`` supports Python versions **since 3.6**. + **Note** -- ``pcapkit`` declares support for **Python 3.6 and later**, and CI + covers **3.10 through 3.14**, plus 3.15 as an allowed-to-fail leg. + + The sources themselves use 3.8 syntax; the ``bpc-walrus``/``bpc-poseur`` + backport tools in ``setup.py`` convert it at install time, which is what makes + the lower bound possible. Measured: 3.9 and 3.8 import and extract straight + from source with no conversion needed, and 3.7 needs the conversion. + + **That conversion is currently blocked by an upstream bug**, so below 3.8 the + declaration is intent rather than something that works today: ``bpc-poseur`` + 0.4.3.post1 crashes on positional-only parameters declared on a *method* + rather than a plain function, and exits 0 so the build does not notice. The + 12 such parameters in ``pcapkit/corekit/io.py`` then survive into the + installed package and ``import pcapkit`` fails. It is a one-line fix + upstream -- ``poseur.py:744`` passes ``cls_ctx=name.name`` where ``name`` is + already a parso ``Name`` and wants ``.value`` -- and with it applied, + ``walrus`` then ``poseur`` produce a file Python 3.7 parses cleanly. Tracking + that fix is what will make 3.6/3.7 real again. + + 3.8 and 3.9 are end-of-life and best-effort. Individual *engines* also stop + earlier than the library does; see `Engine prerequisites`_. Simply run the following to install the current version from PyPI: @@ -188,28 +264,60 @@ plug-in functions, you may want to install the optional ones: pip install pypcapkit[PyPCAPFile] # for PyPCAP only -- see the note below, this one builds from source pip install pypcapkit[PyPCAP] + # for pcap-ct only -- the pure-Python alternative to PyPCAP, and the one that + # works on Python 3.12+; do not install it alongside PyPCAP + pip install pypcapkit[PCAP_CT] # for ESP payload decryption pip install pypcapkit[crypto] - # and to install the optional packages -- note this excludes PyPCAP + # and to install the optional packages -- note this excludes PyPCAP and pcap-ct pip install pypcapkit[all] # or to do this explicitly pip install pypcapkit dpkt scapy pyshark pypcapfile - **Important** -- The ``all`` extra deliberately does **not** include - ``pypcap``. Everything + **Important** -- The ``all`` extra deliberately excludes both ``pypcap`` and + ``pcap-ct``, for different reasons. Everything else in ``all`` is a pure-Python wheel, whereas ``pypcap`` compiles a C extension; pulling it into ``all`` would demand a working compiler and the `libpcap`_ development files from everyone installing ``pypcapkit[all]``. - Install it explicitly with ``pip install pypcapkit[PyPCAP]``. + ``pcap-ct`` needs no compiler, but it and its ``libpcap`` dependency are + published only as **pre-releases** (1.3.0b3 and 1.11.0b29), and ``all`` should + not be how somebody ends up with a beta they did not ask for. Install either + explicitly: ``pip install pypcapkit[PyPCAP]`` or + ``pip install pypcapkit[PCAP_CT]``. + + **Install only one of them.** Both distributions own the top-level ``pcap`` + module, and ``pip`` will install both without complaint. With both present the + ``pcap-ct`` package wins the import and ``pypcap``'s extension module is + shadowed and unreachable, so ``engine='pypcap'`` stops working. ``pcapkit`` + detects that state and warns, naming both distributions and which one won, but + it cannot undo it. Engine prerequisites -------------------- -Three of the engines need something beyond a ``pip install``: +Four of the engines need something beyond a ``pip install``. Each constraint is +also enforced in code -- the engine's ``unsupported_reason()`` is consulted before +anything is imported -- so hitting one produces a warning naming the cause and a +fall back to ``pcapkit``'s own parser, not an error from inside the third-party +package. ``pyshark`` - Drives Wireshark's ``tshark`` binary, which must be on ``PATH``. Install - Wireshark (or just ``tshark``) from your platform's package manager. + Two requirements, and neither is visible to an import: the package imports + cleanly and then fails when used. + + - Drives Wireshark's ``tshark`` binary. It need not be on ``PATH``: + ``pyshark`` looks at ``tshark_path`` in its ``config.ini`` first, then + ``PATH`` on POSIX, both Program Files directories on Windows, and + ``/Applications/Wireshark.app`` on macOS. Install Wireshark (or just + ``tshark``) from your platform's package manager. + - Requires Python **3.13 or older**. ``pyshark`` 0.6 builds its event loop + with ``asyncio.get_event_loop_policy().get_event_loop()``, and from Python + **3.14** ``asyncio.get_event_loop()`` raises ``RuntimeError`` when there + is no current event loop instead of quietly creating one. Measured: a loop + is returned silently on 3.10 and 3.11, returned with a + ``DeprecationWarning`` on 3.12, and refused on 3.14. (3.13 was not available + to test and is expected to work, being on the deprecated-but-functional side + of that change.) ``pypcap`` Ships **no wheels** -- only an sdist -- so ``pip`` compiles it, and the build @@ -239,6 +347,27 @@ Three of the engines need something beyond a ``pip install``: found even though it is installed. Installing into ``sys.prefix``, or into ``/opt/libpcap``, is what that search will pick up. +``pcap_ct`` + The way to drive the same `libpcap`_ interface on Python **3.12 and newer**, + where ``pypcap`` cannot be built. Nothing to compile and no ``pcap.h`` needed: + `pcap-ct`_ is a ``ctypes`` reimplementation, and both it and its ``libpcap`` + dependency ship ``py3-none-any`` wheels. Verified reading a capture on Python + 3.10 and 3.14. + + Two caveats: + + - **A system ``libpcap`` is still required at run time.** The ``libpcap`` + distribution ships a vendored ``libpcap.so`` and, as published, does not use + it: its ``libpcap.cfg`` says ``LIBPCAP = None``, which sends its loader to + ``ctypes.util.find_library('pcap')``. So the library actually loaded is the + host's ``libpcap.so.1``, and with none present ``import pcap`` raises + ``OSError`` rather than ``ImportError``. Set ``LIBPCAP = tcpdump`` in + ``libpcap.cfg`` to use the vendored copy instead. + - Both distributions are **pre-releases**, and ``pcap-ct`` documents itself as + tracking the ``pypcap`` *1.2.3* interface. Every attribute the engine uses + was measured behaving identically to ``pypcap`` 1.3.0, but that is a + statement about the versions tested. + ``pypcapfile`` Version 0.12.0 imports the ``imp`` module, which was **removed in Python 3.12**, so ``pcapfile.savefile`` -- the module needed to read a capture -- @@ -248,10 +377,11 @@ Three of the engines need something beyond a ``pip install``: **Note** -- ``pcapkit`` itself, and its ``default``, ``dpkt`` and ``scapy`` engines, work - fine on current Python versions. Only the three engines above carry these - extra constraints, and asking for an engine whose package is unavailable - emits a warning and falls back to ``pcapkit``'s own parser rather than - failing outright. + fine on current Python versions -- ``dpkt`` 1.9.8 and ``scapy`` 2.7.0 were both + measured reading a capture on Python 3.14. Only the four engines above carry + extra constraints, and asking for an engine that cannot run in the current + environment emits a warning naming the reason and falls back to ``pcapkit``'s + own parser rather than failing outright. For CLI usage, you will need to install the optional packages: @@ -294,11 +424,20 @@ engine, and is not needed by the test suite. .. _DPKT: https://dpkt.readthedocs.io .. _PyShark: https://kiminewt.github.io/pyshark .. _PyPCAP: https://github.com/pynetwork/pypcap +.. _pcap-ct: https://pypi.org/project/pcap-ct/ .. _PyPCAPFile: https://github.com/kisom/pypcapfile .. _libpcap: https://www.tcpdump.org .. _DictDumper: https://github.com/JarryShaw/DictDumper .. _engine support documentation: https://jarryshaw.github.io/PyPCAPKit/pcapkit/foundation/engines/index.html +.. [3] This figure is **historical**. `PyShark`_ 0.6 builds its event loop with + ``asyncio.get_event_loop_policy().get_event_loop()``, and Python 3.14 made + ``asyncio.get_event_loop()`` raise ``RuntimeError`` when no current event + loop exists rather than quietly creating one -- measured working on 3.10 and + 3.11, working with a ``DeprecationWarning`` on 3.12, and raising on 3.14. The + number therefore cannot be reproduced on a current interpreter; it stands as + what was measured when it could be. + .. [1] `PyPCAP`_ could not be installed on the machine available for benchmarking, so no figure was taken. Its 1.3.0 sdist compiles a C extension and needs `libpcap`_'s headers *and* shared library present, and the @@ -307,6 +446,12 @@ engine, and is not needed by the test suite. hardware and a different Python from the rows above -- which would not be comparable with them -- the cell is left empty. +.. [4] `pcap-ct`_ was verified working on Python 3.10 and 3.14, so unlike the two + rows above it *could* be timed -- but only on the machine this engine was added + on, which is neither the hardware nor the operating system the rows above were + measured with. A number from it would not be comparable, so the cell is left + empty rather than filled with something misleading. + .. [2] `PyPCAPFile`_ 0.12.0 cannot be imported on Python 3.12 or newer, so it could only be timed on an older interpreter than the rows above were measured with. That number would not be comparable, so the cell is left empty. diff --git a/docs/source/ext.rst b/docs/source/ext.rst index 4956018672..fb8826f2a2 100644 --- a/docs/source/ext.rst +++ b/docs/source/ext.rst @@ -325,23 +325,35 @@ file formats: | +-----------------------------------------------------------+----------------------------------------+ | | :class:`pcapkit.foundation.engines.pypcap.PyPCAP` | PCAP only, and from a file on disk | | +-----------------------------------------------------------+----------------------------------------+ +| | :class:`pcapkit.foundation.engines.pcap_ct.PCAP_CT` | PCAP only, and from a file on disk | +| +-----------------------------------------------------------+----------------------------------------+ | | :class:`pcapkit.foundation.engines.pypcapfile.PyPCAPFile` | PCAP only | +---------------------+-----------------------------------------------------------+----------------------------------------+ .. note:: An engine is free to support less than :class:`~pcapkit.foundation.extraction.Extractor` - offers, and several do. `PyPCAP`_ performs no protocol dissection whatsoever, so - it supports neither reassembly nor flow tracing; `PyPCAPFile`_ has no IPv6 - decoder, so it supports IPv4 and TCP reassembly but not IPv6. What matters is - that the gap is *announced* -- each engine warns, or raises, for the capability - it cannot provide, rather than silently producing an empty result. See + offers, and several do. `PyPCAP`_ and `pcap-ct`_ perform no protocol dissection + whatsoever, so they support neither reassembly nor flow tracing; `PyPCAPFile`_ + has no IPv6 decoder, so it supports IPv4 and TCP reassembly but not IPv6. What + matters is that the gap is *announced* -- each engine warns, or raises, for the + capability it cannot provide, rather than silently producing an empty result. See :doc:`pcapkit/foundation/engines/index` for the full table. +.. note:: + + `PyPCAP`_ and `pcap-ct`_ are two distributions of one :manpage:`libpcap(3)` + interface, and both install a top-level :mod:`pcap` module, so they are two + engines rather than one: ``engine='pypcap'`` and ``engine='pcap_ct'``. Upstream + `PyPCAP`_ cannot be installed on Python 3.12 or newer, and `pcap-ct`_ can -- + which is why both exist. See + :doc:`pcapkit/foundation/engines/index` for which to pick. + .. _Scapy: https://scapy.net .. _DPKT: https://dpkt.readthedocs.io .. _PyShark: https://kiminewt.github.io/pyshark .. _PyPCAP: https://github.com/pynetwork/pypcap +.. _pcap-ct: https://pypi.org/project/pcap-ct/ .. _PyPCAPFile: https://github.com/kisom/pypcapfile Samples diff --git a/docs/source/index.rst b/docs/source/index.rst index f485db4bc9..638a26ac4f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -13,7 +13,7 @@ construction and analysis library. .. important:: - The whole project supports **Python 3.6** or later. + The whole project supports **Python 3.6** or later; CI covers 3.10 to 3.14, and 3.15 as an allowed-to-fail leg. .. .. contents:: .. :depth: 2 @@ -112,20 +112,66 @@ fact that :mod:`pcapkit` is a **comprehensive** packet processing module. Additionally, :mod:`pcapkit` introduced alternative extraction engines to accelerate this procedure. By now :mod:`pcapkit` supports `Scapy`_, `DPKT`_, -`PyShark`_, `PyPCAP`_ and `PyPCAPFile`_, selected through ``engine='scapy'``, -``'dpkt'``, ``'pyshark'``, ``'pypcap'`` and ``'pypcapfile'`` respectively; -``engine='default'`` (also spelled ``'pcapkit'``) is :mod:`pcapkit`'s own parser -and the only one with no third-party requirement. +`PyShark`_, `PyPCAP`_, `pcap-ct`_ and `PyPCAPFile`_, selected through +``engine='scapy'``, ``'dpkt'``, ``'pyshark'``, ``'pypcap'``, ``'pcap_ct'`` and +``'pypcapfile'`` respectively; ``engine='default'`` (also spelled ``'pcapkit'``) +is :mod:`pcapkit`'s own parser and the only one with no third-party requirement. + +`PyPCAP`_ and `pcap-ct`_ are two independent distributions of the same +:manpage:`libpcap(3)` interface and both install a top-level :mod:`pcap` module, +so they are two engines rather than one: upstream `PyPCAP`_ stops at Python 3.11, +`pcap-ct`_ covers 3.10 and newer. **Install exactly one of them** -- with both +present, ``pcap-ct`` wins the import and the other becomes unselectable. Speed is not free. Every third-party engine supports **less** than the -``default`` one, and the two newest support markedly less -- -:class:`~pcapkit.foundation.engines.pypcap.PyPCAP` performs no protocol -dissection at all, so it offers neither reassembly nor flow tracing, and +``default`` one, and the newest ones support markedly less -- +:class:`~pcapkit.foundation.engines.pypcap.PyPCAP` and +:class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` perform no protocol +dissection at all, so they offer neither reassembly nor flow tracing, and :class:`~pcapkit.foundation.engines.pypcapfile.PyPCAPFile` has no IPv6 decoder, so IPv6 reassembly is unavailable. Each gap is announced with a warning or an exception rather than silently returning nothing; :doc:`pcapkit/foundation/engines/index` tabulates them. +Every engine also answers a preflight check -- +:meth:`~pcapkit.foundation.engines.engine.EngineBase.unsupported_reason`, which +:meth:`Extractor.run ` consults +before anything is imported -- so asking for one that cannot run in the current +environment gives a single warning naming the actual cause (a Python version, a +missing :program:`tshark`, a missing :manpage:`libpcap(3)`, the wrong ``pcap`` +distribution) and a clean fall back, rather than an error from inside the +third-party package. + +Engine support by Python version +-------------------------------- + +Which engines can run at all, by interpreter. Verified by installing each engine +and extracting a capture on 3.10, 3.11, 3.12 and 3.14; 3.13 and 3.15 were not +available and are marked accordingly. + +================== ======== ======== ======== =========== ======== =========== +Engine 3.10 3.11 3.12 3.13 [*]_ 3.14 3.15 [*]_ +================== ======== ======== ======== =========== ======== =========== +``pcapkit`` yes yes yes yes yes yes +``dpkt`` yes yes yes yes yes yes +``scapy`` yes yes yes yes yes yes +``pcap_ct`` yes yes yes yes yes yes +``pypcap`` yes yes no no no no +``pypcapfile`` yes yes no no no no +``pyshark`` [*]_ yes yes yes yes no no +================== ======== ======== ======== =========== ======== =========== + +.. [*] Inferred rather than measured: no 3.13 interpreter was available. +.. [*] Inferred rather than measured: no 3.15 interpreter was available. +.. [*] ``pyshark`` also needs Wireshark's :program:`tshark`, which was absent here, + so only the interpreter half of each verdict in this row was verified. + +``pypcap`` and ``pypcapfile`` stop at 3.11, and ``pyshark`` at 3.13, for the +reasons under `Engine prerequisites`_. **Python 3.11 is the last version on which +every engine can run** -- and even there ``pypcap`` and ``pcap_ct`` are mutually +exclusive, since both provide the :mod:`pcap` module, so no single environment ever +has all seven at once. + Test Environment ---------------- @@ -151,20 +197,58 @@ Engine Performance (ms per packet) ``dpkt`` 0.010390_056723 ``scapy`` 0.091690_233567 ``pcapkit`` 0.200390_390390 -``pyshark`` 24.682185_018351 +``pyshark`` 24.682185_018351 [3]_ ``pypcap`` *not measured* [1]_ +``pcap_ct`` *not measured* [4]_ ``pypcapfile`` *not measured* [2]_ ============== =========================== -Both figures will be filled in once the two engines can be timed on the same -host, capture and iteration count as the existing rows. +.. warning:: + + **These figures are historical, and three of the rows can no longer be + reproduced on a current Python.** The table was taken on the environment above, + whose interpreter still ran every engine. Since then ``pyshark``, ``pypcap`` + and ``pypcapfile`` have each acquired a hard Python ceiling -- 3.13, 3.11 and + 3.11 respectively, for the reasons under `Engine prerequisites`_ -- so on the + latest Python only ``pcapkit``, ``dpkt``, ``scapy`` and ``pcap_ct`` can be timed + at all. A re-run on a modern interpreter would not extend this table; it would + replace it with a shorter one, measured on different hardware and not + comparable row-for-row with what is here. + +The empty cells stay empty for the same reason: a figure taken on a different +host, capture or iteration count is not comparable with these, and inventing one +would be worse than admitting the gap. Installation ============ .. note:: - :mod:`pcapkit` supports Python versions **since 3.6**. + :mod:`pcapkit` declares support for **Python 3.6 and later**, and CI verifies + **3.10 through 3.14**, plus 3.15 as an allowed-to-fail leg. + + The sources themselves use 3.8 syntax; the ``bpc-walrus``/``bpc-poseur`` + backport tools in :file:`setup.py` convert it at install time, which is what + makes the lower bound possible. Measured: 3.9 and 3.8 import and extract + straight from source with no conversion needed, and 3.7 needs the conversion. + +.. warning:: + + **The conversion is currently blocked by an upstream bug**, so below 3.8 the + declaration is intent rather than something that works today. ``bpc-poseur`` + 0.4.3.post1 crashes on positional-only parameters declared on a *method* + rather than a plain function, and exits 0 so the build does not notice; the 12 + such parameters in :mod:`pcapkit.corekit.io` then survive into the installed + package and ``import pcapkit`` fails with :exc:`SyntaxError`. + + It is a one-line fix upstream -- ``poseur.py:744`` passes ``cls_ctx=name.name`` + where ``name`` is already a parso ``Name`` and wants ``.value`` -- and with it + applied, ``walrus`` then ``poseur`` produce a file Python 3.7 parses cleanly. + +.. note:: + + 3.8 and 3.9 are end-of-life and best-effort. Individual *engines* also stop + earlier than the library does; see `Engine prerequisites`_. Simply run the following to install the current version from PyPI: @@ -197,31 +281,67 @@ plug-in functions, you may want to install the optional ones: pip install pypcapkit[PyPCAPFile] # for PyPCAP only -- see the note below, this one builds from source pip install pypcapkit[PyPCAP] + # for pcap-ct only -- the pure-Python alternative to PyPCAP, and the one that + # works on Python 3.12+; do not install it alongside PyPCAP + pip install pypcapkit[PCAP_CT] # for ESP payload decryption pip install pypcapkit[crypto] - # and to install the optional packages -- note this excludes PyPCAP + # and to install the optional packages -- note this excludes PyPCAP and pcap-ct pip install pypcapkit[all] # or to do this explicitly pip install pypcapkit dpkt scapy pyshark pypcapfile .. important:: - The ``all`` extra deliberately does **not** include ``pypcap``. Everything + The ``all`` extra deliberately excludes both ``pypcap`` and ``pcap-ct``, for + different reasons. Everything else in ``all`` is a pure-Python wheel, whereas ``pypcap`` compiles a C extension; pulling it into ``all`` would demand a working compiler and the `libpcap`_ development files from everyone installing ``pypcapkit[all]``. - Install it explicitly with ``pip install pypcapkit[PyPCAP]``. + ``pcap-ct`` needs no compiler, but it and its ``libpcap`` dependency are + published only as **pre-releases** (1.3.0b3 and 1.11.0b29), and ``all`` should + not be how somebody ends up with a beta they did not ask for. Install either + explicitly: ``pip install pypcapkit[PyPCAP]`` or + ``pip install pypcapkit[PCAP_CT]``. + +.. warning:: + + **Install only one of ``pypcap`` and ``pcap-ct``.** Both own the top-level + :mod:`pcap` module, and ``pip`` will install both without complaint. With both + present the ``pcap-ct`` package wins the import and ``pypcap``'s extension + module is shadowed and unreachable, so ``engine='pypcap'`` stops working -- + measured on Python 3.10. :mod:`pcapkit` detects that state and warns with an + :class:`~pcapkit.utilities.warnings.EngineWarning` naming both distributions + and which one won, but it cannot undo it. -------------------- Engine prerequisites -------------------- -Three of the engines need something beyond a ``pip install``: +Four of the engines need something beyond a ``pip install``. Each constraint is +also enforced in code, through the engine's +:meth:`~pcapkit.foundation.engines.engine.EngineBase.unsupported_reason`, so +hitting one produces a warning naming the cause and a fall back to +:mod:`pcapkit`'s own parser rather than an error from inside the third-party +package. :class:`~pcapkit.foundation.engines.pyshark.PyShark` - Drives Wireshark's :program:`tshark` binary, which must be on ``PATH``. - Install Wireshark (or just :program:`tshark`) from your platform's package - manager. + Two requirements, and neither is visible to an import: the package imports + cleanly and then fails when used. + + - Drives Wireshark's :program:`tshark` binary. It need not be on ``PATH``: + ``pyshark`` looks at ``tshark_path`` in its :file:`config.ini` first, then + ``PATH`` on POSIX, both Program Files directories on Windows, and + :file:`/Applications/Wireshark.app` on macOS. Install Wireshark (or just + :program:`tshark`) from your platform's package manager. + - Requires Python **3.13 or older**. ``pyshark`` 0.6 builds its event loop + with ``asyncio.get_event_loop_policy().get_event_loop()``, and from Python + **3.14** :func:`asyncio.get_event_loop` raises :exc:`RuntimeError` when there + is no current event loop instead of quietly creating one. Measured: a loop is + returned silently on 3.10 and 3.11, returned with a + :exc:`DeprecationWarning` on 3.12, and refused on 3.14. (3.13 was not + available to test and is expected to work, being on the + deprecated-but-functional side of that change.) :class:`~pcapkit.foundation.engines.pypcap.PyPCAP` `PyPCAP`_ ships **no wheels** -- only an sdist -- so :program:`pip` compiles @@ -253,6 +373,27 @@ Three of the engines need something beyond a ``pip install``: it is installed. Installing into :data:`sys.prefix`, or into :file:`/opt/libpcap`, is what that search will pick up. +:class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` + The way to drive the same `libpcap`_ interface on Python **3.12 and newer**, + where ``pypcap`` cannot be built. Nothing to compile and no :file:`pcap.h` + needed: `pcap-ct`_ is a :mod:`ctypes` reimplementation, and both it and its + ``libpcap`` dependency ship ``py3-none-any`` wheels. Verified reading a capture + on Python 3.10 and 3.14. + + Two caveats: + + - **A system** :manpage:`libpcap(3)` **is still required at run time.** The + ``libpcap`` distribution ships a vendored :file:`libpcap.so` and, as + published, does not use it: its :file:`libpcap.cfg` says ``LIBPCAP = None``, + which sends its loader to :func:`ctypes.util.find_library`. So the library + actually loaded is the host's ``libpcap.so.1``, and with none present + ``import pcap`` raises :exc:`OSError` rather than :exc:`ImportError`. Set + ``LIBPCAP = tcpdump`` in :file:`libpcap.cfg` to use the vendored copy instead. + - Both distributions are **pre-releases**, and ``pcap-ct`` documents itself as + tracking the ``pypcap`` *1.2.3* interface. Every attribute the engine uses was + measured behaving identically to ``pypcap`` 1.3.0, but that is a statement + about the versions tested. + :class:`~pcapkit.foundation.engines.pypcapfile.PyPCAPFile` `PyPCAPFile`_ 0.12.0 imports the ``imp`` module, which was **removed in Python 3.12**, so ``pcapfile.savefile`` -- the module needed to read a @@ -263,10 +404,12 @@ Three of the engines need something beyond a ``pip install``: .. note:: :mod:`pcapkit` itself, and its ``default``, ``dpkt`` and ``scapy`` engines, - work fine on current Python versions. Only the three engines above carry these - extra constraints, and asking for an engine whose package is unavailable emits - an :class:`~pcapkit.utilities.warnings.EngineWarning` and falls back to - :mod:`pcapkit`'s own parser rather than failing outright. + work fine on current Python versions -- ``dpkt`` 1.9.8 and ``scapy`` 2.7.0 were + both measured reading a capture on Python 3.14. Only the four engines above + carry extra constraints, and asking for an engine that cannot run in the + current environment emits an + :class:`~pcapkit.utilities.warnings.EngineWarning` naming the reason and falls + back to :mod:`pcapkit`'s own parser rather than failing outright. For CLI usage, you will need to install the optional packages: @@ -289,10 +432,19 @@ Indices and tables .. _DPKT: https://dpkt.readthedocs.io .. _PyShark: https://kiminewt.github.io/pyshark .. _PyPCAP: https://github.com/pynetwork/pypcap +.. _pcap-ct: https://pypi.org/project/pcap-ct/ .. _PyPCAPFile: https://github.com/kisom/pypcapfile .. _libpcap: https://www.tcpdump.org .. _DictDumper: https://github.com/JarryShaw/DictDumper +.. [3] This figure is **historical**. `PyShark`_ 0.6 builds its event loop with + ``asyncio.get_event_loop_policy().get_event_loop()``, and Python 3.14 made + :func:`asyncio.get_event_loop` raise :exc:`RuntimeError` when no current event + loop exists rather than quietly creating one -- measured working on 3.10 and + 3.11, working with a :exc:`DeprecationWarning` on 3.12, and raising on 3.14. The + number cannot be reproduced on a current interpreter; it stands as what was + measured when it could be. + .. [1] `PyPCAP`_ could not be installed on the machine available for benchmarking, so no figure was taken. Its 1.3.0 sdist compiles a C extension and needs `libpcap`_'s headers *and* shared library present, and the @@ -301,6 +453,12 @@ Indices and tables hardware and a different Python from the rows above -- which would not be comparable with them -- the cell is left empty. +.. [4] `pcap-ct`_ was verified working on Python 3.10 and 3.14, so unlike the two + rows above it *could* be timed -- but only on the machine this engine was added + on, which is neither the hardware nor the operating system the rows above were + measured with. A number from it would not be comparable, so the cell is left + empty rather than filled with something misleading. + .. [2] `PyPCAPFile`_ 0.12.0 cannot be imported on Python 3.12 or newer, so it could only be timed on an older interpreter than the rows above were measured with. That number would not be comparable, so the cell is left empty. diff --git a/docs/source/pcapkit/foundation/engines/3rdparty.rst b/docs/source/pcapkit/foundation/engines/3rdparty.rst index 847f629800..2f6e9f12e1 100644 --- a/docs/source/pcapkit/foundation/engines/3rdparty.rst +++ b/docs/source/pcapkit/foundation/engines/3rdparty.rst @@ -58,12 +58,39 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. .. _PyShark: https://kiminewt.github.io/pyshark +.. important:: + + `PyShark`_ has **two** requirements beyond installing it, and neither is + visible to an import -- the package imports cleanly and then fails when used, + which is why + :meth:`~pcapkit.foundation.engines.pyshark.PyShark.unsupported_reason` checks + both up front. + + **Python 3.13 or older.** ``pyshark`` 0.6 builds its event loop with + ``asyncio.get_event_loop_policy().get_event_loop()``. Measured in a fresh + interpreter with no running loop: 3.10 and 3.11 return a loop silently, 3.12 + returns one with a :exc:`DeprecationWarning`, and **3.14 raises** + ``RuntimeError: There is no current event loop in thread 'MainThread'``. Python + 3.13 was not available to measure and is expected to work, being on the + deprecated-but-functional side of that change. + + **Wireshark's** :program:`tshark` **binary.** ``pyshark`` shells out to it and + parses nothing itself. It need not be on :envvar:`PATH`: ``pyshark`` consults + ``tshark_path`` in its :file:`config.ini` first, then :envvar:`PATH` on POSIX, + both Program Files directories on Windows, and + :file:`/Applications/Wireshark.app` on macOS. The check delegates to + ``pyshark``'s own resolver for exactly that reason, so a correctly configured + install off :envvar:`PATH` is not refused. + .. autoclass:: pcapkit.foundation.engines.pyshark.PyShark :no-members: :show-inheritance: .. autoattribute:: __engine_name__ .. autoattribute:: __engine_module__ + .. autoattribute:: PYTHON_CEILING + + .. automethod:: unsupported_reason .. automethod:: run .. automethod:: read_frame @@ -93,6 +120,37 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. pip install pypcapkit[PyPCAP] + **On Python 3.12 and newer it cannot be installed at all**, which is why the + extra carries a ``python_version < '3.12'`` marker. `PyPCAP`_ 1.3.0 ships a + ``pcap.c`` pre-generated by Cython 0.29.32 and never runs Cython at build + time, and that generated C does not compile against the 3.12+ C API. Measured + on four interpreters with libpcap present and found: + + =============== =============================================================== + Python ``pip install pypcap`` + =============== =============================================================== + 3.10 builds, imports + 3.11 builds, imports + 3.12 **fails** -- ``ob_digit``, ``curexc_traceback`` + 3.14 **fails** -- those, plus ``ma_version_tag`` and the + ``_PyLong_AsByteArray`` arity + =============== =============================================================== + + Upstream is unmaintained -- one doc-only commit since 1.3.0, and its Python + 3.12 issue (`pynetwork/pypcap#116 + `_) has been open and + uncommented since May 2024 -- so the cap is not expected to lift. Use + :class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` on 3.12 and newer. + +.. important:: + + `pcap-ct`_ installs the same top-level :mod:`pcap` module as `PyPCAP`_. This + engine therefore checks which of the two it got and raises + :exc:`~pcapkit.utilities.exceptions.UnsupportedCall` when it is `pcap-ct`_, + naming ``engine='pcap_ct'`` in the message. Running `pcap-ct`_ under the + ``PyPCAP`` name would report an engine that is not the one in use, and the two + have different install requirements and different version coverage to report. + .. important:: `PyPCAP`_ is a :manpage:`libpcap(3)` binding aimed primarily at live capture. @@ -127,6 +185,155 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. .. autoattribute:: _dlink .. autoattribute:: _closed +pcap-ct Support +=============== + +.. module:: pcapkit.foundation.engines.pcap_ct + +This module contains the implementation for `pcap-ct`_ engine +support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. + +.. _pcap-ct: https://pypi.org/project/pcap-ct/ +.. _libpcap: https://pypi.org/project/libpcap/ + +.. important:: + + `pcap-ct`_ is an independent :mod:`ctypes` reimplementation of the `PyPCAP`_ + interface, by a different author, on top of the `libpcap`_ distribution. It is + a **separate engine** rather than a second backend for + :class:`~pcapkit.foundation.engines.pypcap.PyPCAP`: select it with + ``engine='pcap_ct'``. + + Its reason for existing is coverage. Upstream `PyPCAP`_ stops at Python 3.11 + (see the note under `PyPCAP Support`_ above); `pcap-ct`_ and `libpcap`_ both + publish ``py3-none-any`` wheels, so **installing** this engine needs no + compiler and no ``pcap.h``: + + .. code-block:: shell + + pip install pypcapkit[PCAP_CT] + + Verified end to end on Python **3.10.20** and **3.14.7**: both read + :file:`examples/captures/in.pcap` through ``engine='pcap_ct'`` and return the + same six frames with identical timestamps. So the two engines together cover + every interpreter the project supports, and ``PCAP_CT`` covers all of it on + its own: + + ========================================================================= ================== + Engine Python + ========================================================================= ================== + :class:`~pcapkit.foundation.engines.pypcap.PyPCAP` (``pypcap``) 3.10, 3.11 + :class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` (``pcap-ct``) 3.10 and newer + ========================================================================= ================== + +.. warning:: + + **A system** :manpage:`libpcap(3)` **is still required at run time**, and this + is the easiest thing to get wrong about `pcap-ct`_. `libpcap`_ ships a vendored + :file:`libpcap.so` under ``_platform/{linux,macos,windows}/`` but does **not** + use it by default: its :file:`libpcap.cfg` reads ``LIBPCAP = None`` as + published, which sends the loader to :func:`ctypes.util.find_library`, so what + gets mapped is the host's ``libpcap.so.1``. + + Measured, not assumed: the same ``libpcap`` 1.11.0b29 wheel loaded + :file:`/usr/lib64/libpcap.so.1.5.3` under one interpreter on this host and + linuxbrew's 1.11.0 under another, purely because their loader search paths + differ. Setting ``LIBPCAP = tcpdump`` in :file:`libpcap.cfg` selects the + vendored copy instead, which reports libpcap 1.10.6. + + With no system :manpage:`libpcap(3)` at all, ``import pcap`` raises + :exc:`OSError` rather than :exc:`ImportError`, which would escape + :meth:`Extractor.import_test + ` and abort the + extraction. :meth:`PCAP_CT.unsupported_reason + ` detects it and + reports it as an ordinary "engine unavailable", so the extraction falls back to + the default engine with a warning naming the missing library. + +.. warning:: + + **Both distributions are pre-releases.** ``pcap-ct`` 1.3.0b3 and ``libpcap`` + 1.11.0b29 are the newest published versions, and neither project has ever + published a stable release -- which is also why ``pip install`` resolves them + without ``--pre``. ``pcap-ct`` further documents itself as tracking the + `PyPCAP`_ **1.2.3** interface rather than 1.3.0. + + Every attribute this engine touches -- the ``pcap.pcap(name=..., promisc=...)`` + constructor, :meth:`~pcap.pcap.datalink`, ``snaplen``, iteration yielding + ``(timestamp, bytes)``, and :meth:`~pcap.pcap.close` -- is present on both and + was measured to behave identically, byte for byte and timestamp for timestamp, + on ``pcap-ct`` 1.3.0b3 against ``pypcap`` 1.3.0. That is a statement about the + versions tested, not a guarantee from upstream. ``PCAP_CT`` is therefore not + included in the ``all`` extra: a beta should be asked for by name. + +.. important:: + + Being the same interface, `pcap-ct`_ has the same limits. It performs **no + protocol dissection**: each frame is the ``(timestamp, bytes)`` pair that + :c:func:`pcap_next_ex` produced. Reassembly and flow tracing are therefore + unavailable regardless of which backend is installed, and are disabled -- with + an :class:`~pcapkit.utilities.warnings.AttributeWarning` -- when requested; + the adapters in :mod:`pcapkit.toolkit.pcap_ct` raise + :exc:`~pcapkit.utilities.exceptions.UnsupportedCall` rather than fail + obscurely. It also reads from a file on disk only, because + :c:func:`pcap_open_offline` opens a savefile by *name*; a non-file input is + rejected with an :exc:`~pcapkit.utilities.exceptions.UnsupportedCall`. + + The timestamp is seconds since the epoch as a :class:`float`. `pcap-ct`_ opens + savefiles asking for nanosecond precision via + :c:func:`pcap_open_offline_with_tstamp_precision`, so a microsecond-resolution + savefile is scaled up by :manpage:`libpcap(3)` rather than truncated; for + :file:`examples/captures/in.pcap` the result matches each record header's + ``ts_sec + ts_usec * 1e-6`` exactly. + +.. note:: + + PCAP-NG is rejected with a + :exc:`~pcapkit.utilities.exceptions.FormatError`, for both engines, and the + reason is that accepting it would be *unpredictable* rather than merely + limited. :manpage:`libpcap(3)` can read a PCAP-NG savefile, but how well + depends on the version the host provides -- which, per the warning above, the + `libpcap`_ wheel does not pin. Both measured on + :file:`examples/captures/dhcp.pcapng`: + + ================ =========================================================== + System libpcap Result + ================ =========================================================== + 1.11.0 correct -- the same four frames and the same sub-second + timestamps as + :class:`~pcapkit.foundation.engines.pcapng.PCAPNG` + 1.5.3 four frames, but nonsense sub-second timestamps + (``1102274184.0000002`` and the like), silently and with no + error + ================ =========================================================== + + Even on a good version, :c:func:`pcap_datalink` reports a single link type for + the whole file, so a capture whose interfaces differ would have one interface's + link type applied to every frame. + :class:`~pcapkit.foundation.engines.pcapng.PCAPNG` reads the per-interface + blocks properly and does not depend on the host's library at all, so PCAP-NG is + routed there rather than read approximately -- or wrongly -- here. + +.. autoclass:: pcapkit.foundation.engines.pcap_ct.PCAP_CT + :no-members: + :show-inheritance: + + .. autoattribute:: __engine_name__ + .. autoattribute:: __engine_module__ + + .. autoproperty:: dlink + + .. automethod:: __init__ + .. automethod:: run + .. automethod:: read_frame + .. automethod:: close + + .. autoattribute:: _expkg + .. autoattribute:: _handle + .. autoattribute:: _extmp + .. autoattribute:: _dlink + .. autoattribute:: _closed + PyPCAPFile Support ================== diff --git a/docs/source/pcapkit/foundation/engines/index.rst b/docs/source/pcapkit/foundation/engines/index.rst index 7091f912a4..d6b7da480c 100644 --- a/docs/source/pcapkit/foundation/engines/index.rst +++ b/docs/source/pcapkit/foundation/engines/index.rst @@ -6,7 +6,8 @@ Engine Support :mod:`pcapkit.foundation.engines` is a collection of engines support for :mod:`pcapkit`, including but not limited to the built-in PCAP and `PCAP-NG`_ file support, `Scapy`_, `PyShark`_, -`DPKT`_, `PyPCAP`_ and `PyPCAPFile`_ 3rd party engine support. +`DPKT`_, `PyPCAP`_, `pcap-ct`_ and `PyPCAPFile`_ 3rd party engine +support. .. seealso:: @@ -45,6 +46,7 @@ class hierarchy of :mod:`pcapkit.foundation.engines`: DPKT PyShark PyPCAP + PCAP_CT PyPCAPFile end B --> third-party @@ -64,6 +66,7 @@ class hierarchy of :mod:`pcapkit.foundation.engines`: click DPKT "/pcapkit/foundation/engines/3rdparty.html#pcapkit.foundation.engines.dpkt.DPKT" click PyShark "/pcapkit/foundation/engines/3rdparty.html#pcapkit.foundation.engines.pyshark.PyShark" click PyPCAP "/pcapkit/foundation/engines/3rdparty.html#pcapkit.foundation.engines.pypcap.PyPCAP" + click PCAP_CT "/pcapkit/foundation/engines/3rdparty.html#pcapkit.foundation.engines.pcap_ct.PCAP_CT" click PyPCAPFile "/pcapkit/foundation/engines/3rdparty.html#pcapkit.foundation.engines.pypcapfile.PyPCAPFile" Not every engine can do everything :class:`~pcapkit.foundation.extraction.Extractor` @@ -82,6 +85,10 @@ offers, and the ones that cannot say so rather than quietly doing less: | | :exc:`~pcapkit.utilities.exceptions.FormatError` / | | | :exc:`~pcapkit.utilities.exceptions.UnsupportedCall` | +-----------------------------------------------------------------+---------------------------------------------------------------+ +| :class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` | the same gaps as ``PyPCAP`` above -- it reads the same | +| | interface, so no dissection, no reassembly and no flow | +| | tracing, and PCAP savefiles on disk only | ++-----------------------------------------------------------------+---------------------------------------------------------------+ | :class:`~pcapkit.foundation.engines.pypcapfile.PyPCAPFile` | no IPv6 decoder, hence no IPv6 reassembly -- disabled with an | | | :class:`~pcapkit.utilities.warnings.AttributeWarning`, and | | | :func:`~pcapkit.toolkit.pypcapfile.ipv6_reassembly` raises; | @@ -105,30 +112,106 @@ so it is worth knowing in advance. | :class:`~pcapkit.foundation.engines.pcap.PCAP`, | nothing -- built in | | :class:`~pcapkit.foundation.engines.pcapng.PCAPNG` | | +-----------------------------------------------------------------+---------------------------------------------------------------+ -| :class:`~pcapkit.foundation.engines.dpkt.DPKT`, | nothing -- both ship pure-Python wheels | -| :class:`~pcapkit.foundation.engines.scapy.Scapy` | | +| :class:`~pcapkit.foundation.engines.dpkt.DPKT`, | nothing -- both ship pure-Python wheels, and both were | +| :class:`~pcapkit.foundation.engines.scapy.Scapy` | measured reading a capture on Python 3.14, so neither | +| | overrides ``unsupported_reason()`` | +-----------------------------------------------------------------+---------------------------------------------------------------+ -| :class:`~pcapkit.foundation.engines.pyshark.PyShark` | Wireshark's :program:`tshark` binary on ``PATH`` | +| :class:`~pcapkit.foundation.engines.pyshark.PyShark` | Wireshark's :program:`tshark` binary -- on :envvar:`PATH`, or | +| | wherever ``pyshark``'s :file:`config.ini` points -- **and** | +| | Python **3.13 or older**: ``pyshark`` 0.6 builds its event | +| | loop with :func:`asyncio.get_event_loop`, which raises from | +| | 3.14 | +-----------------------------------------------------------------+---------------------------------------------------------------+ | :class:`~pcapkit.foundation.engines.pypcap.PyPCAP` | `libpcap`_ headers and library, a C compiler, and Python | | | **3.11 or older** -- ``pypcap`` 1.3.0 publishes no wheel and | | | its pre-generated :file:`pcap.c` does not compile on 3.12+ | +-----------------------------------------------------------------+---------------------------------------------------------------+ +| :class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` | a system ``libpcap.so.1`` at *run* time -- nothing to build, | +| | since ``pcap-ct`` and ``libpcap`` ship pure-Python wheels, | +| | but see the warning below: the vendored library those wheels | +| | carry is not what gets loaded by default | ++-----------------------------------------------------------------+---------------------------------------------------------------+ | :class:`~pcapkit.foundation.engines.pypcapfile.PyPCAPFile` | Python **3.11 or older** -- ``pypcapfile`` 0.12.0 imports the | | | ``imp`` module, removed in Python 3.12 | +-----------------------------------------------------------------+---------------------------------------------------------------+ +Every one of these constraints is also enforced in code rather than only +documented: each engine overrides +:meth:`~pcapkit.foundation.engines.engine.EngineBase.unsupported_reason`, which +:meth:`Extractor.run ` consults +*before* the import test, so asking for an engine that cannot run here produces +one warning naming the actual cause and a clean fall back to the built-in parser. + .. seealso:: :doc:`../../../index` covers the installation prerequisites in full, including how ``pypcap``'s :file:`setup.py` looks for :file:`pcap.h` and why a Homebrew ``libpcap`` is not always found. +Two engines, one interface: choosing between PyPCAP and PCAP_CT +--------------------------------------------------------------- + +:class:`~pcapkit.foundation.engines.pypcap.PyPCAP` and +:class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` read the same +:manpage:`libpcap(3)` interface from two independent distributions, both of which +install a top-level :mod:`pcap` module. They are separate engines because they +are separate projects, with different authors, different install requirements +and different interpreter coverage: + ++---------------------------------------------------------------+---------------------+-------------------+---------------------------------------------------+ +| Engine (``engine=``) | Distribution | Python | What it needs | ++===============================================================+=====================+===================+===================================================+ +| :class:`~pcapkit.foundation.engines.pypcap.PyPCAP` | `PyPCAP`_ 1.3.0 | 3.10, 3.11 only | to build: a C compiler, ``pcap.h`` and a system | +| (``'pypcap'``) | (stable) | | libpcap -- no wheels are published | ++---------------------------------------------------------------+---------------------+-------------------+---------------------------------------------------+ +| :class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` | `pcap-ct`_ 1.3.0b3 | 3.10 and newer | to install: nothing -- ``py3-none-any`` wheels. | +| (``'pcap_ct'``) | + `libpcap`_ | | At run time: a system ``libpcap.so.1`` all the | +| | 1.11.0b29 (**beta**)| | same -- see the warning below | ++---------------------------------------------------------------+---------------------+-------------------+---------------------------------------------------+ + +Neither engine dissects anything, so the capability gaps in the table above are +identical for both and are a property of :manpage:`libpcap(3)`, not of the +distribution: **reassembly and flow tracing are unavailable whichever one is +installed.** Pick on availability, not on features. + +.. warning:: + + ``PCAP_CT`` needs no toolchain to *install*, but it does need a system + :manpage:`libpcap(3)` to *run*. `libpcap`_ ships a vendored + :file:`libpcap.so` and, as published, does not use it: its + :file:`libpcap.cfg` says ``LIBPCAP = None``, which sends the loader to + :func:`ctypes.util.find_library`. With no system library at all + ``import pcap`` raises :exc:`OSError` rather than :exc:`ImportError`; + :meth:`PCAP_CT.unsupported_reason + ` catches that + and turns it into an ordinary fall back to the default engine, instead of the + hard error it would otherwise be. + +**The two distributions collide, so install exactly one.** Both own the top-level +:mod:`pcap` module, and pip will happily install both -- measured on Python 3.10, +the ``pcap-ct`` package then wins the import and upstream's extension module is +shadowed and unreachable. Each engine therefore detects which distribution it +actually got, via +:func:`pcapkit.foundation.engines._pcap_backend.probe`, and: + +* reports it -- :attr:`PyPCAP.backend + ` and + :attr:`PCAP_CT.backend ` + name the distribution, version and file actually in use, so a bug report about + "the pypcap engine" says which one ran; +* declines to run on the other one, through ``unsupported_reason()``, with a + message naming the ``engine=`` string that does want it; and +* warns with an :class:`~pcapkit.utilities.warnings.EngineWarning` when it finds + both installed, since that state makes one of the two engines permanently + unselectable and nothing else would explain why. + .. _PCAP-NG: https://wiki.wireshark.org/Development/PcapNg +.. _libpcap: https://pypi.org/project/libpcap/ .. _Scapy: https://scapy.net .. _DPKT: https://dpkt.readthedocs.io .. _PyShark: https://kiminewt.github.io/pyshark .. _PyPCAP: https://github.com/pynetwork/pypcap +.. _pcap-ct: https://pypi.org/project/pcap-ct/ .. _PyPCAPFile: https://github.com/kisom/pypcapfile .. _libpcap: https://www.tcpdump.org diff --git a/docs/source/pcapkit/toolkit/3rdparty.rst b/docs/source/pcapkit/toolkit/3rdparty.rst index 18167dce0b..11070ec49c 100644 --- a/docs/source/pcapkit/toolkit/3rdparty.rst +++ b/docs/source/pcapkit/toolkit/3rdparty.rst @@ -122,6 +122,46 @@ Auxiliary Functions .. autofunction:: pcapkit.toolkit.pypcap.packet2dict +pcap-ct Tools +============= + +.. module:: pcapkit.toolkit.pcap_ct + +:mod:`pcapkit.toolkit.pcap_ct` contains all you need for +:mod:`pcapkit` handy usage with `pcap-ct`_ engine. All reforming +functions returns with a flag to indicate if usable for +its caller. + +.. _pcap-ct: https://pypi.org/project/pcap-ct/ + +.. note:: + + `pcap-ct`_ is an independent reimplementation of the `PyPCAP`_ interface, so + this module is deliberately a sibling of :mod:`pcapkit.toolkit.pypcap` rather + than an alias of it: each engine names its own adapter, so a change made for + one cannot quietly alter the other. + + Like `PyPCAP`_ it performs no protocol dissection, so the reassembly and flow + tracing adapters below cannot be implemented. They are defined all the same, + so that reaching for one fails with an explanatory + :exc:`~pcapkit.utilities.exceptions.UnsupportedCall` rather than an + :exc:`ImportError`. + +.. autofunction:: pcapkit.toolkit.pcap_ct.ipv4_reassembly + +.. autofunction:: pcapkit.toolkit.pcap_ct.ipv6_reassembly + +.. autofunction:: pcapkit.toolkit.pcap_ct.tcp_reassembly + +.. autofunction:: pcapkit.toolkit.pcap_ct.tcp_traceflow + +Auxiliary Functions +------------------- + +.. autofunction:: pcapkit.toolkit.pcap_ct.packet2chain + +.. autofunction:: pcapkit.toolkit.pcap_ct.packet2dict + PyPCAPFile Tools ================ diff --git a/examples/legacy_smoke/_engine_support.py b/examples/legacy_smoke/_engine_support.py index 7fbd1e4bed..0ba6da170a 100644 --- a/examples/legacy_smoke/_engine_support.py +++ b/examples/legacy_smoke/_engine_support.py @@ -39,7 +39,7 @@ #: ``'pcapkit'``) and is the only one with no third-party requirement; further #: engines can be added at runtime with #: :func:`pcapkit.foundation.registry.foundation.register_extractor_engine`. -ENGINES = ('default', 'pyshark', 'scapy', 'dpkt') +ENGINES = ('default', 'pyshark', 'scapy', 'dpkt', 'pypcap', 'pcap_ct', 'pypcapfile') #: ``__engine_name__`` of the driver each ``engine=`` value should end up using. #: ``'default'`` and ``'pcapkit'`` pick their parser from the file's magic number, @@ -50,6 +50,9 @@ 'dpkt': ('DPKT',), 'scapy': ('Scapy',), 'pyshark': ('PyShark',), + 'pypcap': ('PyPCAP',), + 'pcap_ct': ('PCAP_CT',), + 'pypcapfile': ('PyPCAPFile',), } @@ -101,6 +104,34 @@ def ran_as_asked(engine: 'str', extraction: 'Extractor') -> 'tuple[str, bool]': return driver, driver in ENGINE_DRIVERS.get(engine, (engine,)) +def _declared_reason(engine: 'str') -> 'str | None': + """What the engine itself says about running here, if it says anything. + + Looks the engine class up in ``Extractor.__engine__`` and asks its + ``unsupported_reason()``. Returns :data:`None` for ``'default'``, for an engine + name the registry does not know, or for anything that goes wrong on the way -- + this is a nicety for the demos' skip messages, and must never be the reason one + of them fails. + + Args: + engine: Engine name, as passed to ``pcapkit.extract``. + + Returns: + The engine's own reason, or :data:`None`. + + """ + try: + from pcapkit.foundation.extraction import Extractor + + registered = Extractor.__engine__.get(engine) + if registered is None: + return None + klass = getattr(registered, 'klass', registered) + return klass.unsupported_reason() + except Exception: # pylint: disable=broad-except + return None + + def report(engine: 'str', detail: 'str') -> 'None': """Print one line of an engine report. @@ -133,6 +164,16 @@ def preflight(engine: 'str', fin: 'str') -> 'str | None': """ import pcapkit # imported here so this module stays importable on its own + # Ask the engine first. ``unsupported_reason`` is the same preflight check + # ``Extractor.run`` consults, and it names the actual cause -- a Python + # ceiling, a missing tshark, a missing libpcap, the wrong ``pcap`` + # distribution. Without it the fallback branch below is all that fires, and it + # can only guess "its package is not installed", which is wrong whenever the + # package is installed and unusable. + reason = _declared_reason(engine) + if reason is not None: + return reason + try: extraction = pcapkit.extract(fin=fin, store=False, nofile=True, verbose=False, engine=engine) # type: ignore[arg-type] diff --git a/pcapkit/foundation/engines/__init__.py b/pcapkit/foundation/engines/__init__.py index bf68327fd0..32ad7dc0aa 100644 --- a/pcapkit/foundation/engines/__init__.py +++ b/pcapkit/foundation/engines/__init__.py @@ -7,8 +7,10 @@ :mod:`pcapkit.foundation.engines` is a collection of engines support for :mod:`pcapkit`, including but not limited to the built-in PCAP and `PCAP-NG`_ file support, :mod:`Scapy `, -:mod:`PyShark `, :mod:`DPKT `, :mod:`PyPCAP ` -and :mod:`PyPCAPFile ` 3rd party engine support. +:mod:`PyShark `, :mod:`DPKT `, :mod:`PyPCAP `, +`pcap-ct`_ and :mod:`PyPCAPFile ` 3rd party engine support. + +.. _pcap-ct: https://pypi.org/project/pcap-ct/ .. _PCAPNG: https://wiki.wireshark.org/Development/PcapNg @@ -25,10 +27,11 @@ from pcapkit.foundation.engines.dpkt import DPKT from pcapkit.foundation.engines.pyshark import PyShark from pcapkit.foundation.engines.pypcap import PyPCAP +from pcapkit.foundation.engines.pcap_ct import PCAP_CT from pcapkit.foundation.engines.pypcapfile import PyPCAPFile __all__ = [ 'PCAP', 'PCAPNG', - 'Scapy', 'DPKT', 'PyShark', 'PyPCAP', 'PyPCAPFile', + 'Scapy', 'DPKT', 'PyShark', 'PyPCAP', 'PCAP_CT', 'PyPCAPFile', ] diff --git a/pcapkit/foundation/engines/_pcap_backend.py b/pcapkit/foundation/engines/_pcap_backend.py new file mode 100644 index 0000000000..581ec19997 --- /dev/null +++ b/pcapkit/foundation/engines/_pcap_backend.py @@ -0,0 +1,273 @@ +# -*- coding: utf-8 -*- +"""Which distribution owns the :mod:`pcap` module? +===================================================== + +.. module:: pcapkit.foundation.engines._pcap_backend + +Two unrelated PyPI distributions install a top-level module named :mod:`pcap`: + +* `PyPCAP`_ -- a Cython binding, shipped as a single extension module + (:file:`pcap.cpython-*.so`), installable only up to Python 3.11. +* `pcap-ct`_ -- an independent :mod:`ctypes` reimplementation of the same + interface, shipped as a *package* (:file:`pcap/__init__.py`), installable on + 3.10 and newer. + +They therefore **collide**: nothing stops both from being installed, and +``import pcap`` then silently resolves to whichever the import system finds +first. Measured on Python 3.10 with both present, the ``pcap/`` package wins and +upstream's extension module is shadowed and unreachable -- so ``pcap-ct`` always +takes precedence, and no amount of ordering by the caller changes it. + +:class:`~pcapkit.foundation.engines.pypcap.PyPCAP` and +:class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` are separate engines, so each +has to know which distribution it actually got rather than assume. This module is +the one place that answers that, deliberately shared: the two engines must agree +on the answer, and two copies of the detection would be two chances to disagree. +It is *only* detection -- nothing here is engine behaviour -- and it imports +nothing from :mod:`pcapkit`, so it cannot introduce an import cycle. + +.. _PyPCAP: https://github.com/pynetwork/pypcap +.. _pcap-ct: https://pypi.org/project/pcap-ct/ + +""" +import importlib +import importlib.metadata +import sys +from typing import TYPE_CHECKING, NamedTuple + +__all__ = [ + 'PYPCAP', 'PCAP_CT', 'DISTRIBUTIONS', 'ENGINE_NAMES', + 'Probe', 'probe', 'identify', 'installed_distributions', + 'wrong_backend_reason', 'collision_reason', +] + +if TYPE_CHECKING: + from types import ModuleType + from typing import Optional + +#: Distribution name of upstream `PyPCAP`_. +PYPCAP = 'pypcap' +#: Distribution name of `pcap-ct`_. +PCAP_CT = 'pcap-ct' +#: Every distribution known to provide :mod:`pcap`, in a fixed order so that +#: messages naming several of them read the same way every time. +DISTRIBUTIONS = (PYPCAP, PCAP_CT) +#: The ``engine=`` string that drives each distribution. Kept here rather than in +#: either engine so that a message pointing the user at the *other* engine cannot +#: name one that does not exist. +ENGINE_NAMES = { + PYPCAP: 'pypcap', + PCAP_CT: 'pcap_ct', +} + + +class Probe(NamedTuple): + """What one attempt to import :mod:`pcap` found.""" + + #: Distribution that provided the imported module -- :data:`PYPCAP`, + #: :data:`PCAP_CT`, or :data:`None` when the import did not succeed. + name: 'Optional[str]' + #: ``pcap.__version__``, when there was a module to read it from. + version: 'Optional[str]' + #: ``pcap.__file__``, which is what distinguishes an extension module from a + #: package directory to a human reading a bug report. + origin: 'Optional[str]' + #: Why the import failed, as a short phrase, or :data:`None` on success. + failure: 'Optional[str]' + #: Whether the failure was simply "not installed", i.e. an + #: :exc:`ImportError`. This matters because + #: :meth:`Extractor.import_test + #: ` already reports that + #: case perfectly well, whereas the *other* kind of failure escapes it -- + #: see :func:`probe`. + missing: 'bool' + #: Distributions found installed, whether or not their module was importable. + #: More than one means the collision described in this module's docstring. + installed: 'tuple[str, ...]' + + def describe(self) -> 'str': + """A one-line description fit for a warning or a bug report. + + Returns: + Something like ``pcap-ct 1.3.0b3 (/.../pcap/__init__.py)``, or a + phrase naming the failure when there is no module to describe. + + """ + if self.name is None: + return f'no usable `pcap` module ({self.failure})' + version = self.version or 'unknown version' + origin = self.origin or 'unknown location' + return f'{self.name} {version} ({origin})' + + +def identify(module: 'ModuleType') -> 'str': + """Which distribution does an imported :mod:`pcap` module come from? + + `pcap-ct`_ ships :mod:`pcap` as a package whose ``__init__`` does + ``from ._pcap import *``, which binds the submodule as an attribute; upstream + `PyPCAP`_ ships a single extension module, which has no such attribute. That + is a structural difference rather than a cosmetic one, which is why it is + preferred here over the alternatives: + + * ``pcap.__version__`` is ``1.3.0b3`` against ``1.3.0`` today, but that is a + coincidence of release timing and would stop separating them the moment + ``pcap-ct`` cuts a 1.3.0 final. + * ``pcap.ex_name`` looked like a ``pcap-ct`` marker and is **not** -- measured + present on upstream ``pypcap`` 1.3.0 as well. + + Args: + module: An already-imported :mod:`pcap` module. + + Returns: + :data:`PCAP_CT` or :data:`PYPCAP`. + + .. _PyPCAP: https://github.com/pynetwork/pypcap + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + + """ + return PCAP_CT if hasattr(module, '_pcap') else PYPCAP + + +def installed_distributions() -> 'tuple[str, ...]': + """Which of :data:`DISTRIBUTIONS` are installed, per package metadata. + + Asked of the metadata rather than of :mod:`pcap` itself, because that is the + only way to see the distribution the import did *not* resolve to -- which is + exactly the collision worth reporting. + + Returns: + The installed subset of :data:`DISTRIBUTIONS`, in that order. + + """ + found = [] # type: list[str] + for name in DISTRIBUTIONS: + try: + importlib.metadata.distribution(name) + except importlib.metadata.PackageNotFoundError: + continue + except Exception: # pylint: disable=broad-except + # A corrupt or unreadable ``dist-info`` is not a reason to fail an + # extraction. Detection is a courtesy; treat "cannot tell" as "not + # installed" and let the import attempt be the authority. + continue + found.append(name) + return tuple(found) + + +def _purge() -> 'None': + """Drop the ``pcap`` and ``libpcap`` module trees from :data:`sys.modules`. + + Called after a failed import so that the *next* probe reproduces the same + failure. Without it a second attempt reports something else entirely, which + was measured rather than imagined. + + Both ``pcap-ct``'s and ``libpcap``'s package initialisers open with + ``from .__about__ import * ; del __about__``, which is **not safe to re-run**: + the ``del`` needs a name that only gets bound as a side effect of importing + the submodule fresh. When the initialiser fails part-way -- as it does with no + system :manpage:`libpcap(3)`, where the real error is + ``OSError: Cannot find libpcap.so library`` -- Python removes the package it + was executing but leaves the ``__about__`` submodule cached, so the retry + reaches the ``del`` with nothing bound and dies with + ``NameError: name '__about__' is not defined``. + + That message names neither the missing library nor the package it came from, + and it is what the user would otherwise be shown. ``libpcap`` is purged as + well as ``pcap`` because the residue is in whichever of the two got part-way: + purging only ``pcap`` moved the ``NameError`` from one to the other rather + than removing it. + + """ + roots = ('pcap', 'libpcap') + # computed once rather than per entry in sys.modules, which can be large + prefixes = tuple(f'{root}.' for root in roots) + for name in [name for name in sys.modules + if name in roots or name.startswith(prefixes)]: + del sys.modules[name] + + +def probe() -> 'Probe': + """Import :mod:`pcap` and report what was found. + + Not cached. The cost after a successful first call is a :data:`sys.modules` + lookup, and a cache would make the answer depend on when it was first asked -- + which the tests, and anything that manipulates :data:`sys.path`, would have to + work around. It is idempotent instead, via :func:`_purge`. + + Note: + The bare ``except Exception`` is deliberate and is much of the point of + this function. ``pcap-ct`` imports the ``libpcap`` distribution, whose + Linux loader calls :func:`ctypes.util.find_library` and raises + :exc:`OSError` -- ``Cannot find libpcap.so library`` -- when no system + :manpage:`libpcap(3)` is present. :exc:`OSError` is not an + :exc:`ImportError`, so :meth:`Extractor.import_test + ` does not catch it + and it escapes as a hard error instead of degrading to the default engine. + Catching it here is what lets an engine report it as a reason instead. + + Returns: + A :class:`Probe` describing the outcome. + + """ + installed = installed_distributions() + + try: + module = importlib.import_module('pcap') + except ImportError as exc: + _purge() + return Probe(None, None, None, str(exc) or 'no module named `pcap`', True, installed) + except Exception as exc: # pylint: disable=broad-except + _purge() + return Probe(None, None, None, f'{type(exc).__name__}: {exc}', False, installed) + + return Probe( + identify(module), + getattr(module, '__version__', None), + getattr(module, '__file__', None), + None, + False, + installed, + ) + + +def wrong_backend_reason(wanted: 'str', found: 'Probe') -> 'Optional[str]': + """Why ``wanted`` cannot run, when some *other* distribution owns :mod:`pcap`. + + Args: + wanted: The distribution the calling engine drives -- :data:`PYPCAP` or + :data:`PCAP_CT`. + found: The result of :func:`probe`. + + Returns: + A reason naming what was found and which ``engine=`` string wants it, or + :data:`None` when ``wanted`` is what is installed, or when nothing is -- + an absent module is not this function's business, since + :meth:`Extractor.import_test + ` reports that. + + """ + if found.name is None or found.name == wanted: + return None + + alternative = ENGINE_NAMES.get(found.name) + suggestion = f"; use 'engine={alternative}' for it" if alternative else '' + return (f'the installed `pcap` module is {found.name} ' + f'({found.version or "unknown version"}), not {wanted}{suggestion}') + + +def collision_reason(found: 'Probe') -> 'Optional[str]': + """A description of both distributions being installed at once, if they are. + + Returns: + A phrase naming every installed distribution and which one ``import pcap`` + actually resolved to, or :data:`None` when at most one is installed. + + """ + if len(found.installed) < 2: + return None + + names = ' and '.join(found.installed) + return (f'{names} are installed together, and both provide the `pcap` module; ' + f'`import pcap` resolved to {found.describe()}, leaving the other ' + f'shadowed and unreachable -- install exactly one of them so the ' + f'choice is explicit') diff --git a/pcapkit/foundation/engines/pcap_ct.py b/pcapkit/foundation/engines/pcap_ct.py new file mode 100644 index 0000000000..6e66ec3188 --- /dev/null +++ b/pcapkit/foundation/engines/pcap_ct.py @@ -0,0 +1,488 @@ +# -*- coding: utf-8 -*- +"""pcap-ct Support +================== + +.. module:: pcapkit.foundation.engines.pcap_ct + +This module contains the implementation for `pcap-ct`_ engine +support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. + +.. _pcap-ct: https://pypi.org/project/pcap-ct/ + +""" +import os +from typing import TYPE_CHECKING, cast + +from pcapkit.const.reg.linktype import LinkType as Enum_LinkType +from pcapkit.foundation.engines import _pcap_backend +from pcapkit.foundation.engines.engine import EngineBase as Engine +from pcapkit.foundation.reassembly import ReassemblyManager +from pcapkit.foundation.traceflow import TraceFlowManager +from pcapkit.utilities.exceptions import FormatError, UnsupportedCall, stacklevel +from pcapkit.utilities.warnings import AttributeWarning, EngineWarning, warn + +__all__ = ['PCAP_CT'] + +if TYPE_CHECKING: + from typing import Iterator, Optional + + from pcap import pcap as Handle + + from pcapkit.foundation.extraction import Extractor + + #: A `pcap-ct`_ "frame": the ``(timestamp, bytes)`` pair that + #: :class:`pcap.pcap` iteration yields. Deliberately *not* named ``Frame``, + #: so that it is not mistaken for + #: :class:`pcapkit.protocols.misc.pcap.frame.Frame`, which is what the + #: built-in engines return. + RawFrame = tuple[float, bytes] + + +class PCAP_CT(Engine['RawFrame']): + """pcap-ct engine support. + + `pcap-ct`_ is a :mod:`ctypes` reimplementation of the `PyPCAP`_ API on top of + the `libpcap`_ package, which supplies the :manpage:`libpcap(3)` bindings + (and ships a vendored copy of the library that it does not, by default, use + -- see the warning below). Both distributions install a + top-level :mod:`pcap` module and expose the same interface, so this engine is + a sibling of :class:`~pcapkit.foundation.engines.pypcap.PyPCAP` rather than a + replacement for it -- see :attr:`__engine_module__` for how the two are told + apart, and :doc:`the engine documentation + ` for which one to install. + + It exists because upstream `PyPCAP`_ cannot be installed on a current + interpreter: it ships no wheels and its ``pcap.c`` was pre-generated by + Cython 0.29.32, which does not compile against the Python 3.12+ C API. + `pcap-ct`_ and `libpcap`_ both publish ``py3-none-any`` wheels, so **installing** + this engine needs no compiler and no ``pcap.h``. + + .. warning:: + + It does still need a **system** :manpage:`libpcap(3)` at *run* time, and + this is easy to get wrong: `libpcap`_ ships a vendored + :file:`libpcap.so` under :file:`_platform/{linux,macos,windows}/` but does + **not** use it by default. Its :file:`libpcap.cfg` reads ``LIBPCAP = None`` + as published, which sends its loader to + :func:`ctypes.util.find_library`, so the library actually mapped is the + host's ``libpcap.so.1``. Measured on this host: the same + ``libpcap`` 1.11.0b29 wheel loaded ``/usr/lib64/libpcap.so.1.5.3`` under one + interpreter and linuxbrew's 1.11.0 under another, purely because their + loader search paths differ. + + Two consequences. With no system :manpage:`libpcap(3)` at all, + ``import pcap`` raises :exc:`OSError` -- not :exc:`ImportError` -- which + :meth:`unsupported_reason` exists partly to catch. And the + :manpage:`libpcap(3)` *version* in play is a property of the host rather + than of the wheel, which is why the PCAP-NG note on :meth:`run` does not + rely on it. Setting ``LIBPCAP = tcpdump`` in :file:`libpcap.cfg` selects the + vendored copy instead -- verified loading, and reporting libpcap 1.10.6. + + Being the same interface, it has the same limits. Offline, :mod:`pcap` is a + savefile reader and nothing more: iteration yields the ``(timestamp, bytes)`` + pair from :c:func:`pcap_next_ex` and no protocol dissection is performed. + Consequently this engine + + * returns each frame as a ``(timestamp, bytes)`` :obj:`tuple` rather than as + a parsed packet object, and + * disables both reassembly and flow tracing, warning as it does so, since + neither can be derived without an IP or TCP layer to read. + + It also requires the input to be a real file on disk, because + :c:func:`pcap_open_offline` opens by *name* -- there is no way to hand + :manpage:`libpcap(3)` an already-open Python stream. + + .. important:: + + Both distributions are published as pre-releases only -- ``pcap-ct`` + 1.3.0b3 and ``libpcap`` 1.11.0b29 at the time of writing -- and + ``pcap-ct`` documents itself as tracking the `PyPCAP`_ **1.2.3** API + rather than 1.3.0. Every attribute this engine touches is present and + behaves identically on both (measured on ``pcap-ct`` 1.3.0b3 against + ``pypcap`` 1.3.0), but that is a statement about the versions tested, not + a guarantee from upstream. + + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + .. _libpcap: https://pypi.org/project/libpcap/ + .. _PyPCAP: https://github.com/pynetwork/pypcap + + Args: + extractor: :class:`~pcapkit.foundation.extraction.Extractor` instance. + + """ + if TYPE_CHECKING: + import pcap + + #: Engine extraction package. + _expkg: 'pcap' + #: Capture handle, kept separately from the iterator it is read through + #: so that :meth:`close` does not depend on ``iter(handle) is handle``. + _handle: 'Handle' + #: Engine extraction temporary storage. + _extmp: 'Iterator[RawFrame]' + #: Data link layer protocol, from the capture handle. + _dlink: 'Enum_LinkType' + #: Closed flag, so that the handle is not closed twice. + _closed: 'bool' + #: What the :mod:`pcap` import actually found, c.f. :attr:`backend`. + _backend: '_pcap_backend.Probe' + + ########################################################################## + # Defaults. + ########################################################################## + + #: Engine name. + __engine_name__ = 'PCAP_CT' + + #: Engine module name. Note that this is ``pcap._pcap``, not ``pcap``: + #: `pcap-ct`_ and upstream `PyPCAP`_ both install a top-level :mod:`pcap`, + #: so importing that name cannot tell which of the two is present, and + #: :meth:`Extractor.import_test + #: ` decides engine + #: availability by import alone. ``pcap._pcap`` is the `pcap-ct`_ + #: implementation module; upstream ships :mod:`pcap` as a single extension + #: module rather than a package, so the submodule import fails there and the + #: two engines stay distinguishable. + __engine_module__ = 'pcap._pcap' + + #: Distribution this engine drives. + __engine_distribution__ = _pcap_backend.PCAP_CT + + ########################################################################## + # Class methods. + ########################################################################## + + @classmethod + def _reason_for(cls, found: '_pcap_backend.Probe') -> 'Optional[str]': + """The verdict on an already-taken probe. + + Split out from :meth:`unsupported_reason` so that the public hook keeps the + exact no-argument signature + :meth:`EngineBase.unsupported_reason + ` declares, + while :meth:`__init__` can pass the probe it already has. Probing twice is + not merely wasteful -- see + :func:`~pcapkit.foundation.engines._pcap_backend._purge` for why a repeated + import after a failed one does not reproduce the same error. + + """ + wrong = _pcap_backend.wrong_backend_reason(cls.__engine_distribution__, found) + if wrong is not None: + return wrong + + # ``missing`` separates "not installed", which the import test reports + # well, from a failure that would otherwise escape it -- the missing + # ``libpcap.so`` being the case this exists for. + if found.name is None and not found.missing: + return f'the `pcap` module is installed but unusable -- {found.failure}' + + return None + + @classmethod + def unsupported_reason(cls) -> 'Optional[str]': + """Why this engine cannot run here, or :data:`None` when it can. + + Consulted by :meth:`pcapkit.foundation.extraction.Extractor.run` *before* + the import test, and it answers two questions the import test cannot. + + **Which distribution owns :mod:`pcap`.** Upstream `PyPCAP`_ owns that name + just as legitimately, and ``import pcap._pcap`` -- what + :attr:`__engine_module__` names -- separates them only as long as upstream + keeps shipping a single extension module. Asking + :func:`~pcapkit.foundation.engines._pcap_backend.probe` states the check + rather than inferring it from an import that happens to fail. + + **Whether a system :manpage:`libpcap(3)` exists at all.** This is the + important one, and it is measured rather than theoretical. `pcap-ct`_ needs + no compiler because it is :mod:`ctypes`, but at *runtime* it imports the + `libpcap`_ distribution, whose Linux loader calls + :func:`ctypes.util.find_library` and raises :exc:`OSError` -- + ``Cannot find libpcap.so library`` -- when there is none. + :exc:`OSError` is not an :exc:`ImportError`, so + :meth:`Extractor.import_test + ` lets it through and + the extraction dies rather than falling back. Reporting it here turns that + into the ordinary "engine unavailable" path. + + There is deliberately **no Python version bound in either direction.** + Measured on 3.10.20 and 3.14.7: ``pcap-ct`` 1.3.0b3 with ``libpcap`` + 1.11.0b29 installs and reads :file:`examples/captures/in.pcap` identically + on both, so this engine covers the whole range the project supports. The + floor in the ``PCAP_CT`` extra's markers exists only because + :file:`pyproject.toml` still advertises ``requires-python >= 3.6``, and on + an interpreter below 3.10 the distributions simply will not be installed -- + which the import test then reports correctly on its own. + + Returns: + A phrase naming the real cause, or :data:`None`. An :mod:`pcap` that is + merely absent returns :data:`None`, since the import test reports that + case in its own words. + + .. _PyPCAP: https://github.com/pynetwork/pypcap + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + .. _libpcap: https://pypi.org/project/libpcap/ + + """ + return cls._reason_for(_pcap_backend.probe()) + + ########################################################################## + # Properties. + ########################################################################## + + @property + def dlink(self) -> 'Enum_LinkType': + """Data link layer protocol, as reported by the capture handle.""" + return self._dlink + + @property + def backend(self) -> 'str': + """The distribution, version and location this engine is driving. + + Two distributions provide :mod:`pcap`, so "the pcap-ct engine failed" is + not a complete statement of what ran. This says which one it was, e.g. + ``pcap-ct 1.3.0b3 (/.../pcap/__init__.py)``. + + """ + return self._backend.describe() + + ########################################################################## + # Data models. + ########################################################################## + + def __init__(self, extractor: 'Extractor') -> 'None': + """Initialise the engine. + + Warns: + EngineWarning: If both distributions that provide :mod:`pcap` are + installed. Only one of them can win the import -- measured on + Python 3.10 with both present, the ``pcap-ct`` package wins and + upstream's extension module is shadowed -- so the other is + unreachable, and nothing else would say why + ``engine='pypcap'`` had stopped working. + + Raises: + UnsupportedCall: If the environment cannot support this engine, for + either of the reasons :meth:`unsupported_reason` describes. That + hook is only consulted by :meth:`Extractor.run + `, which degrades to + the default engine with a warning; constructing the engine + directly bypasses it, so this is the backstop for that path. + + """ + # NOTE: the probe comes *before* ``import pcap``, and the order is + # load-bearing rather than stylistic. A bare ``import pcap`` here would + # raise ``OSError: Cannot find libpcap.so library`` on a host with no + # system libpcap, escaping both this constructor and ``Extractor``'s + # import test; ``probe`` performs the same import with that failure + # caught, so the check below can report it as a reason instead. + self._backend = _pcap_backend.probe() + + collision = _pcap_backend.collision_reason(self._backend) + if collision is not None: + warn(f"'Extractor(engine=pcap_ct)': {collision}", + EngineWarning, stacklevel=stacklevel()) + + # Hand the probe over rather than letting it be taken again: a second + # import attempt after a failed one does not reproduce the same error, for + # the reason ``_pcap_backend._purge`` documents. + reason = self._reason_for(self._backend) + if reason is not None: + raise UnsupportedCall( + f"'Extractor(engine=pcap_ct)' requires 'pcap-ct': {reason}" + ) + + import pcap # isort:skip # safe: ``probe`` already imported it cleanly + + self._expkg = pcap + self._handle = cast('Handle', None) + self._extmp = cast('Iterator[RawFrame]', None) + self._dlink = cast('Enum_LinkType', None) + self._closed = False + + super().__init__(extractor) + + ########################################################################## + # Methods. + ########################################################################## + + def run(self) -> 'None': + """Call :class:`pcap.pcap` to extract PCAP files. + + This method assigns :attr:`self._expkg ` + as :mod:`pcap`, :attr:`self._handle ` as the + :class:`pcap.pcap` capture handle and :attr:`self._extmp + ` as an iterator over it. + + Warns: + AttributeWarning: Warns under following circumstances: + + * if :attr:`self.extractor._exlyr ` + and/or :attr:`self.extractor._exptl ` + is provided as the pcap-ct engine currently does not + support such operations. + * if reassembly and/or flow tracing is enabled, as the pcap-ct + engine performs no protocol dissection and so cannot support + either operation. + + Raises: + FormatError: If the file format is not supported, i.e., not a PCAP + file. PCAP-NG is rejected explicitly even though the vendored + :manpage:`libpcap(3)` can read one -- see the note below. + UnsupportedCall: If the input is not a file on disk, as + :c:func:`pcap_open_offline` can only open a savefile by name. + + Note: + The PCAP-NG rejection is a deliberate narrowing, and the reason is + that the alternative is *unpredictable* rather than merely limited. + :manpage:`libpcap(3)` can read a PCAP-NG savefile, but how well + depends on the version the host happens to provide -- which, per the + warning in this class's docstring, is not something the `libpcap`_ + wheel pins. Both measured on + :file:`examples/captures/dhcp.pcapng`: + + * against libpcap **1.11.0** it reads correctly, yielding the same + four frames and the same sub-second timestamps as + :class:`~pcapkit.foundation.engines.pcapng.PCAPNG`; + * against libpcap **1.5.3** it yields the four frames but with + nonsense sub-second timestamps -- ``1102274184.0000002`` and the + like -- silently, with no error. + + Even on a good version it cannot represent more than one interface: + :c:func:`pcap_datalink` returns a single link type for the whole file, + so a capture whose interfaces differ would have one interface's link + type applied to every frame. + :class:`~pcapkit.foundation.engines.pcapng.PCAPNG` reads the + per-interface blocks properly and does not depend on the host's + library at all, so PCAP-NG is routed there rather than read + approximately -- or wrongly -- here. + + .. _libpcap: https://pypi.org/project/libpcap/ + + """ + from pcapkit.foundation.engines.pcap import PCAP # isort:skip + + ext = self._extractor + + if ext._exlyr != 'none' or ext._exptl != 'null': + warn("'Extractor(engine=pcap_ct)' does not support protocol and layer threshold; " + f"'layer={ext._exlyr}' and 'protocol={ext._exptl}' ignored", + AttributeWarning, stacklevel=stacklevel()) + + if ext.magic_number not in PCAP.MAGIC_NUMBER: + raise FormatError(f'unsupported file format: {ext.magic_number!r}; ' + 'the pcap-ct engine reads PCAP savefiles only') + + if not os.path.isfile(ext._ifnm): + raise UnsupportedCall(f"'Extractor(engine=pcap_ct)' requires a file on disk, " + f'but {ext._ifnm!r} is not one; libpcap opens savefiles ' + 'by name and cannot read an in-memory stream') + + if ext._flag_r and (ext._ipv4 or ext._ipv6 or ext._tcp): + ext._flag_r = False + ext._reasm = ReassemblyManager(ipv4=None, ipv6=None, tcp=None) + warn("'Extractor(engine=pcap_ct)' object does not support reassembly; " + f"so 'ipv4={ext._ipv4}', 'ipv6={ext._ipv6}' and 'tcp={ext._tcp}' will be ignored", + AttributeWarning, stacklevel=stacklevel()) + + if ext._flag_t and ext._tcp: + ext._flag_t = False + ext._trace = TraceFlowManager(tcp=None) + warn("'Extractor(engine=pcap_ct)' object does not support flow tracing; " + f"so 'tcp={ext._tcp}' will be ignored", AttributeWarning, stacklevel=stacklevel()) + + # NOTE: ``promisc=False`` is defensive only -- the offline branch of + # ``pcap.pcap`` ignores it, but ``pcap-ct`` falls through to opening the + # name as a live device when ``pcap_open_offline`` fails, and we do not + # want that attempt to request promiscuous mode. (That fallback is also + # why a bad path surfaces as ``OSError: ... No such device exists``, + # which is what the ``os.path.isfile`` check above pre-empts.) + self._handle = cast('Handle', self._expkg.pcap(name=ext._ifnm, promisc=False)) + self._dlink = Enum_LinkType.get(self._handle.datalink()) + + # setup verbose handler + if ext._flag_v: + from pcapkit.toolkit.pcap_ct import packet2chain # isort:skip + ext._vfunc = lambda e, f: print( + f'Frame {e._frnum:>3d}: {packet2chain(f[1], data_link=self._dlink)}' # pylint: disable=protected-access + ) # pylint: disable=logging-fstring-interpolation + + # extract & analyse file + self._extmp = iter(self._handle) + + def read_frame(self) -> 'RawFrame': + """Read frames with pcap-ct engine. + + Returns: + The ``(timestamp, bytes)`` pair as yielded by :class:`pcap.pcap`. + + Note: + The timestamp is seconds since the epoch as a :class:`float`. + `pcap-ct`_ opens savefiles with + :c:func:`pcap_open_offline_with_tstamp_precision` asking for + nanosecond precision, so a microsecond-resolution savefile is scaled + up by :manpage:`libpcap(3)` rather than truncated; the resulting + value matches the record header's ``ts_sec + ts_usec * 1e-6`` + exactly for :file:`examples/captures/in.pcap`. + + See Also: + Please refer to :meth:`PCAP.read_frame ` + for more operational information. + + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + + """ + from pcapkit.toolkit.pcap_ct import packet2dict # isort:skip + ext = self._extractor + + # fetch pcap-ct packet + frame = cast('RawFrame', next(self._extmp)) + timestamp, packet = frame + + # verbose output + ext._frnum += 1 + ext._vfunc(ext, frame) + + # write plist + frnum = f'Frame {ext._frnum}' + if not ext._flag_q: + info = packet2dict(packet, timestamp, data_link=self._dlink) + if ext._flag_f: + ofile = ext._ofile(f'{ext._ofnm}/{frnum}.{ext._fext}') + ofile(info, name=frnum) + else: + ext._ofile(info, name=frnum) + ofile = ext._ofile + ext._offmt = ofile.kind + + # NOTE: reassembly and flow tracing are disabled in ``run``, so there is + # deliberately no bookkeeping for either here. + + # record frames + if ext._flag_d: + ext._frame.append(frame) + + # return frame record + return frame + + def close(self) -> 'None': + """Close engine. + + This method closes the underlying :class:`pcap.pcap` handle. It is + idempotent, as :meth:`Extractor._cleanup + ` and + :meth:`Extractor.__exit__ ` + may both reach it. + + Note: + ``pcap-ct`` 1.3.0b3's own :meth:`pcap.pcap.close` happens to tolerate + a second call, where upstream ``pypcap`` 1.3.0 segfaults on one. The + :attr:`_closed ` flag is kept regardless: it is not + this engine's business to depend on a pre-release's undocumented + forgiveness, and the flag also covers the case of an engine that was + never opened. + + """ + if self._closed or self._handle is None: + return + self._closed = True + self._handle.close() diff --git a/pcapkit/foundation/engines/pypcap.py b/pcapkit/foundation/engines/pypcap.py index 664824f212..498f070814 100644 --- a/pcapkit/foundation/engines/pypcap.py +++ b/pcapkit/foundation/engines/pypcap.py @@ -7,23 +7,31 @@ This module contains the implementation for `PyPCAP`_ engine support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. +.. seealso:: + + :mod:`pcapkit.foundation.engines.pcap_ct` is the engine for `pcap-ct`_, an + independent reimplementation of the same interface that installs on Python + 3.12 and newer, where upstream `PyPCAP`_ does not. + .. _PyPCAP: https://github.com/pynetwork/pypcap +.. _pcap-ct: https://pypi.org/project/pcap-ct/ """ import os from typing import TYPE_CHECKING, cast from pcapkit.const.reg.linktype import LinkType as Enum_LinkType +from pcapkit.foundation.engines import _pcap_backend from pcapkit.foundation.engines.engine import EngineBase as Engine from pcapkit.foundation.reassembly import ReassemblyManager from pcapkit.foundation.traceflow import TraceFlowManager from pcapkit.utilities.exceptions import FormatError, UnsupportedCall, stacklevel -from pcapkit.utilities.warnings import AttributeWarning, warn +from pcapkit.utilities.warnings import AttributeWarning, EngineWarning, warn __all__ = ['PyPCAP'] if TYPE_CHECKING: - from typing import Iterator + from typing import Iterator, Optional from pcap import pcap as Handle @@ -53,11 +61,29 @@ class PyPCAP(Engine['RawFrame']): :c:func:`pcap_open_offline` opens by *name* -- there is no way to hand :manpage:`libpcap(3)` an already-open Python stream. + .. important:: + + This engine is upstream `PyPCAP`_ specifically, which cannot be installed + on Python 3.12 or newer: it ships no wheels and its ``pcap.c`` was + pre-generated by Cython 0.29.32, which does not compile against the + 3.12+ C API. `pcap-ct`_ is an independent reimplementation of the same + interface that does install there, and + :class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` is the engine for it. + + Because `pcap-ct`_ installs the same top-level :mod:`pcap` module, this + engine checks which distribution it got and refuses the other one rather + than running under the wrong name -- see :meth:`__init__`. + .. _PyPCAP: https://github.com/pynetwork/pypcap + .. _pcap-ct: https://pypi.org/project/pcap-ct/ Args: extractor: :class:`~pcapkit.foundation.extraction.Extractor` instance. + Raises: + UnsupportedCall: If the installed :mod:`pcap` is `pcap-ct`_ rather than + upstream `PyPCAP`_. + """ if TYPE_CHECKING: import pcap @@ -70,6 +96,8 @@ class PyPCAP(Engine['RawFrame']): _dlink: 'Enum_LinkType' #: Closed flag, so that the handle is not closed twice. _closed: 'bool' + #: What the :mod:`pcap` import actually found, c.f. :attr:`backend`. + _backend: '_pcap_backend.Probe' ########################################################################## # Defaults. @@ -78,9 +106,64 @@ class PyPCAP(Engine['RawFrame']): #: Engine name. __engine_name__ = 'PyPCAP' - #: Engine module name. + #: Engine module name. Note that this cannot separate the two distributions + #: that own it -- see :func:`pcapkit.foundation.engines._pcap_backend.probe` + #: and :meth:`unsupported_reason`, which is where that is done. __engine_module__ = 'pcap' + #: Distribution this engine drives. + __engine_distribution__ = _pcap_backend.PYPCAP + + ########################################################################## + # Class methods. + ########################################################################## + + @classmethod + def _reason_for(cls, found: '_pcap_backend.Probe') -> 'Optional[str]': + """The verdict on an already-taken probe. + + Split out from :meth:`unsupported_reason` so that the public hook keeps the + exact no-argument signature + :meth:`EngineBase.unsupported_reason + ` declares, + while :meth:`__init__` can pass the probe it already has. Probing twice is + not merely wasteful -- see + :func:`~pcapkit.foundation.engines._pcap_backend._purge` for why a repeated + import after a failed one does not reproduce the same error. + + """ + return _pcap_backend.wrong_backend_reason(cls.__engine_distribution__, found) + + @classmethod + def unsupported_reason(cls) -> 'Optional[str]': + """Why this engine cannot run here, or :data:`None` when it can. + + Consulted by :meth:`pcapkit.foundation.extraction.Extractor.run` *before* + the import test, because the import test cannot answer this question: + ``import pcap`` succeeds when `pcap-ct`_ is installed and upstream + `PyPCAP`_ is not, so the guard is satisfied and this engine would run on a + distribution it does not drive -- reporting ``PyPCAP`` for work that + ``pcap-ct`` did. + + The condition is deliberately *not* a Python version ceiling, even though + upstream cannot be installed on 3.12 or newer. What matters is which + distribution is actually present: a version check would refuse a + hypothetical upstream build that someone got working on a newer + interpreter, and would say nothing useful on 3.10 and 3.11, where both + distributions install happily and either could be the one in place. + + Returns: + A phrase naming the real cause, or :data:`None`. An absent :mod:`pcap` + returns :data:`None`, since the import test reports that case in its + own words and duplicating it would produce two warnings for one + problem. + + .. _PyPCAP: https://github.com/pynetwork/pypcap + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + + """ + return cls._reason_for(_pcap_backend.probe()) + ########################################################################## # Properties. ########################################################################## @@ -90,12 +173,68 @@ def dlink(self) -> 'Enum_LinkType': """Data link layer protocol, as reported by the capture handle.""" return self._dlink + @property + def backend(self) -> 'str': + """The distribution, version and location this engine is driving. + + Two distributions provide :mod:`pcap`, so "the PyPCAP engine failed" is + not a complete statement of what ran. This says which one it was, e.g. + ``pypcap 1.3.0 (/.../pcap.cpython-310-x86_64-linux-gnu.so)``. + + """ + return self._backend.describe() + ########################################################################## # Data models. ########################################################################## def __init__(self, extractor: 'Extractor') -> 'None': - import pcap # isort:skip + """Initialise the engine. + + Warns: + EngineWarning: If both distributions that provide :mod:`pcap` are + installed. Only one of them can win the import, so the other is + shadowed and unreachable -- a state worth naming, because + ``engine=`` then selects an engine that cannot run and nothing + else would say why. + + Raises: + UnsupportedCall: If the installed :mod:`pcap` is `pcap-ct`_ rather + than upstream `PyPCAP`_. This is the same condition + :meth:`unsupported_reason` reports, kept here as well because that + hook is only consulted by + :meth:`Extractor.run `; + constructing the engine directly bypasses it, and running + `pcap-ct`_ under the ``PyPCAP`` name would misreport what did the + work. Via ``Extractor`` the hook fires first and degrades to the + default engine with a warning, so this is a backstop rather than + the usual path. + + .. _PyPCAP: https://github.com/pynetwork/pypcap + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + + """ + # NOTE: the probe comes *before* ``import pcap`` so that the two engines + # behave the same way here. It matters more for + # :class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT`, whose import can + # raise ``OSError`` rather than ``ImportError``, but a shared order is one + # fewer difference between them. + self._backend = _pcap_backend.probe() + + collision = _pcap_backend.collision_reason(self._backend) + if collision is not None: + warn(f"'Extractor(engine=pypcap)': {collision}", + EngineWarning, stacklevel=stacklevel()) + + # Hand the probe over rather than letting it be taken again -- see + # ``_pcap_backend._purge`` for why a repeated import is not free. + reason = self._reason_for(self._backend) + if reason is not None: + raise UnsupportedCall( + f"'Extractor(engine=pypcap)' requires upstream 'pypcap': {reason}" + ) + + import pcap # isort:skip # safe: ``probe`` already imported it cleanly self._expkg = pcap self._extmp = cast('Iterator[RawFrame]', None) diff --git a/pcapkit/foundation/engines/pyshark.py b/pcapkit/foundation/engines/pyshark.py index 89e0506c8d..99c81d106d 100644 --- a/pcapkit/foundation/engines/pyshark.py +++ b/pcapkit/foundation/engines/pyshark.py @@ -10,6 +10,7 @@ .. _PyShark: https://kiminewt.github.io/pyshark """ +import sys from typing import TYPE_CHECKING, cast from pcapkit.foundation.engines.engine import EngineBase as Engine @@ -21,6 +22,8 @@ __all__ = ['PyShark'] if TYPE_CHECKING: + from typing import Optional + from pyshark.capture.file_capture import FileCapture from pyshark.packet.packet import Packet as PySharkPacket @@ -56,6 +59,101 @@ class PyShark(Engine['PySharkPacket']): #: Engine module name. __engine_module__ = 'pyshark' + #: First Python version `PyShark`_ does not work on, as a ``(major, minor)`` + #: pair. Released 0.6 builds its event loop with + #: ``asyncio.get_event_loop_policy().get_event_loop()``, and Python 3.14 made + #: :func:`asyncio.get_event_loop` raise :exc:`RuntimeError` when no current + #: event loop exists instead of quietly creating one. + PYTHON_CEILING = (3, 14) + + ########################################################################## + # Class methods. + ########################################################################## + + @classmethod + def unsupported_reason(cls) -> 'Optional[str]': + """Why this engine cannot run here, or :data:`None` when it can. + + Consulted by :meth:`pcapkit.foundation.extraction.Extractor.run` *before* + the import test, because neither of the two things that stop this engine + is visible to an import. `PyShark`_ imports perfectly well and then fails + when it is used, which without this hook escapes from :meth:`run` as a hard + error rather than degrading to the default engine with a warning. + + **The interpreter.** ``pyshark`` 0.6 does + ``asyncio.get_event_loop_policy().get_event_loop()`` at + :file:`pyshark/capture/capture.py:183`, in a fresh interpreter with no + running loop. Measured on four interpreters: 3.10 and 3.11 return a loop + silently, 3.12 returns one with a :exc:`DeprecationWarning`, and 3.14 + raises ``RuntimeError: There is no current event loop in thread + 'MainThread'``. Hence :attr:`PYTHON_CEILING` is ``(3, 14)``. Python 3.13 was + not available on the machine this was measured on; it is expected to work, + since it is on the deprecated-but-functional side of that progression, and + that expectation is the one thing here that is inferred rather than + observed. + + **The** :program:`tshark` **binary.** ``pyshark`` is a wrapper around + Wireshark's command-line tool and does no parsing itself, so it is useless + without it. The check delegates to ``pyshark``'s own + ``get_process_path()`` rather than calling :func:`shutil.which`, because + the two are not equivalent and ``which`` would refuse setups that work: + ``pyshark`` looks at ``tshark_path`` in its :file:`config.ini` *first*, and + then at :envvar:`PATH` on POSIX, at both Program Files directories on + Windows, and at :file:`/Applications/Wireshark.app` on macOS. Asking + ``pyshark`` gets all of that for free and cannot disagree with what + ``pyshark`` will do a moment later. + + Note: + Deliberately **not cached**, and the cost was measured rather than + assumed: the failing path -- which is the expensive one, since it + exhausts every candidate -- takes about 290 microseconds with 39 + :envvar:`PATH` entries, against about 120 for a bare + :func:`shutil.which`. This runs once per + :class:`~pcapkit.foundation.extraction.Extractor`, not once per frame, + so it is far below the cost of opening the capture. Caching would trade + that for an answer about the *environment* that cannot change within + the process -- so installing Wireshark, or fixing + :envvar:`PATH`, would not take effect until restart. The Windows path + does more work than the POSIX one (two Program Files directories, and + :func:`shutil.which` there would multiply by ``PATHEXT``), but it is + still a bounded handful of :func:`os.stat` calls. + + Returns: + A short phrase naming the limitation, or :data:`None`. + + .. _PyShark: https://kiminewt.github.io/pyshark + + """ + if sys.version_info[:2] >= cls.PYTHON_CEILING: + return (f'pyshark does not support Python ' + f'{sys.version_info[0]}.{sys.version_info[1]}; it builds its event ' + 'loop with `asyncio.get_event_loop_policy().get_event_loop()`, which ' + 'raises RuntimeError ' + 'from Python 3.14 when no current event loop exists') + + try: + from pyshark.tshark.tshark import get_process_path # isort:skip + except ImportError: + # Not installed, which is ``Extractor.import_test``'s business -- it + # reports that case in its own words, and answering here as well would + # produce two warnings for one problem. An ImportError from a *renamed* + # upstream helper lands here too, and the engine then simply proceeds + # as it did before this check existed. + return None + + try: + get_process_path() + except Exception as exc: # pylint: disable=broad-except + # ``TSharkNotFoundException`` by name, but caught broadly: the whole + # point is to turn any failure to locate the binary into a reason + # rather than let it escape, and upstream is free to raise something + # else. Its own message lists every path it searched, which is exactly + # what the user needs, so it is quoted rather than summarised. + return (f'pyshark requires Wireshark\'s `tshark` binary, which pyshark ' + f'could not find -- {exc}') + + return None + ########################################################################## # Data models. ########################################################################## diff --git a/pcapkit/foundation/extraction.py b/pcapkit/foundation/extraction.py index 544ab1a254..f31a928ef0 100644 --- a/pcapkit/foundation/extraction.py +++ b/pcapkit/foundation/extraction.py @@ -70,11 +70,14 @@ # NOTE: this alias is duplicated verbatim in ``pcapkit.interface.misc``; both # copies need updating when a new engine lands. The duplication predates the # engines added here and is left as-is on purpose. - Engines = Literal['default', 'pcapkit', 'dpkt', 'scapy', 'pyshark', 'pypcap', 'pypcapfile'] + Engines = Literal['default', 'pcapkit', 'dpkt', 'scapy', 'pyshark', 'pypcap', 'pcap_ct', + 'pypcapfile'] Layers = Literal['link', 'internet', 'transport', 'application', 'none'] - # NOTE: the PyPCAP engine performs no dissection, so its "packet" is the - # ``(timestamp, bytes)`` pair that ``pcap.pcap`` yields. + # NOTE: the PyPCAP and PCAP_CT engines perform no dissection, so their + # "packet" is the ``(timestamp, bytes)`` pair that ``pcap.pcap`` yields. + # One member covers both: they read the same interface, from two independent + # distributions of it. Packet = Union[Frame, PCAPNG, ScapyPacket, DPKTPacket, PySharkPacket, PCAPFilePacket, tuple[float, bytes]] @@ -212,6 +215,13 @@ class Extractor(Generic[_P]): 'dpkt': ModuleDescriptor('pcapkit.foundation.engines.dpkt', 'DPKT'), 'pyshark': ModuleDescriptor('pcapkit.foundation.engines.pyshark', 'PyShark'), 'pypcap': ModuleDescriptor('pcapkit.foundation.engines.pypcap', 'PyPCAP'), + # NOTE: ``pcap-ct`` is a separate distribution that reimplements the + # ``pypcap`` interface, and it installs the same top-level ``pcap`` + # module. It gets its own entry rather than sharing ``pypcap``'s because + # the two are independent projects with different install requirements + # and different Python version coverage; ``PCAP_CT.__engine_module__`` + # explains how ``import_test`` tells them apart. + 'pcap_ct': ModuleDescriptor('pcapkit.foundation.engines.pcap_ct', 'PCAP_CT'), 'pypcapfile': ModuleDescriptor('pcapkit.foundation.engines.pypcapfile', 'PyPCAPFile'), } # type: dict[str, ModuleDescriptor[Engine] | Type[Engine]] @@ -442,6 +452,7 @@ def run(self) -> 'None': # pylint: disable=inconsistent-return-statements * Scapy driver: :class:`pcapkit.foundation.engines.scapy.Scapy` * PyShark driver: :class:`pcapkit.foundation.engines.pyshark.PyShark` * PyPCAP driver: :class:`pcapkit.foundation.engines.pypcap.PyPCAP` + * pcap-ct driver: :class:`pcapkit.foundation.engines.pcap_ct.PCAP_CT` * PyPCAPFile driver: :class:`pcapkit.foundation.engines.pypcapfile.PyPCAPFile` Warns: diff --git a/pcapkit/interface/misc.py b/pcapkit/interface/misc.py index 8f3281253f..ce63e96d26 100644 --- a/pcapkit/interface/misc.py +++ b/pcapkit/interface/misc.py @@ -47,7 +47,8 @@ Formats = Literal['pcap', 'cap', 'json', 'tree', 'text', 'txt', 'plist', 'xml'] # NOTE: this alias duplicates the one in ``pcapkit.foundation.extraction``; # both copies need updating when a new engine lands. - Engines = Literal['default', 'pcapkit', 'dpkt', 'scapy', 'pyshark', 'pypcap', 'pypcapfile'] + Engines = Literal['default', 'pcapkit', 'dpkt', 'scapy', 'pyshark', 'pypcap', 'pcap_ct', + 'pypcapfile'] __all__ = ['follow_tcp_stream'] @@ -92,11 +93,11 @@ def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, List of extracted TCP streams. """ - # NOTE: both of these engines disable TCP flow tracing outright -- PyShark - # because ``pcapkit`` has no reassembly adapter for it, PyPCAP because it - # performs no protocol dissection at all -- so ``extraction.trace`` below - # would raise instead of yielding streams. - if engine is not None and engine.lower() in ('pyshark', 'pypcap'): + # NOTE: all of these engines disable TCP flow tracing outright -- PyShark + # because ``pcapkit`` has no reassembly adapter for it, PyPCAP and PCAP_CT + # because they perform no protocol dissection at all -- so + # ``extraction.trace`` below would raise instead of yielding streams. + if engine is not None and engine.lower() in ('pyshark', 'pypcap', 'pcap_ct'): warn(f'unsupported extraction engine: {engine}; fallback to default engine', EngineWarning, stacklevel=stacklevel()) engine = None diff --git a/pcapkit/toolkit/__init__.py b/pcapkit/toolkit/__init__.py index d52d5542f1..5869b0914c 100644 --- a/pcapkit/toolkit/__init__.py +++ b/pcapkit/toolkit/__init__.py @@ -42,6 +42,10 @@ # from pcapkit.toolkit.pypcap import packet2chain as pypcap_packet2chain # from pcapkit.toolkit.pypcap import packet2dict as pypcap_packet2dict +# # tools for pcap-ct engine +# from pcapkit.toolkit.pcap_ct import packet2chain as pcap_ct_packet2chain +# from pcapkit.toolkit.pcap_ct import packet2dict as pcap_ct_packet2dict + # # tools for PyPCAPFile engine # from pcapkit.toolkit.pypcapfile import packet2timestamp as pypcapfile_packet2timestamp # from pcapkit.toolkit.pypcapfile import ipv4_header as pypcapfile_ipv4_header @@ -71,6 +75,9 @@ # # PyPCAP engine # 'pypcap_packet2chain', 'pypcap_packet2dict', + # # pcap-ct engine + # 'pcap_ct_packet2chain', 'pcap_ct_packet2dict', + # # PyPCAPFile engine # 'pypcapfile_packet2timestamp', 'pypcapfile_ipv4_header', # 'pypcapfile_packet2chain', 'pypcapfile_packet2dict', diff --git a/pcapkit/toolkit/pcap_ct.py b/pcapkit/toolkit/pcap_ct.py new file mode 100644 index 0000000000..23ac454a2e --- /dev/null +++ b/pcapkit/toolkit/pcap_ct.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +"""pcap-ct Tools +================ + +.. module:: pcapkit.toolkit.pcap_ct + +:mod:`pcapkit.toolkit.pcap_ct` contains all you need for +:mod:`pcapkit` handy usage with `pcap-ct`_ engine. All reforming +functions returns with a flag to indicate if usable for +its caller. + +.. _pcap-ct: https://pypi.org/project/pcap-ct/ + +.. important:: + + `pcap-ct`_ is a :mod:`ctypes` reimplementation of the `PyPCAP`_ interface over + :manpage:`libpcap(3)`: it hands back the ``(timestamp, bytes)`` pair that + :c:func:`pcap_next_ex` produced and performs **no protocol dissection + whatsoever**. There is therefore no IP or TCP layer for this module to read, + and the reassembly and flow tracing adapters that :mod:`pcapkit.toolkit.dpkt` + and :mod:`pcapkit.toolkit.scapy` provide cannot be implemented here. + + They are still defined below, but only so that reaching for one fails loudly + with :exc:`~pcapkit.utilities.exceptions.UnsupportedCall` rather than with an + :exc:`ImportError` that says nothing about why. Dissecting the raw bytes with + :mod:`pcapkit`'s own parsers would defeat the point of selecting a third-party + engine, so it is deliberately not done. + +.. seealso:: + + :mod:`pcapkit.toolkit.pypcap` is the same adapter for upstream `PyPCAP`_. The + two are kept apart because the engines are: the distributions are independent + projects that happen to share the :mod:`pcap` module name, and each engine + names its own adapter so that a change made for one cannot quietly alter the + other. + +.. _PyPCAP: https://github.com/pynetwork/pypcap + +""" +from typing import TYPE_CHECKING + +from pcapkit.utilities.exceptions import UnsupportedCall + +if TYPE_CHECKING: + from ipaddress import IPv4Address, IPv6Address + from typing import Any + + from pcapkit.const.reg.linktype import LinkType as Enum_LinkType + from pcapkit.foundation.reassembly.data.ip import Packet as IP_Packet + from pcapkit.foundation.reassembly.data.tcp import Packet as TCP_Packet + from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet + +__all__ = [ + 'packet2chain', 'packet2dict', + 'ipv4_reassembly', 'ipv6_reassembly', 'tcp_reassembly', 'tcp_traceflow', +] + +#: Explanatory suffix shared by every unsupported adapter below. +_NO_DISSECTION = ("'pcap-ct' is a libpcap binding and performs no protocol " + "dissection, so there is no protocol layer to read") + + +def packet2chain(packet: 'bytes', *, data_link: 'Enum_LinkType') -> 'str': + """Fetch pcap-ct packet protocol chain. + + Args: + packet: Raw packet bytes, as returned by :class:`pcap.pcap` iteration. + data_link: Data link type, from the capture handle. + + Returns: + Colon (``:``) separated list of protocol chain. + + Note: + As `pcap-ct`_ does not dissect the packet, the chain is only ever the + link layer type followed by ``Raw``, e.g. ``ETHERNET:Raw``. + + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + + """ + return f'{data_link.name}:Raw' + + +def packet2dict(packet: 'bytes', timestamp: 'float', *, + data_link: 'Enum_LinkType') -> 'dict[str, Any]': + """Convert pcap-ct packet into :obj:`dict`. + + Args: + packet: Raw packet bytes, as returned by :class:`pcap.pcap` iteration. + timestamp: Timestamp of packet, as returned by :class:`pcap.pcap` iteration. + data_link: Data link type, from the capture handle. + + Returns: + Dict[str, Any]: A :obj:`dict` mapping of packet data. + + Note: + The mapping carries the captured bytes verbatim rather than a decoded + protocol tree, since `pcap-ct`_ does not decode anything. + + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + + """ + return { + 'timestamp': timestamp, + 'packet': packet, + data_link.name: { + 'raw_len': len(packet), + 'raw': packet, + }, + } + + +def ipv4_reassembly(packet: 'bytes', *, count: 'int' = -1) -> 'IP_Packet[IPv4Address] | None': + """Make data for IPv4 reassembly. + + Args: + packet: Raw packet bytes. + count: Packet index. If not provided, default to ``-1``. + + Raises: + UnsupportedCall: Always, as `pcap-ct`_ provides no IPv4 layer to read. + + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + + """ + raise UnsupportedCall(f'IPv4 reassembly is not supported by the pcap-ct engine: {_NO_DISSECTION}') + + +def ipv6_reassembly(packet: 'bytes', *, count: 'int' = -1) -> 'IP_Packet[IPv6Address] | None': + """Make data for IPv6 reassembly. + + Args: + packet: Raw packet bytes. + count: Packet index. If not provided, default to ``-1``. + + Raises: + UnsupportedCall: Always, as `pcap-ct`_ provides no IPv6 layer to read. + + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + + """ + raise UnsupportedCall(f'IPv6 reassembly is not supported by the pcap-ct engine: {_NO_DISSECTION}') + + +def tcp_reassembly(packet: 'bytes', *, count: 'int' = -1) -> 'TCP_Packet | None': + """Make data for TCP reassembly. + + Args: + packet: Raw packet bytes. + count: Packet index. If not provided, default to ``-1``. + + Raises: + UnsupportedCall: Always, as `pcap-ct`_ provides no TCP layer to read. + + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + + """ + raise UnsupportedCall(f'TCP reassembly is not supported by the pcap-ct engine: {_NO_DISSECTION}') + + +def tcp_traceflow(packet: 'bytes', timestamp: 'float', *, data_link: 'Enum_LinkType', + count: 'int' = -1) -> 'TF_TCP_Packet | None': + """Trace packet flow for TCP. + + Args: + packet: Raw packet bytes. + timestamp: Timestamp of the packet. + data_link: Data link layer protocol (from the capture handle). + count: Packet index. If not provided, default to ``-1``. + + Raises: + UnsupportedCall: Always, as `pcap-ct`_ provides no TCP layer to read. + + .. _pcap-ct: https://pypi.org/project/pcap-ct/ + + """ + raise UnsupportedCall(f'TCP flow tracing is not supported by the pcap-ct engine: {_NO_DISSECTION}') diff --git a/pcapkit/toolkit/pypcap.py b/pcapkit/toolkit/pypcap.py index 681f26d031..e81673806f 100644 --- a/pcapkit/toolkit/pypcap.py +++ b/pcapkit/toolkit/pypcap.py @@ -26,6 +26,15 @@ :mod:`pcapkit`'s own parsers would defeat the point of selecting a third-party engine, so it is deliberately not done. +.. seealso:: + + :mod:`pcapkit.toolkit.pcap_ct` is the same adapter for `pcap-ct`_, the + independent reimplementation of this interface that + :class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` drives. Each engine names + its own adapter, so a change made for one cannot quietly alter the other. + +.. _pcap-ct: https://pypi.org/project/pcap-ct/ + """ from typing import TYPE_CHECKING diff --git a/pyproject.toml b/pyproject.toml index 593d50a5c5..c8779cf2f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,33 @@ maintainers = [ { name="Jarry Shaw" }, ] license = { text="BSD 3-Clause License" } +# Kept at 3.6, which the ``bpc-*`` requirements above are what make possible: the +# sources use 3.8 syntax (walrus, positional-only parameters) and ``refactor()`` in +# setup.py converts it at install time. Measured, so that the state of that is on +# the record rather than assumed: +# +# 3.9.25, 3.8.20 import and extract straight from source, no conversion needed +# 3.7.16 SyntaxError from source, as expected -- this is what the +# ``bpc-*`` conversion exists for +# +# **The conversion path has an upstream bug today.** ``bpc-poseur`` 0.4.3.post1 +# crashes on any positional-only parameter that is on a *method* rather than a +# plain function -- ``AttributeError: 'Name' object has no attribute 'name'`` at +# ``poseur.py:744`` in ``_process_classdef``, which does ``cls_ctx=name.name`` +# where ``name`` is already a parso ``Name`` leaf and wants ``.value``. It then +# **exits 0**, so ``refactor()``'s ``check_call`` cannot see the failure, and the +# 12 positional-only parameters in ``pcapkit/corekit/io.py`` survive into the +# installed package -- making ``import pcapkit`` fail with ``SyntaxError`` on an +# interpreter below 3.8. +# +# Verified that this is the *only* blocker: with that one line changed to +# ``name.value``, ``walrus`` followed by ``poseur`` converts ``io.py`` into +# something Python 3.7 parses cleanly. So this is an upstream fix to follow, not a +# reason to withdraw the declaration. Until it lands, sub-3.8 support is intent +# rather than something that works. +# +# Independently: CI covers **3.10 through 3.14** (3.15 runs allowed-to-fail), so 3.8 and 3.9 are best-effort +# (both are end-of-life), and the ``PCAP_CT`` extra needs 3.10 regardless. requires-python = ">=3.6, <4" description = "PyPCAPKit: comprehensive network packet analysis library" keywords = [ "network", "pcap", "packet" ] @@ -102,6 +129,48 @@ PyShark = [ "pyshark" ] # error instead of a resolver message; upstream sets no ``requires_python`` of # its own, and is unmaintained. PyPCAP = [ "pypcap; python_version < '3.12'" ] +# ``pcap-ct`` is what covers 3.12+, and it is a *separate extra* rather than a +# complementary marker on ``PyPCAP`` above. Two reasons, and the first is +# decisive: +# +# * They are two engines, not one engine with two backends -- ``engine='pypcap'`` +# selects ``PyPCAP`` and ``engine='pcap_ct'`` selects ``PCAP_CT``, and +# ``PyPCAP`` refuses to run on pcap-ct rather than mislabel itself. An extra +# whose contents changed with the interpreter would therefore change which +# ``engine=`` string works, under one unchanged extra name. +# * These are pre-releases. Folding them into ``PyPCAP`` would install a beta +# on a newer interpreter for someone who asked for the stable thing, without +# ever saying so. +# +# No compiler and no ``pcap.h`` are needed: both publish ``py3-none-any`` wheels. +# A system libpcap *is* still needed at run time, though -- the ``libpcap`` +# distribution ships a vendored ``libpcap.so`` but its published ``libpcap.cfg`` +# says ``LIBPCAP = None``, which sends its loader to ``find_library("pcap")``. +# That is a runtime prerequisite rather than a resolution problem, so it is not +# expressible here; the engine detects it and degrades to the default engine +# instead of dying, c.f. ``PCAP_CT.unsupported_reason``. +# +# The pins name a pre-release on purpose. Neither project has ever published a +# stable release, so pip's PEP 440 fallback ("if all versions are +# pre-releases, allow them") means a bare ``pcap-ct`` already resolves without +# ``--pre``; naming the version keeps that from being an accident, floors the +# requirement at the versions actually tested here, and still accepts a future +# stable release, since PEP 440 orders 1.3.0 above 1.3.0b3. ``pcap-ct`` itself +# depends on ``libpcap>=1.11.0b16``; ``libpcap`` is listed anyway because it is +# what supplies the ctypes bindings this engine runs on, and that should not be +# implicit in a transitive dependency. +# +# The ``>= '3.10'`` marker is the same courtesy as the ceilings above, pointing +# the other way. Taken from the wheel metadata rather than guessed: ``pcap-ct`` +# 1.3.0b3 declares ``Requires-Python: <4.0.0,>=3.9.0`` and ``libpcap`` +# 1.11.0b29 declares ``<4.0.0,>=3.10.0``, so 3.10 is the binding floor. This +# project's own floor is now ``>=3.10`` too, so the marker is belt-and-braces +# rather than load-bearing -- it keeps the requirement honest on its own terms, and +# would matter again if the project floor ever moved back down. +PCAP_CT = [ + "pcap-ct>=1.3.0b3; python_version >= '3.10'", + "libpcap>=1.11.0b29; python_version >= '3.10'", +] # pypcapfile 0.12.0's ``linklayer`` module imports :mod:`imp`, removed in Python # 3.12, and ``savefile`` imports ``linklayer`` -- so the package installs cleanly # and then raises ``ModuleNotFoundError`` the moment the engine is used. Upstream @@ -114,6 +183,11 @@ vendor = [ "requests[socks]", "beautifulsoup4[html5lib]" ] # development files on every platform, and fails where only the header is # present -- as on the macOS runners. Install it explicitly with # ``pip install pypcapkit[PyPCAP]`` once libpcap is available. +# +# ``pcap-ct`` is left out too, for a different reason: it needs no toolchain at +# all, but it and ``libpcap`` are published only as pre-releases, and ``all`` +# should not be the way somebody ends up with a beta they did not ask for. It is +# ``pip install pypcapkit[PCAP_CT]``, which says what it is installing. all = [ "emoji", "cryptography>=3.4", diff --git a/tests/foundation/engines/test_new_engine_parity.py b/tests/foundation/engines/test_new_engine_parity_runtime.py similarity index 90% rename from tests/foundation/engines/test_new_engine_parity.py rename to tests/foundation/engines/test_new_engine_parity_runtime.py index 561a8e6bbf..f2cf23979e 100644 --- a/tests/foundation/engines/test_new_engine_parity.py +++ b/tests/foundation/engines/test_new_engine_parity_runtime.py @@ -8,6 +8,16 @@ the absence of any :class:`~pcapkit.utilities.warnings.EngineWarning` -- and only then compares. +This module is named ``*_runtime.py`` deliberately, which puts it in the +fixture-dependent tier (see :mod:`tests._tiers`). It reads ``arp.pcap``, +``tcp.pcap``, ``ipv4.pcap`` and ``test.pcapng``, none of which git tracks -- +:file:`examples/generators/make_samples.py` writes them -- so it cannot be +unit-tier, and it was one only because the guard added in GitHub pull request #393 +could not see it: the reads go through ``sample_path(capture)`` with a *variable*, +which only the runtime half of the guard catches, and that half was never reached +on a machine where the engine packages were absent and every test skipped. +Installing an engine made all four reads fail at once. + Note also that the frame objects a stored extraction hands back are not a usable source of raw bytes: :attr:`Frame.packet ` re-reads from the (by then @@ -46,7 +56,17 @@ def _importable(*modules: str) -> bool: return True -HAS_PYPCAP = _importable('pcap') +#: Whether upstream ``pypcap`` is installed, as opposed to ``pcap-ct``. +#: +#: Both distributions own the top-level :mod:`pcap` name, so ``_importable('pcap')`` +#: alone is not the question these tests want to ask: ``pcap-ct`` is driven by +#: :class:`~pcapkit.foundation.engines.pcap_ct.PCAP_CT` and +#: :class:`~pcapkit.foundation.engines.pypcap.PyPCAP` refuses it outright, so +#: letting it satisfy this gate would run the ``pypcap`` tests against an engine +#: that declines to start. ``pcap-ct`` ships :mod:`pcap` as a package whose +#: ``__init__`` does ``from ._pcap import *``; upstream ships a single extension +#: module, which has no such submodule. +HAS_PYPCAP = _importable('pcap') and not _importable('pcap._pcap') HAS_PYPCAPFILE = _importable('pcapfile.savefile', 'pcapfile.linklayer') #: Captures the parity comparison runs over. All are Ethernet PCAP savefiles. diff --git a/tests/foundation/engines/test_pcap_backend.py b/tests/foundation/engines/test_pcap_backend.py new file mode 100644 index 0000000000..f64cde3a4d --- /dev/null +++ b/tests/foundation/engines/test_pcap_backend.py @@ -0,0 +1,248 @@ +"""Unit tests for :mod:`pcapkit.foundation.engines._pcap_backend`. + +Two unrelated distributions -- upstream `pypcap`_ and `pcap-ct`_ -- both install a +top-level :mod:`pcap` module, so an engine cannot know which one it got without +asking. This module tests the asking. + +Nothing here needs either distribution installed: every case is built from a +stand-in module or a patched import, which is the only way to cover the +combinations that cannot all exist in one environment at once. The combinations +*were* also exercised against real installations -- pcap-ct alone on 3.10 and +3.14, upstream pypcap alone on 3.10, both together on 3.10, neither, and pcap-ct +with no system libpcap -- and the expectations below are what those runs produced. + +.. _pypcap: https://github.com/pynetwork/pypcap +.. _pcap-ct: https://pypi.org/project/pcap-ct/ + +""" +from __future__ import annotations + +import importlib.util +import types +import unittest +from unittest import mock + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) + + +def pcap_ct_module(): + """A stand-in for `pcap-ct`_'s :mod:`pcap`, which is a package. + + Its ``__init__`` does ``from ._pcap import *``, so the submodule is bound as + an attribute -- that binding is the discriminator. + + """ + module = types.ModuleType('pcap') + module.__version__ = '1.3.0b3' # type: ignore[attr-defined] + module.__file__ = '/somewhere/site-packages/pcap/__init__.py' + module.__path__ = ['/somewhere/site-packages/pcap'] # type: ignore[attr-defined] + module._pcap = types.ModuleType('pcap._pcap') # type: ignore[attr-defined] + return module + + +def pypcap_module(): + """A stand-in for upstream `pypcap`_'s :mod:`pcap`, a single extension module.""" + module = types.ModuleType('pcap') + module.__version__ = '1.3.0' # type: ignore[attr-defined] + module.__file__ = '/somewhere/site-packages/pcap.cpython-310-x86_64-linux-gnu.so' + return module + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class IdentifyTests(unittest.TestCase): + def test_identifies_pcap_ct_by_its_submodule_attribute(self) -> None: + from pcapkit.foundation.engines import _pcap_backend + + self.assertEqual(_pcap_backend.identify(pcap_ct_module()), _pcap_backend.PCAP_CT) + + def test_identifies_upstream_pypcap_by_the_absence_of_it(self) -> None: + from pcapkit.foundation.engines import _pcap_backend + + self.assertEqual(_pcap_backend.identify(pypcap_module()), _pcap_backend.PYPCAP) + + def test_does_not_key_on_ex_name_which_both_distributions_have(self) -> None: + from pcapkit.foundation.engines import _pcap_backend + + # Measured on upstream pypcap 1.3.0: ``pcap.ex_name`` is present there + # too, so it looks like a pcap-ct marker and is not one. Guard against + # anybody "simplifying" the check to use it. + upstream = pypcap_module() + upstream.ex_name = lambda name: name # type: ignore[attr-defined] + self.assertEqual(_pcap_backend.identify(upstream), _pcap_backend.PYPCAP) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class ProbeTests(unittest.TestCase): + def probe_with(self, module=None, *, error=None, installed=()): + """Probe with the ``pcap`` import and the metadata both under control.""" + from pcapkit.foundation.engines import _pcap_backend + + if error is not None: + target = mock.patch('importlib.import_module', side_effect=error) + else: + target = mock.patch('importlib.import_module', return_value=module) + + with mock.patch.object(_pcap_backend, 'installed_distributions', + return_value=tuple(installed)): + with target: + return _pcap_backend.probe() + + def test_reports_pcap_ct_with_its_version_and_origin(self) -> None: + from pcapkit.foundation.engines import _pcap_backend + + found = self.probe_with(pcap_ct_module(), installed=('pcap-ct',)) + self.assertEqual(found.name, _pcap_backend.PCAP_CT) + self.assertEqual(found.version, '1.3.0b3') + self.assertEqual(found.origin, '/somewhere/site-packages/pcap/__init__.py') + self.assertIsNone(found.failure) + self.assertFalse(found.missing) + self.assertIn('pcap-ct 1.3.0b3', found.describe()) + + def test_reports_upstream_pypcap(self) -> None: + from pcapkit.foundation.engines import _pcap_backend + + found = self.probe_with(pypcap_module(), installed=('pypcap',)) + self.assertEqual(found.name, _pcap_backend.PYPCAP) + self.assertEqual(found.version, '1.3.0') + self.assertIn('pypcap 1.3.0', found.describe()) + + def test_an_absent_module_is_missing_rather_than_broken(self) -> None: + found = self.probe_with(error=ImportError("No module named 'pcap'")) + + self.assertIsNone(found.name) + self.assertTrue(found.missing) + self.assertIn('pcap', found.failure) + + def test_an_oserror_is_broken_rather_than_missing(self) -> None: + # The case this distinction exists for: with no system libpcap, + # ``import pcap`` raises OSError, which is *not* an ImportError, so + # ``Extractor.import_test`` does not catch it. Recording it as "installed + # but unusable" is what lets an engine report it instead of dying. + found = self.probe_with(error=OSError('Cannot find libpcap.so library'), + installed=('pcap-ct',)) + + self.assertIsNone(found.name) + self.assertFalse(found.missing) + self.assertIn('OSError', found.failure) + self.assertIn('Cannot find libpcap.so library', found.failure) + self.assertEqual(found.installed, ('pcap-ct',)) + + def test_a_failed_probe_leaves_no_residue_for_the_next_one(self) -> None: + import sys + + from pcapkit.foundation.engines import _pcap_backend + + # Both distributions' package initialisers open with + # ``from .__about__ import * ; del __about__``, which cannot be re-run: a + # part-way failure leaves the ``__about__`` submodule cached and the retry + # dies with ``NameError`` instead of the real cause. Measured, and the + # reason ``probe`` purges both module trees on failure. + sys.modules['pcap.__about__'] = types.ModuleType('pcap.__about__') + sys.modules['libpcap.__about__'] = types.ModuleType('libpcap.__about__') + self.addCleanup(lambda: [sys.modules.pop(name, None) + for name in ('pcap.__about__', 'libpcap.__about__')]) + + with mock.patch.object(_pcap_backend, 'installed_distributions', return_value=()): + with mock.patch('importlib.import_module', + side_effect=OSError('Cannot find libpcap.so library')): + _pcap_backend.probe() + + self.assertNotIn('pcap.__about__', sys.modules) + self.assertNotIn('libpcap.__about__', sys.modules) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class InstalledDistributionsTests(unittest.TestCase): + def installed(self, present): + import importlib.metadata + + from pcapkit.foundation.engines import _pcap_backend + + def distribution(name): + if name in present: + return mock.Mock(version='stub') + raise importlib.metadata.PackageNotFoundError(name) + + with mock.patch('importlib.metadata.distribution', side_effect=distribution): + return _pcap_backend.installed_distributions() + + def test_reports_only_what_is_installed_in_a_fixed_order(self) -> None: + self.assertEqual(self.installed(()), ()) + self.assertEqual(self.installed({'pcap-ct'}), ('pcap-ct',)) + self.assertEqual(self.installed({'pypcap'}), ('pypcap',)) + # order comes from DISTRIBUTIONS, not from the argument, so messages + # naming both read the same way every time + self.assertEqual(self.installed({'pcap-ct', 'pypcap'}), ('pypcap', 'pcap-ct')) + + def test_an_unreadable_dist_info_is_treated_as_absent_not_fatal(self) -> None: + from pcapkit.foundation.engines import _pcap_backend + + # Detection is a courtesy; a corrupt ``dist-info`` must not be able to + # fail an extraction. + with mock.patch('importlib.metadata.distribution', side_effect=ValueError('corrupt')): + self.assertEqual(_pcap_backend.installed_distributions(), ()) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class ReasonTests(unittest.TestCase): + def make_probe(self, **overrides): + from pcapkit.foundation.engines import _pcap_backend + + values = { + 'name': _pcap_backend.PCAP_CT, + 'version': '1.3.0b3', + 'origin': '/somewhere/pcap/__init__.py', + 'failure': None, + 'missing': False, + 'installed': ('pcap-ct',), + } + values.update(overrides) + return _pcap_backend.Probe(**values) + + def test_wrong_backend_names_the_engine_that_wants_what_is_installed(self) -> None: + from pcapkit.foundation.engines import _pcap_backend + + reason = _pcap_backend.wrong_backend_reason(_pcap_backend.PYPCAP, self.make_probe()) + self.assertIsNotNone(reason) + self.assertIn('pcap-ct', reason) + self.assertIn("engine=pcap_ct", reason) + + reason = _pcap_backend.wrong_backend_reason( + _pcap_backend.PCAP_CT, + self.make_probe(name=_pcap_backend.PYPCAP, version='1.3.0'), + ) + self.assertIsNotNone(reason) + self.assertIn('pypcap', reason) + self.assertIn("engine=pypcap", reason) + + def test_the_right_backend_and_an_absent_one_are_both_no_reason(self) -> None: + from pcapkit.foundation.engines import _pcap_backend + + self.assertIsNone( + _pcap_backend.wrong_backend_reason(_pcap_backend.PCAP_CT, self.make_probe())) + # an absent module is the import test's business, not this function's -- + # reporting it here would produce two warnings for one problem + self.assertIsNone( + _pcap_backend.wrong_backend_reason(_pcap_backend.PCAP_CT, + self.make_probe(name=None, missing=True))) + + def test_collision_is_reported_only_when_both_are_installed(self) -> None: + from pcapkit.foundation.engines import _pcap_backend + + self.assertIsNone(_pcap_backend.collision_reason(self.make_probe(installed=()))) + self.assertIsNone( + _pcap_backend.collision_reason(self.make_probe(installed=('pcap-ct',)))) + + reason = _pcap_backend.collision_reason( + self.make_probe(installed=('pypcap', 'pcap-ct'))) + self.assertIsNotNone(reason) + self.assertIn('pypcap', reason) + self.assertIn('pcap-ct', reason) + # it has to say which one won, since that is what the user cannot see + self.assertIn('/somewhere/pcap/__init__.py', reason) + self.assertIn('shadowed', reason) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/foundation/engines/test_pcap_ct_engine.py b/tests/foundation/engines/test_pcap_ct_engine.py new file mode 100644 index 0000000000..fb4ea26571 --- /dev/null +++ b/tests/foundation/engines/test_pcap_ct_engine.py @@ -0,0 +1,556 @@ +"""Unit tests for :mod:`pcapkit.foundation.engines.pcap_ct`. + +The engine is exercised with a stand-in for :class:`pcap.pcap`, so that the +routing decisions -- format gating, capability warnings, output, storage, close -- +are covered whether or not `pcap-ct`_ is installed. One test at the end runs the +real backend against a committed capture, and skips when it is absent. + +`pcap-ct`_ and upstream `pypcap`_ both install a top-level :mod:`pcap`, so the +tests here look almost exactly like :mod:`tests.foundation.engines.test_pypcap_engine`. +They are kept apart because the engines are: each names its own toolkit and its +own module marker, and a shared test could not tell which of the two it had +proved anything about. + +.. _pcap-ct: https://pypi.org/project/pcap-ct/ +.. _pypcap: https://github.com/pynetwork/pypcap + +""" +from __future__ import annotations + +import importlib +import importlib.util +import io +import os +import tempfile +import types +import unittest +from unittest import mock + +from tests._support import close_extractor, purge_modules, sample_path + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) + + +def _importable(*modules: str) -> bool: + """Test if every named module can actually be imported.""" + for module in modules: + try: + importlib.import_module(module) + except ImportError: + return False + return True + + +#: Whether the `pcap-ct`_ backend is present. Gated on ``pcap._pcap`` rather than +#: on ``pcap``, for the same reason :attr:`PCAP_CT.__engine_module__ +#: ` names it: +#: upstream `pypcap`_ owns the ``pcap`` name just as legitimately and ships it as +#: a single extension module, so the submodule import is what tells the two +#: distributions apart. Importing rather than :func:`importlib.util.find_spec`, +#: since ``find_spec('pcap._pcap')`` has to import the ``pcap`` parent anyway and +#: raises rather than answering when that parent is not a package. +HAS_PCAP_CT = _importable('pcap._pcap') + +#: Magic number of a little-endian, microsecond-resolution PCAP savefile. +PCAP_MAGIC = b'\xd4\xc3\xb2\xa1' +#: Magic number of a PCAP-NG section header block. +PCAPNG_MAGIC = b'\x0a\x0d\x0d\x0a' + +#: Frame count of :file:`examples/captures/in.pcap`, and the timestamp and +#: capture length of its first frame. Committed capture, so these are fixed. +IN_PCAP_FRAMES = 6 +IN_PCAP_FIRST_TIMESTAMP = 1511106545.471719 +IN_PCAP_FIRST_LENGTH = 86 + + +class OutputSink: + """Stand-in for a :class:`dictdumper.dumper.Dumper`, recording what it is handed.""" + + kind = 'unit' + + def __init__(self) -> None: + self.paths: list[str] = [] + self.records: list[tuple[object, str | None]] = [] + + def __call__(self, *args, **kwargs): + if len(args) == 1 and isinstance(args[0], str) and not kwargs: + self.paths.append(args[0]) + return self + self.records.append((args[0] if args else None, kwargs.get('name'))) + return self + + +class FrameIterator: + """Iterator over a :class:`FakeHandle`'s frames, and a separate object from it. + + It carries a ``close`` of its own purely so that closing the wrong thing is + observable. The engine keeps the handle and the iterator apart precisely so + that :meth:`PCAP_CT.close ` + does not have to assume ``iter(handle) is handle``, and a stand-in that + iterated itself would make that distinction untestable. + + """ + + def __init__(self, frames) -> None: + self._iter = iter(frames) + self.closed = 0 + + def __iter__(self): + return self + + def __next__(self): + return next(self._iter) + + def close(self) -> None: + self.closed += 1 + + +class FakeHandle: + """Stand-in for :class:`pcap.pcap`, yielding ``(timestamp, bytes)`` pairs.""" + + def __init__(self, frames=None, datalink: int = 1) -> None: + self.frames = list(frames if frames is not None else [(1.5, b'payload')]) + self.iterator: FrameIterator | None = None + self._datalink = datalink + self.closed = 0 + self.setup = 0 + + def datalink(self) -> int: + return self._datalink + + def __iter__(self) -> FrameIterator: + self.setup += 1 + self.iterator = FrameIterator(self.frames) + return self.iterator + + def close(self) -> None: + self.closed += 1 + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class PCAP_CTEngineTests(unittest.TestCase): + def setUp(self) -> None: + purge_modules(['pcapkit']) + + handle, path = tempfile.mkstemp(suffix='.pcap') + os.close(handle) + self.addCleanup(os.unlink, path) + self.ifnm = path + + def make_extractor(self, **overrides): + sink = OutputSink() + reasm = types.SimpleNamespace(ipv4=mock.Mock(), ipv6=mock.Mock(), tcp=mock.Mock()) + trace = types.SimpleNamespace(tcp=mock.Mock()) + values = { + '_ifile': io.BytesIO(b'capture'), + '_ifnm': self.ifnm, + '_ofile': sink, + '_ofnm': 'out', + '_fext': 'json', + '_offmt': None, + '_flag_q': False, + '_flag_f': False, + '_flag_v': False, + '_flag_r': False, + '_flag_t': False, + '_flag_d': True, + '_ipv4': False, + '_ipv6': False, + '_tcp': False, + '_reasm': reasm, + '_trace': trace, + '_frame': [], + '_frnum': 0, + '_exlyr': 'none', + '_exptl': 'null', + '_vfunc': mock.Mock(), + 'magic_number': PCAP_MAGIC, + } + values.update(overrides) + return types.SimpleNamespace(**values), sink + + def engine(self, extractor): + from pcapkit.foundation.engines import _pcap_backend + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + + # ``__new__`` rather than the constructor: ``PCAP_CT.__init__`` insists on + # the real ``pcap-ct`` being importable, and these tests are meant to run + # with neither ``pcap-ct`` nor ``pypcap`` installed. The constructor gets + # its own tests below, against a stand-in module. + engine = PCAP_CT.__new__(PCAP_CT) + engine._expkg = types.SimpleNamespace(pcap=FakeHandle) + engine._handle = None + engine._extmp = None + engine._dlink = None + engine._closed = False + engine._backend = _pcap_backend.Probe( + _pcap_backend.PCAP_CT, '1.3.0b3', '/stub/site-packages/pcap/__init__.py', + None, False, ('pcap-ct',), + ) + engine._extractor = extractor + return engine + + ########################################################################## + # unsupported_reason() and __init__() + ########################################################################## + + def fake_pcap_module(self, *, is_pcap_ct: bool): + """A stand-in :mod:`pcap` module, as one distribution or the other. + + ``pcap-ct`` ships :mod:`pcap` as a package whose ``__init__`` does + ``from ._pcap import *``, so the submodule ends up bound as an attribute; + upstream ``pypcap`` ships a single extension module with no such + attribute. That difference is what + :func:`pcapkit.foundation.engines._pcap_backend.identify` keys on. + + """ + module = types.ModuleType('pcap') + module.pcap = FakeHandle # type: ignore[attr-defined] + module.__version__ = '1.3.0b3' if is_pcap_ct else '1.3.0' # type: ignore[attr-defined] + module.__file__ = ('/stub/site-packages/pcap/__init__.py' if is_pcap_ct + else '/stub/site-packages/pcap.cpython-310.so') + if is_pcap_ct: + module._pcap = types.ModuleType('pcap._pcap') # type: ignore[attr-defined] + return module + + def installed(self, *names: str): + """Patch the distribution metadata to report exactly ``names``.""" + from pcapkit.foundation.engines import _pcap_backend + + return mock.patch.object(_pcap_backend, 'installed_distributions', + return_value=names) + + def test_unsupported_reason_is_silent_on_pcap_ct(self) -> None: + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=True)}): + with self.installed('pcap-ct'): + self.assertIsNone(PCAP_CT.unsupported_reason()) + + def test_unsupported_reason_names_upstream_pypcap_and_its_engine(self) -> None: + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=False)}): + with self.installed('pypcap'): + reason = PCAP_CT.unsupported_reason() + + self.assertIsNotNone(reason) + self.assertIn('pypcap', reason) + self.assertIn('engine=pypcap', reason) + + def test_unsupported_reason_reports_a_missing_system_libpcap(self) -> None: + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + + # The important case, and measured rather than hypothetical: ``pcap-ct`` + # imports the ``libpcap`` distribution, whose Linux loader raises + # ``OSError: Cannot find libpcap.so library`` when no system libpcap is + # findable. OSError is not an ImportError, so ``Extractor.import_test`` + # lets it through and the extraction dies. Reporting it here turns that + # into the ordinary "engine unavailable" fall back. + with mock.patch('importlib.import_module', + side_effect=OSError('Cannot find libpcap.so library')): + with self.installed('pcap-ct'): + reason = PCAP_CT.unsupported_reason() + + self.assertIsNotNone(reason) + self.assertIn('installed but unusable', reason) + self.assertIn('Cannot find libpcap.so library', reason) + + def test_unsupported_reason_is_silent_when_no_pcap_is_installed(self) -> None: + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + + with mock.patch('importlib.import_module', side_effect=ImportError('no pcap')): + with self.installed(): + self.assertIsNone(PCAP_CT.unsupported_reason()) + + def test_unsupported_reason_declares_no_python_version_bound(self) -> None: + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + + # Verified against real installations on 3.10.20 and 3.14.7, both of which + # read in.pcap identically, so this engine covers the whole supported + # range and must not refuse either end of it. + for version in ((3, 10, 0, 'final', 0), (3, 14, 0, 'final', 0)): + with self.subTest(version=version): + with mock.patch.dict('sys.modules', + {'pcap': self.fake_pcap_module(is_pcap_ct=True)}): + with self.installed('pcap-ct'): + with mock.patch('sys.version_info', version): + self.assertIsNone(PCAP_CT.unsupported_reason()) + + def test_init_refuses_upstream_pypcap_and_names_the_engine_for_it(self) -> None: + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + from pcapkit.utilities.exceptions import UnsupportedCall + + extractor, _ = self.make_extractor() + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=False)}): + with self.installed('pypcap'): + with self.assertRaises(UnsupportedCall) as caught: + PCAP_CT(extractor) + + message = str(caught.exception) + self.assertIn('pypcap', message) + self.assertIn('engine=pypcap', message) + + def test_init_accepts_pcap_ct_and_reports_the_backend(self) -> None: + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + + extractor, _ = self.make_extractor() + module = self.fake_pcap_module(is_pcap_ct=True) + with mock.patch.dict('sys.modules', {'pcap': module}): + with self.installed('pcap-ct'): + engine = PCAP_CT(extractor) + + self.assertIs(engine._expkg, module) + self.assertIsNone(engine._handle) + self.assertFalse(engine._closed) + + self.assertIn('pcap-ct', engine.backend) + self.assertIn('1.3.0b3', engine.backend) + self.assertIn('pcap/__init__.py', engine.backend) + + def test_init_warns_when_both_distributions_are_installed(self) -> None: + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + from pcapkit.utilities.warnings import EngineWarning + + # This engine still works in that state -- ``pcap-ct`` is the one that wins + # the import -- so it warns and carries on rather than refusing. + extractor, _ = self.make_extractor() + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=True)}): + with self.installed('pypcap', 'pcap-ct'): + with mock.patch('pcapkit.foundation.engines.pcap_ct.warn') as warn: + engine = PCAP_CT(extractor) + + collisions = [call.args[0] for call in warn.call_args_list + if len(call.args) > 1 and call.args[1] is EngineWarning] + self.assertEqual(len(collisions), 1, warn.call_args_list) + self.assertIn('pypcap', collisions[0]) + self.assertIn('pcap-ct', collisions[0]) + self.assertIn('shadowed', collisions[0]) + self.assertIn('pcap-ct', engine.backend) + + def test_init_does_not_warn_when_only_one_is_installed(self) -> None: + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + from pcapkit.utilities.warnings import EngineWarning + + extractor, _ = self.make_extractor() + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=True)}): + with self.installed('pcap-ct'): + with mock.patch('pcapkit.foundation.engines.pcap_ct.warn') as warn: + PCAP_CT(extractor) + + self.assertEqual([call for call in warn.call_args_list + if len(call.args) > 1 and call.args[1] is EngineWarning], []) + + ########################################################################## + # run() + ########################################################################## + + def test_run_opens_handle_and_records_link_type(self) -> None: + from pcapkit.const.reg.linktype import LinkType + + extractor, _ = self.make_extractor() + engine = self.engine(extractor) + + handle = FakeHandle() + with mock.patch.object(engine._expkg, 'pcap', return_value=handle) as ctor: + engine.run() + + ctor.assert_called_once_with(name=self.ifnm, promisc=False) + self.assertEqual(engine.dlink, LinkType.ETHERNET) + + # the handle and the iterator are held separately, and it is the iterator + # that frames are read through + self.assertIs(engine._handle, handle) + self.assertEqual(handle.setup, 1) + self.assertIs(engine._extmp, handle.iterator) + self.assertIsNot(engine._extmp, handle) + self.assertEqual(next(engine._extmp), (1.5, b'payload')) + + def test_run_warns_on_layer_and_protocol_threshold(self) -> None: + from pcapkit.utilities.warnings import AttributeWarning + + # NOTE: ``BaseWarning.__init__`` installs an ``ignore`` filter for its own + # category outside development mode, so the warning never reaches + # ``assertWarns``; the ``warn`` call itself is what can be observed. + for overrides in ({'_exlyr': 'internet'}, {'_exptl': 'tcp'}): + with self.subTest(overrides=overrides): + extractor, _ = self.make_extractor(**overrides) + engine = self.engine(extractor) + with mock.patch.object(engine._expkg, 'pcap', return_value=FakeHandle()): + with mock.patch('pcapkit.foundation.engines.pcap_ct.warn') as warn: + engine.run() + self.assertEqual(warn.call_count, 1) + self.assertIn('protocol and layer threshold', warn.call_args.args[0]) + self.assertIs(warn.call_args.args[1], AttributeWarning) + + def test_run_rejects_pcapng_rather_than_reading_it_approximately(self) -> None: + from pcapkit.utilities.exceptions import FormatError + + # the vendored libpcap *can* read a PCAP-NG savefile, but only ever reports + # one link type for it, so the engine refuses rather than applying one + # interface's link type to every frame + extractor, _ = self.make_extractor(magic_number=PCAPNG_MAGIC) + engine = self.engine(extractor) + with mock.patch.object(engine._expkg, 'pcap', return_value=FakeHandle()) as ctor: + with self.assertRaises(FormatError): + engine.run() + ctor.assert_not_called() + + def test_run_rejects_input_that_is_not_a_file_on_disk(self) -> None: + from pcapkit.utilities.exceptions import UnsupportedCall + + extractor, _ = self.make_extractor(_ifnm=os.path.join(self.ifnm, 'nope.pcap')) + engine = self.engine(extractor) + with mock.patch.object(engine._expkg, 'pcap', return_value=FakeHandle()) as ctor: + with self.assertRaises(UnsupportedCall): + engine.run() + ctor.assert_not_called() + + def test_run_disables_reassembly_and_flow_tracing(self) -> None: + from pcapkit.utilities.warnings import AttributeWarning + + extractor, _ = self.make_extractor(_flag_r=True, _flag_t=True, _ipv4=True, + _ipv6=True, _tcp=True) + engine = self.engine(extractor) + with mock.patch.object(engine._expkg, 'pcap', return_value=FakeHandle()): + with mock.patch('pcapkit.foundation.engines.pcap_ct.warn') as warn: + engine.run() + + messages = [call.args[0] for call in warn.call_args_list + if call.args[1] is AttributeWarning] + self.assertTrue(any('reassembly' in message for message in messages), messages) + self.assertTrue(any('flow tracing' in message for message in messages), messages) + + self.assertFalse(extractor._flag_r) + self.assertFalse(extractor._flag_t) + self.assertIsNone(extractor._reasm.ipv4) + self.assertIsNone(extractor._reasm.ipv6) + self.assertIsNone(extractor._reasm.tcp) + self.assertIsNone(extractor._trace.tcp) + + def test_run_leaves_disabled_reassembly_and_tracing_untouched(self) -> None: + extractor, _ = self.make_extractor(_flag_r=True, _flag_t=True) + original_reasm, original_trace = extractor._reasm, extractor._trace + engine = self.engine(extractor) + with mock.patch.object(engine._expkg, 'pcap', return_value=FakeHandle()): + engine.run() + + # no protocol was requested, so there is nothing to warn about or replace + self.assertTrue(extractor._flag_r) + self.assertTrue(extractor._flag_t) + self.assertIs(extractor._reasm, original_reasm) + self.assertIs(extractor._trace, original_trace) + + def test_run_installs_verbose_handler_reporting_the_raw_chain(self) -> None: + extractor, _ = self.make_extractor(_flag_v=True) + engine = self.engine(extractor) + with mock.patch.object(engine._expkg, 'pcap', return_value=FakeHandle()): + engine.run() + + with mock.patch('builtins.print') as printer: + extractor._frnum = 3 + extractor._vfunc(extractor, (1.5, b'payload')) + printer.assert_called_once() + self.assertIn('ETHERNET:Raw', printer.call_args.args[0]) + + ########################################################################## + # read_frame() + ########################################################################## + + def prepared(self, **overrides): + extractor, sink = self.make_extractor(**overrides) + engine = self.engine(extractor) + handle = FakeHandle(frames=[(1.5, b'payload'), (2.5, b'second')]) + with mock.patch.object(engine._expkg, 'pcap', return_value=handle): + engine.run() + return extractor, sink, engine + + def test_read_frame_returns_the_pair_and_routes_output_and_storage(self) -> None: + extractor, sink, engine = self.prepared() + + frame = engine.read_frame() + + self.assertEqual(frame, (1.5, b'payload')) + self.assertEqual(extractor._frnum, 1) + extractor._vfunc.assert_called_once_with(extractor, frame) + self.assertEqual(sink.records[-1][1], 'Frame 1') + self.assertEqual(sink.records[-1][0]['timestamp'], 1.5) + self.assertEqual(sink.records[-1][0]['packet'], b'payload') + self.assertEqual(sink.records[-1][0]['ETHERNET'], {'raw_len': 7, 'raw': b'payload'}) + self.assertEqual(extractor._offmt, 'unit') + self.assertEqual(extractor._frame, [frame]) + + self.assertEqual(engine.read_frame(), (2.5, b'second')) + self.assertEqual(extractor._frnum, 2) + with self.assertRaises(StopIteration): + engine.read_frame() + + def test_read_frame_splits_files_when_asked(self) -> None: + extractor, sink, engine = self.prepared(_flag_f=True) + engine.read_frame() + self.assertEqual(sink.paths[-1], 'out/Frame 1.json') + + def test_read_frame_writes_nothing_and_stores_nothing_when_disabled(self) -> None: + extractor, sink, engine = self.prepared(_flag_q=True, _flag_d=False) + engine.read_frame() + self.assertEqual(sink.records, []) + self.assertEqual(sink.paths, []) + self.assertEqual(extractor._frame, []) + self.assertIsNone(extractor._offmt) + + ########################################################################## + # close() + ########################################################################## + + def test_close_is_idempotent_and_tolerates_an_unopened_engine(self) -> None: + extractor, _, engine = self.prepared() + handle = engine._handle + + engine.close() + engine.close() + self.assertEqual(handle.closed, 1) + # the savefile belongs to the handle, not to the iterator read off it + self.assertEqual(handle.iterator.closed, 0) + + extractor, _ = self.make_extractor() + unopened = self.engine(extractor) + unopened.close() # must not raise + + ########################################################################## + # The real backend. + ########################################################################## + + @unittest.skipUnless(HAS_PCAP_CT, 'pcap-ct not installed') + def test_the_real_backend_reads_a_committed_capture(self) -> None: + from pcapkit.foundation.engines.pcap_ct import PCAP_CT + from pcapkit.interface import extract + from pcapkit.utilities.warnings import EngineWarning + + with mock.patch('pcapkit.foundation.extraction.warn') as warn: + extractor = extract(fin=sample_path('in.pcap'), engine='pcap_ct', + store=True, nofile=True) + self.addCleanup(close_extractor, extractor) + + # A missing engine module only warns and falls back to pcapkit's own + # parser, so a successful extraction proves nothing on its own -- it is the + # absence of that warning, plus the engine actually on the extractor, that + # says this ran through ``pcap-ct``. + engines = [call.args[0] for call in warn.call_args_list + if len(call.args) > 1 and call.args[1] is EngineWarning] + self.assertEqual(engines, [], "'pcap_ct' was replaced by the fallback engine") + self.assertEqual(extractor._exnam, 'pcap_ct') + self.assertIsInstance(extractor.engine, PCAP_CT) + self.assertEqual(type(extractor.engine).__engine_name__, 'PCAP_CT') + + self.assertEqual(extractor.length, IN_PCAP_FRAMES) + self.assertEqual(len(extractor.frame), IN_PCAP_FRAMES) + + # a fallback would have produced ``Frame`` objects, not bare pairs + timestamp, packet = extractor.frame[0] + self.assertEqual(timestamp, IN_PCAP_FIRST_TIMESTAMP) + self.assertEqual(len(packet), IN_PCAP_FIRST_LENGTH) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/foundation/engines/test_pypcap_engine.py b/tests/foundation/engines/test_pypcap_engine.py index de5e2ecfd6..39e60ccea7 100644 --- a/tests/foundation/engines/test_pypcap_engine.py +++ b/tests/foundation/engines/test_pypcap_engine.py @@ -124,6 +124,149 @@ def engine(self, extractor): engine._extractor = extractor return engine + ########################################################################## + # unsupported_reason() and __init__() + ########################################################################## + + def fake_pcap_module(self, *, is_pcap_ct: bool): + """A stand-in :mod:`pcap` module, as one distribution or the other. + + ``pcap-ct`` ships :mod:`pcap` as a package whose ``__init__`` does + ``from ._pcap import *``, so the submodule ends up bound as an attribute; + upstream ``pypcap`` ships a single extension module with no such + attribute. That difference is what + :func:`pcapkit.foundation.engines._pcap_backend.identify` keys on, so the + stand-in only has to reproduce it. + + """ + module = types.ModuleType('pcap') + module.pcap = FakeHandle # type: ignore[attr-defined] + module.__version__ = '1.3.0b3' if is_pcap_ct else '1.3.0' # type: ignore[attr-defined] + module.__file__ = ('/stub/site-packages/pcap/__init__.py' if is_pcap_ct + else '/stub/site-packages/pcap.cpython-310.so') + if is_pcap_ct: + module._pcap = types.ModuleType('pcap._pcap') # type: ignore[attr-defined] + return module + + def installed(self, *names: str): + """Patch the distribution metadata to report exactly ``names``.""" + from pcapkit.foundation.engines import _pcap_backend + + return mock.patch.object(_pcap_backend, 'installed_distributions', + return_value=names) + + def test_unsupported_reason_is_silent_on_upstream_pypcap(self) -> None: + from pcapkit.foundation.engines.pypcap import PyPCAP + + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=False)}): + with self.installed('pypcap'): + self.assertIsNone(PyPCAP.unsupported_reason()) + + def test_unsupported_reason_names_pcap_ct_and_the_engine_for_it(self) -> None: + from pcapkit.foundation.engines.pypcap import PyPCAP + + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=True)}): + with self.installed('pcap-ct'): + reason = PyPCAP.unsupported_reason() + + self.assertIsNotNone(reason) + self.assertIn('pcap-ct', reason) + self.assertIn('engine=pcap_ct', reason) + + def test_unsupported_reason_is_silent_when_no_pcap_is_installed(self) -> None: + from pcapkit.foundation.engines.pypcap import PyPCAP + + # Not this hook's business: ``Extractor.import_test`` reports an absent + # module in its own words, and answering here as well would produce two + # warnings for one problem. + with mock.patch('importlib.import_module', side_effect=ImportError('no pcap')): + with self.installed(): + self.assertIsNone(PyPCAP.unsupported_reason()) + + def test_unsupported_reason_is_not_a_python_version_check(self) -> None: + from pcapkit.foundation.engines.pypcap import PyPCAP + + # Upstream cannot be *installed* on 3.12+, but the verdict is about which + # distribution is present rather than which interpreter is running: an + # upstream build that somebody got working on a newer Python must not be + # refused, and on 3.10/3.11 the version says nothing useful because either + # distribution could be the one in place. + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=False)}): + with self.installed('pypcap'): + with mock.patch('sys.version_info', (3, 14, 0, 'final', 0)): + self.assertIsNone(PyPCAP.unsupported_reason()) + + def test_init_refuses_pcap_ct_and_names_the_engine_that_wants_it(self) -> None: + from pcapkit.foundation.engines.pypcap import PyPCAP + from pcapkit.utilities.exceptions import UnsupportedCall + + extractor, _ = self.make_extractor() + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=True)}): + with self.installed('pcap-ct'): + with self.assertRaises(UnsupportedCall) as caught: + PyPCAP(extractor) + + # the message has to be actionable: the whole reason this refuses rather + # than running is that ``pcap-ct`` has an engine of its own + message = str(caught.exception) + self.assertIn('pcap-ct', message) + self.assertIn('engine=pcap_ct', message) + self.assertIn('1.3.0b3', message) + + def test_init_accepts_upstream_pypcap_and_reports_the_backend(self) -> None: + from pcapkit.foundation.engines.pypcap import PyPCAP + + extractor, _ = self.make_extractor() + module = self.fake_pcap_module(is_pcap_ct=False) + with mock.patch.dict('sys.modules', {'pcap': module}): + with self.installed('pypcap'): + engine = PyPCAP(extractor) + + self.assertIs(engine._expkg, module) + self.assertIsNone(engine._extmp) + self.assertFalse(engine._closed) + + # "the PyPCAP engine" is not a complete statement of what ran, since two + # distributions answer to it -- ``backend`` says which one did + self.assertIn('pypcap', engine.backend) + self.assertIn('1.3.0', engine.backend) + self.assertIn('pcap.cpython-310.so', engine.backend) + + def test_init_warns_when_both_distributions_are_installed(self) -> None: + from pcapkit.foundation.engines.pypcap import PyPCAP + from pcapkit.utilities.exceptions import UnsupportedCall + from pcapkit.utilities.warnings import EngineWarning + + # Measured on Python 3.10 with both installed: the ``pcap-ct`` package + # wins the import and upstream's extension module is shadowed, so this + # engine becomes permanently unselectable. Nothing else would say why. + extractor, _ = self.make_extractor() + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=True)}): + with self.installed('pypcap', 'pcap-ct'): + with mock.patch('pcapkit.foundation.engines.pypcap.warn') as warn: + with self.assertRaises(UnsupportedCall): + PyPCAP(extractor) + + collisions = [call.args[0] for call in warn.call_args_list + if len(call.args) > 1 and call.args[1] is EngineWarning] + self.assertEqual(len(collisions), 1, warn.call_args_list) + self.assertIn('pypcap', collisions[0]) + self.assertIn('pcap-ct', collisions[0]) + self.assertIn('shadowed', collisions[0]) + + def test_init_does_not_warn_when_only_one_is_installed(self) -> None: + from pcapkit.foundation.engines.pypcap import PyPCAP + from pcapkit.utilities.warnings import EngineWarning + + extractor, _ = self.make_extractor() + with mock.patch.dict('sys.modules', {'pcap': self.fake_pcap_module(is_pcap_ct=False)}): + with self.installed('pypcap'): + with mock.patch('pcapkit.foundation.engines.pypcap.warn') as warn: + PyPCAP(extractor) + + self.assertEqual([call for call in warn.call_args_list + if len(call.args) > 1 and call.args[1] is EngineWarning], []) + ########################################################################## # run() ########################################################################## diff --git a/tests/foundation/engines/test_pyshark_engine.py b/tests/foundation/engines/test_pyshark_engine.py new file mode 100644 index 0000000000..4d9e03fca9 --- /dev/null +++ b/tests/foundation/engines/test_pyshark_engine.py @@ -0,0 +1,209 @@ +"""Unit tests for :meth:`pcapkit.foundation.engines.pyshark.PyShark.unsupported_reason`. + +The rest of the engine is exercised end to end in +:mod:`tests.foundation.engines.test_runtime_engines`, which needs `PyShark`_ and the +generated captures. This module covers only the preflight check, which is the part +that has to give the right answer on a machine where `PyShark`_ cannot run at all -- +including this one. + +`PyShark`_ is refused for two independent reasons, and both are measured: + +* **the interpreter** -- ``pyshark`` 0.6 builds its event loop with + ``asyncio.get_event_loop_policy().get_event_loop()``. Run in a fresh interpreter + with no loop, that returns a loop on 3.10 and 3.11, returns one with a + :exc:`DeprecationWarning` on 3.12, and raises ``RuntimeError: There is no + current event loop in thread 'MainThread'`` on 3.14. So the ceiling is + ``(3, 14)``. Python 3.13 was not available to measure; it is expected to work, + being on the deprecated-but-functional side of that change, and that is the only + inferred claim here. +* **the** :program:`tshark` **binary** -- ``pyshark`` shells out to it and parses + nothing itself. + +What this host could and could not provide, stated plainly rather than left to a +silent skip: :program:`tshark` is **not** installed here, so the "missing binary" +path is exercised for real. The "binary present" path is exercised by patching +``pyshark``'s own resolver, since installing Wireshark was not an option; and the +interpreter is 3.14, so every version below the ceiling is reached by patching +:data:`sys.version_info`, which is the same technique +:mod:`tests.foundation.engines.test_pypcapfile_engine` uses. + +.. _PyShark: https://kiminewt.github.io/pyshark + +""" +from __future__ import annotations + +import importlib.util +import sys +import unittest +from unittest import mock + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) + +HAS_PYSHARK = importlib.util.find_spec('pyshark') is not None + +#: A version below the ceiling, used to reach the :program:`tshark` branch, which +#: is otherwise unreachable on the 3.14 interpreter these tests run on. +SUPPORTED_VERSION = (3, 11, 0, 'final', 0) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class PySharkUnsupportedReasonTests(unittest.TestCase): + def resolver(self, *, found: bool): + """Install a stand-in ``pyshark.tshark.tshark`` for the duration of a block. + + A stand-in rather than :func:`unittest.mock.patch` on the real module, so + that these tests do not need `PyShark`_ installed. That matters for the + version cases in particular: the whole point of deciding the ceiling from + :data:`sys.version_info` is that the answer is the same on a machine that + has never installed ``pyshark``, and a test that could only run where it + *is* installed would not be checking that. + + """ + import types + + message = ("TShark not found. Try adding its location to the configuration " + "file. Searched these paths: ['/usr/bin/tshark', '/usr/sbin/tshark']") + + def get_process_path(*args, **kwargs): + if found: + return '/usr/bin/tshark' + raise Exception(message) + + module = types.ModuleType('pyshark.tshark.tshark') + module.get_process_path = get_process_path # type: ignore[attr-defined] + return mock.patch.dict('sys.modules', { + 'pyshark': types.ModuleType('pyshark'), + 'pyshark.tshark': types.ModuleType('pyshark.tshark'), + 'pyshark.tshark.tshark': module, + }) + + def found_tshark(self): + """A resolver that reports a binary it found.""" + return self.resolver(found=True) + + def missing_tshark(self): + """A resolver that raises the way ``pyshark``'s does when it finds nothing.""" + return self.resolver(found=False) + + ########################################################################## + # The Python ceiling. + ########################################################################## + + def test_the_ceiling_is_the_measured_one(self) -> None: + from pcapkit.foundation.engines.pyshark import PyShark + + self.assertEqual(PyShark.PYTHON_CEILING, (3, 14)) + + @unittest.skipUnless(HAS_PYSHARK, 'pyshark not installed') + def test_the_reason_tracks_the_running_interpreter(self) -> None: + from pcapkit.foundation.engines.pyshark import PyShark + + reason = PyShark.unsupported_reason() + if sys.version_info[:2] >= PyShark.PYTHON_CEILING: + self.assertIsNotNone(reason) + # the message has to name the cause; "unsupported" sends the reader + # looking in the wrong place + self.assertIn('asyncio', reason) # type: ignore[arg-type] + self.assertIn(f'{sys.version_info[0]}.{sys.version_info[1]}', + reason) # type: ignore[arg-type] + else: + # below the ceiling the verdict is about tshark, which this host does + # not have -- so still a reason, but a different one + self.assertIsNotNone(reason) + self.assertIn('tshark', reason) # type: ignore[arg-type] + + def test_the_ceiling_is_decided_by_version_not_by_an_import(self) -> None: + """The verdict must not depend on whether ``pyshark`` is installed. + + Otherwise the answer differs between a machine that has the package and + one that does not, and the engine would call itself usable on 3.14 purely + because the failing ``asyncio`` call had not been reached yet. + + """ + from pcapkit.foundation.engines.pyshark import PyShark + + for version, refused in (((3, 10), False), ((3, 11), False), ((3, 12), False), + ((3, 13), False), ((3, 14), True), ((3, 15), True)): + with self.subTest(python=version): + with mock.patch.object(sys, 'version_info', (*version, 0, 'final', 0)): + with self.found_tshark(): + reason = PyShark.unsupported_reason() + self.assertEqual(reason is not None, refused, reason) + if refused: + self.assertIn('asyncio', reason) # type: ignore[arg-type] + + def test_the_version_reason_is_checked_before_the_binary(self) -> None: + from pcapkit.foundation.engines.pyshark import PyShark + + # On 3.14 the engine cannot run whatever the binary situation is, so the + # interpreter is the reason worth reporting -- and it costs no filesystem + # probe to decide. + with mock.patch.object(sys, 'version_info', (3, 14, 0, 'final', 0)): + with self.found_tshark(): + reason = PyShark.unsupported_reason() + + self.assertIsNotNone(reason) + self.assertIn('asyncio', reason) # type: ignore[arg-type] + self.assertNotIn('tshark', reason) # type: ignore[arg-type] + + ########################################################################## + # The tshark binary. + ########################################################################## + + def test_a_missing_binary_is_reported_with_what_was_searched(self) -> None: + from pcapkit.foundation.engines.pyshark import PyShark + + with mock.patch.object(sys, 'version_info', SUPPORTED_VERSION): + with self.missing_tshark(): + reason = PyShark.unsupported_reason() + + self.assertIsNotNone(reason) + self.assertIn('tshark', reason) # type: ignore[arg-type] + # upstream's own message lists every candidate path, which is exactly what + # the user needs, so it is quoted rather than summarised away + self.assertIn('Searched these paths', reason) # type: ignore[arg-type] + + def test_a_present_binary_is_no_reason_at_all(self) -> None: + from pcapkit.foundation.engines.pyshark import PyShark + + with mock.patch.object(sys, 'version_info', SUPPORTED_VERSION): + with self.found_tshark(): + self.assertIsNone(PyShark.unsupported_reason()) + + @unittest.skipUnless(HAS_PYSHARK, 'pyshark not installed') + def test_this_host_really_has_no_tshark_so_the_check_is_not_vacuous(self) -> None: + """Exercise the failing path through ``pyshark``'s resolver, unpatched. + + The other tests patch ``get_process_path``, which proves the branch is + wired up but not that the real resolver ever says no. This one calls it, + and is skipped only if :program:`tshark` turns out to be installed after + all -- in which case the complementary + :meth:`test_a_present_binary_is_no_reason_at_all` is the real check. + + """ + import shutil + + from pcapkit.foundation.engines.pyshark import PyShark + + if shutil.which('tshark') is not None: + self.skipTest('tshark is installed on this host') + + with mock.patch.object(sys, 'version_info', SUPPORTED_VERSION): + reason = PyShark.unsupported_reason() + + self.assertIsNotNone(reason) + self.assertIn('tshark', reason) # type: ignore[arg-type] + + def test_an_absent_pyshark_is_left_to_the_import_test(self) -> None: + from pcapkit.foundation.engines.pyshark import PyShark + + # Reporting "not installed" here as well would produce two warnings for + # one problem: ``Extractor.import_test`` already says it, in its own words. + with mock.patch.object(sys, 'version_info', SUPPORTED_VERSION): + with mock.patch.dict('sys.modules', {'pyshark.tshark.tshark': None}): + self.assertIsNone(PyShark.unsupported_reason()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/toolkit/test_pcap_ct_unit.py b/tests/toolkit/test_pcap_ct_unit.py new file mode 100644 index 0000000000..b0f30f9c0f --- /dev/null +++ b/tests/toolkit/test_pcap_ct_unit.py @@ -0,0 +1,89 @@ +"""Unit tests for :mod:`pcapkit.toolkit.pcap_ct`. + +`pcap-ct`_ hands back raw bytes, so this toolkit has only the two auxiliary +functions; the reassembly and flow tracing adapters exist purely to refuse +loudly, and that refusal is asserted here so it cannot regress into a silent +:data:`None`. + +Nothing below imports :mod:`pcap`: the module under test is a pure adapter over +the ``(timestamp, bytes)`` pair, so it is testable with no backend installed at +all -- which is also the point of asserting the refusals here rather than only in +an end-to-end test that skips without one. + +.. _pcap-ct: https://pypi.org/project/pcap-ct/ + +""" +from __future__ import annotations + +import importlib.util +import unittest + +from tests._support import purge_modules + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class PCAP_CTToolkitTests(unittest.TestCase): + def setUp(self) -> None: + purge_modules(['pcapkit']) + + def test_packet2chain_reports_the_link_layer_and_raw(self) -> None: + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit.pcap_ct import packet2chain + + self.assertEqual(packet2chain(b'\x00' * 20, data_link=LinkType.ETHERNET), + 'ETHERNET:Raw') + self.assertEqual(packet2chain(b'', data_link=LinkType.RAW), 'RAW:Raw') + + def test_packet2dict_carries_the_bytes_verbatim(self) -> None: + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit.pcap_ct import packet2dict + + info = packet2dict(b'abcd', 1.25, data_link=LinkType.ETHERNET) + self.assertEqual(info, { + 'timestamp': 1.25, + 'packet': b'abcd', + 'ETHERNET': {'raw_len': 4, 'raw': b'abcd'}, + }) + + # the recorded length is of the captured bytes, not of the original frame: + # libpcap hands over only what it captured + info = packet2dict(b'', 0.0, data_link=LinkType.RAW) + self.assertEqual(info['RAW']['raw_len'], 0) + + def test_unsupported_adapters_refuse_loudly(self) -> None: + from pcapkit.const.reg.linktype import LinkType + from pcapkit.toolkit.pcap_ct import (ipv4_reassembly, ipv6_reassembly, tcp_reassembly, + tcp_traceflow) + from pcapkit.utilities.exceptions import UnsupportedCall + + for name, call in ( + ('ipv4_reassembly', lambda: ipv4_reassembly(b'raw', count=1)), + ('ipv6_reassembly', lambda: ipv6_reassembly(b'raw', count=1)), + ('tcp_reassembly', lambda: tcp_reassembly(b'raw', count=1)), + ('tcp_traceflow', lambda: tcp_traceflow(b'raw', 1.0, + data_link=LinkType.ETHERNET, count=1)), + ): + with self.subTest(function=name): + with self.assertRaises(UnsupportedCall) as caught: + call() + # the message has to say *why*, since "unsupported" alone reads as + # an oversight rather than as a property of a libpcap binding + self.assertIn('no protocol dissection', str(caught.exception)) + self.assertIn('pcap-ct', str(caught.exception)) + + def test_module_exports_the_documented_surface(self) -> None: + from pcapkit.toolkit import pcap_ct + + self.assertEqual(sorted(pcap_ct.__all__), [ + 'ipv4_reassembly', 'ipv6_reassembly', 'packet2chain', 'packet2dict', + 'tcp_reassembly', 'tcp_traceflow', + ]) + for name in pcap_ct.__all__: + self.assertTrue(callable(getattr(pcap_ct, name)), name) + + +if __name__ == '__main__': + unittest.main()