reassembly: make the four IPv6 adapters agree, and analyse datagram payloads lazily

docs/source/pcapkit/foundation/reassembly/index.rst

@@ -53,6 +53,16 @@ Auxiliary Data
    :members:
    :show-inheritance:
 
-.. autoclass:: pcapkit.foundation.reassembly.data.ReassemblyData
+.. module:: pcapkit.foundation.reassembly.data.data
+
+.. autoclass:: pcapkit.foundation.reassembly.data.data.ReassemblyData
+   :members:
+   :show-inheritance:
+
+.. autoclass:: pcapkit.foundation.reassembly.data.data.Deferred
+   :members:
+   :show-inheritance:
+
+.. autoclass:: pcapkit.foundation.reassembly.data.data.DeferredPacket
    :members:
    :show-inheritance:

docs/source/pcapkit/foundation/reassembly/ip/ip.rst

@@ -46,6 +46,7 @@ Data Models
    :members:
    :show-inheritance:
 
+
 Type Variables
 --------------
 

docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst

@@ -82,6 +82,17 @@ Terminology
            |     |--> 'packet' : (None)
            |--> (Info) data ...
 
+       .. note::
+
+          ``packet`` is analysed on the first read, not when the datagram is
+          submitted. A datagram is submitted for *every* frame -- an unfragmented
+          one included, since nothing upstream filters it out -- and the analysis
+          is a second full parse of the payload, so running it eagerly charged
+          every caller for a result most never read. Reading the attribute, or any
+          mapping view of it (``datagram['packet']``, ``to_dict()``, ``items()``,
+          ``repr()``), runs it and keeps the result; see
+          :class:`~pcapkit.foundation.reassembly.data.data.Deferred`.
+
    reasm.ipv4.buffer
        Data structure for internal buffering when performing reassembly algorithms
        (:attr:`IPv4._buffer <pcapkit.foundation.reassembly.reassembly.Reassembly._buffer>`)

docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst

@@ -29,6 +29,9 @@ Terminology
 
        .. code-block:: python
 
+          hdr_len = ipv6_info.hdr_len - ipv6_frag.length
+          payload = bytearray(ipv6_info.fragment.payload)
+
           packet_dict = dict(
             bufid = (
                 ipv6_info.src,              # source IP address
@@ -38,27 +41,24 @@ Terminology
             ),
             num = frame.info.number,        # original packet range number
             fo = ipv6_frag_info.offset,     # fragment offset, in octets
-            ihl = ipv6_info.hdr_len,        # header length, IPv6-Frag included
+            ihl = hdr_len,                  # header length, only headers before IPv6-Frag
             mf = ipv6_frag_info.mf,         # more fragment flag
-            tl = ipv6_info.hdr_len
-                 + ipv6_info.raw_len,       # total length, header includes
+            tl = hdr_len + len(payload),    # total length, header includes
             header = ipv6_info.fragment
-                     .header,               # raw bytes type header, IPv6-Frag included
-            payload = bytearray(
-                ipv6_info.fragment
-                .payload),                  # raw bytearray type payload after IPv6-Frag
+                     .header[:hdr_len],     # raw bytes type header before IPv6-Frag
+            payload = payload,              # raw bytearray type payload after IPv6-Frag
           )
 
-       .. warning::
+       .. note::
 
-          ``ihl`` and ``header`` here **include** the 8-octet Fragment header,
-          because :attr:`IPv6.hdr_len <pcapkit.protocols.data.internet.ipv6.IPv6.hdr_len>`
-          counts every extension header it has walked, the Fragment one included.
-          The ``dpkt`` and ``scapy`` adapters stop short of it and report 40 where
-          this one reports 48 for the same packet, so the value is not comparable
-          across engines -- and :rfc:`8200#section-4.5` says the Fragment header
-          is not present in a reassembled packet at all. Tracked as #415; expect
-          this line to change when that is fixed.
+          ``ihl``, ``header`` and ``tl`` all stop short of the 8-octet Fragment
+          header, because :rfc:`8200#section-4.5` says it is not present in the
+          reassembled packet. :attr:`IPv6.hdr_len <pcapkit.protocols.data.internet.ipv6.IPv6.hdr_len>`
+          does count it -- it is a header length, and the Fragment header is one
+          of the extension headers it has walked -- so the adapters subtract it
+          back off. All four adapters (``pcap``, ``pcapng``, ``dpkt`` and
+          ``scapy``) agree on the three fields; they used to report three
+          different values for ``tl`` alone, which is what #415 was about.
 
        .. note::
 
@@ -104,6 +104,28 @@ Terminology
            |     |--> 'packet' : (None)
            |--> (Info) data ...
 
+       .. note::
+
+          ``header`` is the fragment's unfragmentable part with one field
+          rewritten: the Next Header field of its last header carries the
+          Fragment header's Next Header value, as :rfc:`8200#section-4.5`
+          requires of a reassembled packet. Without that rewrite the datagram
+          would still advertise a Fragment header (``44``) on a datagram that is
+          no longer a fragment. The Payload Length field is *not* adjusted, so it
+          still describes the first fragment rather than the reassembled
+          datagram; use ``len(payload)`` instead.
+
+       .. note::
+
+          ``packet`` is analysed on the first read, not when the datagram is
+          submitted. A datagram is submitted for *every* frame -- an unfragmented
+          one included, since nothing upstream filters it out -- and the analysis
+          is a second full parse of the payload, so running it eagerly charged
+          every caller for a result most never read. Reading the attribute, or any
+          mapping view of it (``datagram['packet']``, ``to_dict()``, ``items()``,
+          ``repr()``), runs it and keeps the result; see
+          :class:`~pcapkit.foundation.reassembly.data.data.Deferred`.
+
    reasm.ipv6.buffer
        Data structure for internal buffering when performing reassembly algorithms
        (:attr:`IPv6._buffer <pcapkit.foundation.reassembly.reassembly.Reassembly._buffer>`)

docs/source/pcapkit/foundation/traceflow/index.rst

@@ -54,6 +54,8 @@ Auxiliary Data
    :members:
    :show-inheritance:
 
-.. autoclass:: pcapkit.foundation.traceflow.data.TraceFlowData
+.. module:: pcapkit.foundation.traceflow.data.data
+
+.. autoclass:: pcapkit.foundation.traceflow.data.data.TraceFlowData
    :members:
    :show-inheritance:

pcapkit/foundation/reassembly/data/__init__.py

@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
 """data models for reassembly"""
 
+# shared
+from pcapkit.foundation.reassembly.data.data import Deferred, DeferredPacket, ReassemblyData
+
 # IP reassembly
 from pcapkit.foundation.reassembly.data.ip import Buffer as IP_Buffer
 from pcapkit.foundation.reassembly.data.ip import BufferID as IP_BufferID
@@ -18,31 +21,11 @@
 from pcapkit.foundation.reassembly.data.tcp import Packet as TCP_Packet
 
 __all__ = [
+    'ReassemblyData', 'Deferred', 'DeferredPacket',
+
     'IP_Packet', 'IP_DatagramID', 'IP_Datagram', 'IP_Buffer',
     'IP_BufferID',
 
     'TCP_Packet', 'TCP_DatagramID', 'TCP_Datagram', 'TCP_Buffer',
     'TCP_Fragment', 'TCP_HoleDescriptor', 'TCP_BufferID',
 ]
-
-from typing import TYPE_CHECKING
-
-from pcapkit.corekit.infoclass import Info, info_final
-
-if TYPE_CHECKING:
-    from typing import Optional
-
-
-@info_final
-class ReassemblyData(Info):
-    """Data storage for reassembly."""
-
-    #: IPv4 reassembled data.
-    ipv4: 'tuple[IP_Datagram, ...]'
-    #: IPv6 reassembled data.
-    ipv6: 'tuple[IP_Datagram, ...]'
-    #: TCP reassembled data.
-    tcp: 'tuple[TCP_Datagram, ...]'
-
-    if TYPE_CHECKING:
-        def __init__(self, ipv4: 'Optional[tuple[IP_Datagram, ...]]', ipv6: 'Optional[tuple[IP_Datagram, ...]]', tcp: 'Optional[tuple[TCP_Datagram, ...]]') -> 'None': ...  # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long

pcapkit/foundation/reassembly/data/data.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+"""shared data models for reassembly"""
+
+from typing import TYPE_CHECKING
+
+from pcapkit.corekit.infoclass import Info, info_final
+
+__all__ = ['ReassemblyData', 'Deferred', 'DeferredPacket']
+
+if TYPE_CHECKING:
+    from typing import Callable, Optional
+
+    from typing import Any
+
+    from pcapkit.const.reg.transtype import TransType
+    from pcapkit.foundation.reassembly.data.ip import Datagram as IP_Datagram
+    from pcapkit.foundation.reassembly.data.tcp import Datagram as TCP_Datagram
+    from pcapkit.protocols.protocol import ProtocolBase as Protocol
+
+
+class Deferred:
+    """A postponed analysis of a reassembled payload.
+
+    A reassembled datagram's ``packet`` is a second, full parse of the payload
+    the datagram just reassembled. Nothing about postponing it is specific to any
+    one reassembler, which is why this lives beside
+    :class:`~pcapkit.foundation.reassembly.data.data.ReassemblyData` rather than in
+    either protocol's data module.
+
+    IP reassembly is the case that made it necessary. It submits a datagram for
+    *every* frame -- not only the fragmented ones, since a frame that is not
+    fragmented in any sense still reaches
+    :meth:`IP.reassembly <pcapkit.foundation.reassembly.ip.IP.reassembly>` and is
+    submitted there as a trivially complete datagram -- so the eager parse
+    re-parsed captures holding no fragments at all: :file:`http.pcap` has 1117
+    IPv4 frames, none of them fragmented, and the parse was 86% of the cost of IP
+    reassembly over it.
+
+    TCP reassembly builds its ``packet`` eagerly too
+    (:meth:`TCP.submit <pcapkit.foundation.reassembly.tcp.TCP.submit>`). It is a
+    far smaller cost there, being FIN/RST-driven rather than per-frame -- 222
+    submits per :file:`http.pcap` pass against 1117 -- so it is left for its own
+    change, but it can use this unmodified when someone gets to it.
+
+    Holding the call here defers it to the first read of
+    :attr:`Datagram.packet`, so a caller that wants the parsed payload still gets
+    exactly the object the eager call produced, and one that does not never pays
+    for it.
+
+    Args:
+        analyze: The analyser to call, i.e.
+            :meth:`Protocol.analyze <pcapkit.protocols.protocol.ProtocolBase.analyze>`
+            bound to the reassembly object's protocol.
+        proto: Payload protocol type.
+        payload: Reassembled payload to parse.
+
+    """
+
+    __slots__ = ('analyze', 'proto', 'payload')
+
+    def __init__(self, analyze: 'Callable[[TransType, bytes], Protocol]',
+                 proto: 'TransType', payload: 'bytes') -> 'None':
+        self.analyze = analyze
+        self.proto = proto
+        self.payload = payload
+
+    def __call__(self) -> 'Protocol':
+        """Run the postponed analysis.
+
+        Returns:
+            Parsed payload.
+
+        """
+        return self.analyze(self.proto, self.payload)
+
+
+class DeferredPacket:
+    """Resolves a :class:`Deferred` ``packet`` field on first read.
+
+    A reassembled datagram's ``packet`` is the parsed form of the payload it just
+    reassembled, and both reassemblers can hand a :class:`Deferred` in its place.
+    This carries the reading half of that arrangement, so the two ``Datagram``
+    models share it rather than each declaring it.
+
+    A subclass has to list ``packet`` in its ``__additional__``. That is what makes
+    the field lazy at all: :class:`~pcapkit.corekit.infoclass.Info` stores a field
+    whose name is a *builtin* name under a mangled key and maps it back on the way
+    out, so ``packet`` never lands in :attr:`~object.__dict__` itself -- which
+    routes reading it through :meth:`__getattr__`, where the deferred analysis can
+    run, while ``dict(datagram)``, :meth:`to_dict` and iteration still report the
+    field under its own name.
+
+    """
+
+    def __analyse__(self) -> 'Optional[Protocol]':
+        """Resolve a deferred analysis, at most once.
+
+        Returns:
+            Parsed IP payload, or :data:`None` for an incomplete datagram.
+
+        """
+        key = self.__map__.get('packet', 'packet')
+        value = self.__dict__[key]
+        if isinstance(value, Deferred):
+            value = value()
+            self.__dict__[key] = value
+        return value
+
+    def __getattr__(self, name: 'str') -> 'Any':
+        # NOTE: reached only for names absent from ``__dict__``, which ``packet``
+        # always is -- see ``__additional__`` above. Everything else has to raise,
+        # or a typo would silently answer with a parsed payload.
+        if name != 'packet':
+            raise AttributeError(f'{type(self).__name__!r} object has no attribute {name!r}')
+        return self.__analyse__()
+
+    def __getitem__(self, name: 'str') -> 'Any':
+        if name == 'packet':
+            return self.__analyse__()
+        return super().__getitem__(name)
+
+    def __contains__(self, name: 'object') -> 'bool':
+        # NOTE: ``Mapping.__contains__`` answers by fetching the value, which
+        # would run the deferred analysis merely to decide that the field exists.
+        # ``packet`` is a declared field, so it is always there.
+        return name == 'packet' or super().__contains__(name)
+
+    def __str__(self) -> 'str':
+        self.__analyse__()
+        return super().__str__()
+
+    def __repr__(self) -> 'str':
+        self.__analyse__()
+        return super().__repr__()
+
+    def to_dict(self) -> 'dict[str, Any]':
+        """Convert :class:`Datagram` into :obj:`dict`.
+
+        Returns:
+            The datagram's fields, with ``packet`` analysed if it had not been
+            read yet -- a :obj:`dict` holding a :class:`Deferred` would leak an
+            implementation detail into what is meant to be plain data.
+
+        """
+        self.__analyse__()
+        return super().to_dict()
+
+
+@info_final
+class ReassemblyData(Info):
+    """Data storage for reassembly."""
+
+    #: IPv4 reassembled data.
+    ipv4: 'tuple[IP_Datagram, ...]'
+    #: IPv6 reassembled data.
+    ipv6: 'tuple[IP_Datagram, ...]'
+    #: TCP reassembled data.
+    tcp: 'tuple[TCP_Datagram, ...]'
+
+    if TYPE_CHECKING:
+        def __init__(self, ipv4: 'Optional[tuple[IP_Datagram, ...]]', ipv6: 'Optional[tuple[IP_Datagram, ...]]', tcp: 'Optional[tuple[TCP_Datagram, ...]]') -> 'None': ...  # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long