From 5aebe4888d8d20f367264b136f182c695039a31d Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 17 Sep 2026 11:54:52 -0400 Subject: [PATCH 1/8] foundation: give IP reassembly the RFC timeout and trace TCP flows bidirectionally Both items are asked for on the Help Wanted page (docs/source/pep.rst, from #419) and in discussion #106. Reassembly timeout - The clock is the capture's own timestamps, never the host's: an offline parser has no other notion of time passing, and keying on time.time() would make the same file reassemble differently on every run. Packet and Buffer models gained a `timestamp`, and Reassembly.expire() abandons a buffer whose first-arriving fragment is older than `timeout` seconds, swept when a packet is handed over -- the only evidence capture time has advanced. - 60s for IPv4 and IPv6. RFC 1122 s3.3.2 supersedes RFC 791's TTL-derived timer ("SHOULD be a fixed value, not set from the remaining TTL... between 60 and 120 seconds") and RFC 8200 s4.5 mandates 60; RFC 791's 15s is an initial lower bound that MAX(TIMER,TTL) raises, not a deadline. TCP gets no timeout: no specification gives stream reassembly one, and an idle connection is ordinary. - Datagram.completed widens from bool to Completion -- COMPLETE, PARTIAL, TIMEOUT -- so an abandoned datagram is distinguishable from one that was merely unfinished. Only COMPLETE is truthy, so `if datagram.completed` is unchanged. - Fixes two cases where strict=False reported an incomplete datagram as *complete*: TCP passed off zero-filled holes as received data, and IP sliced datagram[:TDL] with TDL still -1, handing back 65534 octets of preallocated buffer. Both now report PARTIAL; the payload shape is unchanged, so follow_tcp_stream still reconstructs what it did. Bidirectional flow tracing - TCP.make_bufid() orders the two endpoints canonically, so both halves of a conversation share one buffer, label and output file. Index gained forward and reverse so per-direction ordering stays recoverable. A flow closes once *both* halves have FINed, since a connection is not over while one direction is still sending. - Default; bidirectional=False (trace_bidirectional= on Extractor, extract() and follow_tcp_stream) restores per-direction flows and reproduces main exactly. - Also fixes the Scapy adapter reporting time.time() as a flow timestamp, which put the moment of parsing into every label. Verified against a git archive of the branch point: all 15 sample captures give byte-identical tree, json and reassembly output with ip/tcp/reassembly enabled, and no capture hits a timeout (1479 datagrams, all COMPLETE). Flow tracing goes 355 -> 234 flows over the corpus with traced frames unchanged at 1222, the drop equalling the 121 two-way conversations exactly. make_samples.py regenerates byte-identically. Suite 893 passed / 17 skipped; mypy 127 errors against 128 on the branch point. --- .../pcapkit/foundation/reassembly/ip/ipv4.rst | 22 +- .../pcapkit/foundation/reassembly/ip/ipv6.rst | 6 +- .../foundation/reassembly/reassembly.rst | 5 + .../pcapkit/foundation/reassembly/tcp.rst | 19 +- .../pcapkit/foundation/traceflow/tcp.rst | 33 ++- .../foundation/traceflow/traceflow.rst | 2 + docs/source/pep.rst | 79 ++++- pcapkit/foundation/engines/dpkt.py | 6 +- pcapkit/foundation/extraction.py | 24 +- .../foundation/reassembly/data/__init__.py | 5 +- pcapkit/foundation/reassembly/data/data.py | 55 +++- pcapkit/foundation/reassembly/data/ip.py | 47 ++- pcapkit/foundation/reassembly/data/tcp.py | 39 ++- pcapkit/foundation/reassembly/ip.py | 44 ++- pcapkit/foundation/reassembly/ipv4.py | 28 ++ pcapkit/foundation/reassembly/ipv6.py | 21 ++ pcapkit/foundation/reassembly/reassembly.py | 124 +++++++- pcapkit/foundation/reassembly/tcp.py | 57 +++- pcapkit/foundation/traceflow/data/tcp.py | 56 +++- pcapkit/foundation/traceflow/tcp.py | 120 +++++++- pcapkit/foundation/traceflow/traceflow.py | 16 +- pcapkit/interface/core.py | 27 +- pcapkit/interface/misc.py | 28 +- pcapkit/toolkit/dpkt.py | 19 +- pcapkit/toolkit/pcap.py | 3 + pcapkit/toolkit/pcapng.py | 3 + pcapkit/toolkit/pypcapfile.py | 2 + pcapkit/toolkit/scapy.py | 12 +- .../foundation/reassembly/data/test_models.py | 29 +- tests/foundation/reassembly/test_ip.py | 8 +- tests/foundation/reassembly/test_ipv6.py | 2 +- tests/foundation/reassembly/test_tcp.py | 36 ++- tests/foundation/reassembly/test_timeout.py | 272 ++++++++++++++++++ .../foundation/traceflow/data/test_models.py | 12 +- tests/foundation/traceflow/test_tcp.py | 104 ++++++- .../test_reassembly_engine_parity.py | 6 +- .../integration/test_traceflow_end_to_end.py | 43 ++- tests/interface/test_core.py | 7 +- tests/interface/test_misc.py | 55 +++- tests/toolkit/test_dpkt_unit.py | 57 ++-- tests/toolkit/test_scapy_unit.py | 7 +- 41 files changed, 1351 insertions(+), 189 deletions(-) create mode 100644 tests/foundation/reassembly/test_timeout.py diff --git a/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst b/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst index 8b262531c5..14ae11ebb6 100644 --- a/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst +++ b/docs/source/pcapkit/foundation/reassembly/ip/ipv4.rst @@ -44,6 +44,8 @@ Terminology header = ipv4.packet.header, # raw bytes type header payload = bytearray( ipv4.packet.payload), # raw bytearray type payload + timestamp = float( + frame.info.time_epoch), # capture timestamp ) reasm.ipv4.datagram @@ -55,7 +57,7 @@ Terminology (tuple) datagram |--> (Info) data - | |--> 'completed' : (bool) True --> implemented + | |--> 'completed' : (Completion) COMPLETE --> reassembled in whole | |--> 'id' : (Info) original packet identifier | | |--> 'src' --> (IPv4Address) ipv4.src | | |--> 'dst' --> (IPv4Address) ipv4.dst @@ -67,7 +69,7 @@ Terminology | |--> 'payload' : (bytes) reassembled IPv4 payload | |--> 'packet' : (Protocol) parsed reassembled payload |--> (Info) data - | |--> 'completed' : (bool) False --> not implemented + | |--> 'completed' : (Completion) PARTIAL or TIMEOUT --> incomplete | |--> 'id' : (Info) original packet identifier | | |--> 'src' --> (IPv4Address) ipv4.src | | |--> 'dst' --> (IPv4Address) ipv4.dst @@ -115,4 +117,20 @@ Terminology | | |--> (int) packet range number | |--> 'header' : (bytes) header buffer | |--> 'datagram' : (bytearray) data buffer, holes set to b'\\x00' + | |--> 'timestamp' : (float) capture timestamp of the + | first-arriving fragment |--> (tuple) BUFID ... + + .. note:: + + A buffer is abandoned once the reassembly timeout elapses on the + *capture's* clock -- 60 seconds by default, per + :rfc:`1122#section-3.3.2` for IPv4 and :rfc:`8200#section-4.5` for + IPv6, counted from the first-arriving fragment. Its datagram is + reported with ``completed`` set to + :attr:`Completion.TIMEOUT ` + rather than + :attr:`~pcapkit.foundation.reassembly.data.data.Completion.PARTIAL`, + which is what tells "these fragments are gone" apart from "these + fragments had not arrived yet". See + :meth:`~pcapkit.foundation.reassembly.reassembly.ReassemblyBase.expire`. diff --git a/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst b/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst index bca430df5a..9f3bd73798 100644 --- a/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst +++ b/docs/source/pcapkit/foundation/reassembly/ip/ipv6.rst @@ -47,6 +47,8 @@ Terminology header = ipv6_info.fragment .header[:hdr_len], # raw bytes type header before IPv6-Frag payload = payload, # raw bytearray type payload after IPv6-Frag + timestamp = float( + frame.info.time_epoch), # capture timestamp ) .. note:: @@ -77,7 +79,7 @@ Terminology (tuple) datagram |--> (Info) data - | |--> 'completed' : (bool) True --> implemented + | |--> 'completed' : (Completion) COMPLETE --> reassembled in whole | |--> 'id' : (Info) original packet identifier | | |--> 'src' --> (IPv6Address) ipv6.src | | |--> 'dst' --> (IPv6Address) ipv6.dst @@ -89,7 +91,7 @@ Terminology | |--> 'payload' : (bytes) reassembled IPv6 payload | |--> 'packet' : (Protocol) parsed reassembled payload |--> (Info) data - | |--> 'completed' : (bool) False --> not implemented + | |--> 'completed' : (Completion) PARTIAL or TIMEOUT --> incomplete | |--> 'id' : (Info) original packet identifier | | |--> 'src' --> (IPv6Address) ipv6.src | | |--> 'dst' --> (IPv6Address) ipv6.dst diff --git a/docs/source/pcapkit/foundation/reassembly/reassembly.rst b/docs/source/pcapkit/foundation/reassembly/reassembly.rst index c9785533e4..b8e3d9c5a9 100644 --- a/docs/source/pcapkit/foundation/reassembly/reassembly.rst +++ b/docs/source/pcapkit/foundation/reassembly/reassembly.rst @@ -41,9 +41,11 @@ implements datagram reassembly of IP and TCP packets. .. autoproperty:: count .. autoproperty:: datagram + .. autoproperty:: timeout .. automethod:: reassembly .. automethod:: submit + .. automethod:: expire .. automethod:: fetch .. automethod:: index .. automethod:: run @@ -58,6 +60,8 @@ implements datagram reassembly of IP and TCP packets. :no-value: .. autoattribute:: _flag_n :no-value: + .. autoattribute:: _timeout + :no-value: .. autoattribute:: _buffer :no-value: @@ -69,6 +73,7 @@ implements datagram reassembly of IP and TCP packets. .. autoattribute:: __protocol_name__ .. autoattribute:: __protocol_type__ + .. autoattribute:: __timeout__ Internal Definitions -------------------- diff --git a/docs/source/pcapkit/foundation/reassembly/tcp.rst b/docs/source/pcapkit/foundation/reassembly/tcp.rst index a61f8e19d1..1e49453429 100644 --- a/docs/source/pcapkit/foundation/reassembly/tcp.rst +++ b/docs/source/pcapkit/foundation/reassembly/tcp.rst @@ -175,6 +175,8 @@ Terminology # last sequence number of payload header = tcp.packet.header, # raw bytes type header payload = tcp.raw, # raw bytearray type payload + timestamp = float( + frame.time_epoch), # capture timestamp ) Both ``first`` and ``last`` are absolute TCP sequence numbers and @@ -190,7 +192,7 @@ Terminology (tuple) datagram |--> (Info) data - | |--> 'completed' : (bool) True --> implemented + | |--> 'completed' : (Completion) COMPLETE --> reassembled in whole | |--> 'id' : (Info) original packet identifier | | |--> 'src' --> (tuple) | | | |--> (IPv4Address) ip.src @@ -206,7 +208,7 @@ Terminology | |--> 'payload' : (bytes) reassembled payload | |--> 'packet' : (Protocol) parsed reassembled payload |--> (Info) data - | |--> 'completed' : (bool) False --> not implemented + | |--> 'completed' : (Completion) PARTIAL or TIMEOUT --> incomplete | |--> 'id' : (Info) original packet identifier | | |--> 'src' --> (tuple) | | | |--> (IPv4Address) ip.src @@ -256,8 +258,21 @@ Terminology | | holes set to b'\x00' | |--> (int) ACK ... | |--> ... + | |--> 'timestamp' : (float) capture timestamp of the + | first segment buffered |--> (tuple) BUFID ... + .. note:: + + TCP reassembly has **no** timeout by default: no specification gives + stream reassembly a deadline the way :rfc:`1122#section-3.3.2` and + :rfc:`8200#section-4.5` give IP fragmentation one, and an idle + connection is ordinary rather than pathological. ``timestamp`` is + recorded regardless, so passing ``timeout`` to + :class:`~pcapkit.foundation.reassembly.tcp.TCP` enables the same + eviction the IP reassemblers use -- see + :attr:`TCP.__timeout__ `. + The hole descriptor list is kept in **absolute TCP sequence numbers**, once per ``BUFID``, whereas each ACK's payload buffer is indexed from its own ``isn`` -- ``raw[n]`` holds the octet with sequence number diff --git a/docs/source/pcapkit/foundation/traceflow/tcp.rst b/docs/source/pcapkit/foundation/traceflow/tcp.rst index 5683d9d40b..85973e1eb6 100644 --- a/docs/source/pcapkit/foundation/traceflow/tcp.rst +++ b/docs/source/pcapkit/foundation/traceflow/tcp.rst @@ -14,6 +14,7 @@ TCP flows from a series of packets and connections. .. autoproperty:: protocol .. automethod:: dump + .. automethod:: make_bufid .. automethod:: trace .. automethod:: submit @@ -61,11 +62,24 @@ Terminology | |--> ip.dst | | |--> tcp.dstport | | |--> 'fpout' : (dictdumper.dumper.Dumper) output dumper object - | |--> 'index': (list) list of frame index + | |--> 'index': (list) list of frame index, both directions | | |--> (int) frame index - | |--> 'label': (str) flow label generated from ``BUFID`` + | |--> 'label': (str) flow label generated from the packet + | | that opened the flow + | |--> 'origin': (tuple) (address, port) of the endpoint + | | that opened the flow + | |--> 'forward': (list) frame index sent by ``origin`` + | |--> 'reverse': (list) frame index sent to ``origin`` + | |--> 'fin': (set) endpoints seen to have sent a FIN |--> (tuple) BUFID ... + When tracing bidirectionally -- the default -- ``BUFID`` orders the two + endpoints canonically rather than as (source, destination), so both halves + of a conversation reduce to the same key. It stays a plain :obj:`tuple` + either way, because it is a :obj:`dict` key and an + :class:`~pcapkit.corekit.infoclass.Info` cannot be one -- + :class:`collections.abc.Mapping` sets its ``__hash__`` to :data:`None`. + .. seealso:: :class:`pcapkit.foundation.traceflow.data.tcp.Buffer` trace.tcp.index @@ -78,11 +92,22 @@ Terminology (tuple) index |--> (Info) data | |--> 'fpout' : (Optional[str]) output filename if exists - | |--> 'index': (tuple) tuple of frame index + | |--> 'index': (tuple) tuple of frame index, both directions, + | | in capture order | | |--> (int) frame index - | |--> 'label': (str) flow label generated from ``BUFID`` + | |--> 'label': (str) flow label generated from the packet that + | | opened the flow + | |--> 'forward': (tuple) frame index in the direction that + | | opened the flow + | |--> 'reverse': (tuple) frame index the other way; empty when + | tracing unidirectionally |--> (Info) data ... + ``forward`` and ``reverse`` partition ``index``, so + ``frame_number in flow.forward`` answers which way a packet went without + taking the label apart. ``forward`` is the direction of the packet that + opened the flow, whose endpoints the label names first. + .. seealso:: :class:`pcapkit.foundation.traceflow.data.tcp.Index` Data Structures diff --git a/docs/source/pcapkit/foundation/traceflow/traceflow.rst b/docs/source/pcapkit/foundation/traceflow/traceflow.rst index 568281a4d5..578c7652bd 100644 --- a/docs/source/pcapkit/foundation/traceflow/traceflow.rst +++ b/docs/source/pcapkit/foundation/traceflow/traceflow.rst @@ -55,6 +55,8 @@ which is an abstract base class for all flow tracing classes. :no-value: .. autoattribute:: _stream :no-value: + .. autoattribute:: _bidir + :no-value: .. automethod:: __call__ .. automethod:: __init_subclass__ diff --git a/docs/source/pep.rst b/docs/source/pep.rst index 8ae5400eaf..b14bc4d20e 100644 --- a/docs/source/pep.rst +++ b/docs/source/pep.rst @@ -595,15 +595,76 @@ Two smaller items in the same subsystem: * **Flow tracing is TCP only**, and blocked on the same generalisation -- :class:`~pcapkit.foundation.traceflow.TraceFlowManager` holds a single field, so UDP, SCTP and IP conversation tracing have nowhere to go. The TCP tracer - itself closes a flow on FIN but never on RST, which is not in its packet model - at all, and treats each direction of a connection as a separate flow. -* **Nothing ever times a partial datagram out.** :rfc:`791` gives IP reassembly - a 15-second timer and :rfc:`8200` gives IPv6 60 seconds; neither is - implemented, and neither can be until the buffer models carry a timestamp. A - buffer is released only when its datagram completes or its flow is torn down, - and every in-flight IP datagram identifier holds a fixed 72 KiB of - preallocated space -- so a lossy capture, or one with spoofed identifiers, - grows the buffer monotonically. + also closes a flow on FIN but never on RST, which is not in + :class:`~pcapkit.foundation.traceflow.data.tcp.Packet` at all -- every toolkit + adapter would have to report the flag before the tracer could act on it. + + It no longer *treats each direction of a connection as a separate flow*, + though. :meth:`TCP.make_bufid + ` orders the two endpoints + canonically, so both halves of a conversation reduce to one buffer ID, one + label and one output file, and + :class:`~pcapkit.foundation.traceflow.data.tcp.Index` reports ``forward`` and + ``reverse`` alongside ``index`` so per-direction ordering stays recoverable. + A flow now closes only once **both** halves have sent a FIN, since a + connection is not over while one direction is still sending. This is the + default; ``bidirectional=False`` (``trace_bidirectional=False`` on + :class:`~pcapkit.foundation.extraction.Extractor`, + :func:`~pcapkit.interface.core.extract` and + :func:`~pcapkit.interface.misc.follow_tcp_stream`) restores the older + per-direction behaviour. + + What is still wanted here is **wiring the application layer into flow + tracing**. Reassembly analyses a datagram's payload lazily through + :class:`~pcapkit.foundation.reassembly.data.data.Deferred`; flow tracing + analyses nothing, because it buffers no payload at all -- its + :class:`~pcapkit.foundation.traceflow.data.tcp.Buffer` holds a dumper, frame + indices and a label. So this is not a parse to postpone but a capability to + add, and it needs a decision first: whether the tracer grows a payload buffer + per direction, or delegates to + :class:`~pcapkit.foundation.reassembly.tcp.TCP` the way + :func:`~pcapkit.interface.misc.follow_tcp_stream` already does. +* **Timing a partial datagram out** is implemented, for IP. + :meth:`Reassembly.expire + ` abandons a + buffer whose first-arriving fragment is older than + :attr:`~pcapkit.foundation.reassembly.reassembly.ReassemblyBase.timeout` + seconds, and the buffer models carry the timestamp that makes it possible. + + The clock is the **capture's own timestamps**, not the host's: an offline + parser has no other notion of time passing, and keying on + :func:`time.time` would make the same file reassemble differently on every + run. A fragment being handed over is the only evidence capture time has + advanced, so that is when the sweep happens -- which means the clock advances + only while the reassembler is being fed. For IPv4 and TCP that is nearly every + frame of the protocol; for IPv6 it is only the fragments, so an IPv6 buffer + that stalls with no further fragment behind it is reported as + :attr:`~pcapkit.foundation.reassembly.data.data.Completion.PARTIAL` rather + than + :attr:`~pcapkit.foundation.reassembly.data.data.Completion.TIMEOUT`. That is + the honest answer, since the capture never shows the deadline passing; feeding + every frame's timestamp to every enabled reassembler would close the gap and + is worth doing on its own account. + + On the numbers: the 15 seconds this page used to attribute to :rfc:`791` is + that RFC's *initial* timer setting, a lower bound which + ``TIMER <- MAX(TIMER,TTL)`` then raises toward the 4.25-minute TTL ceiling -- + not a deadline. :rfc:`1122#section-3.3.2` supersedes the scheme outright + ("The reassembly timeout value SHOULD be a fixed value, not set from the + remaining TTL... between 60 seconds and 120 seconds"), so IPv4 uses 60 + seconds, agreeing with the 60 :rfc:`8200#section-4.5` mandates for IPv6. + **TCP is left with no timeout at all**, since no specification gives stream + reassembly a deadline and an idle connection is ordinary rather than + pathological; ``timeout`` enables one on request. + + Two related things are *not* done. :rfc:`8200#section-4.5` and + :rfc:`1122#section-3.3.2` both want an ICMP Time Exceeded sent on expiry when + the offset-zero fragment has been received; a parser sends nothing, so the + condition is only reported in the log record. And the memory motive is + narrowed rather than removed -- an in-flight IP datagram identifier still + holds a fixed 72 KiB of preallocated space, but now for at most the timeout's + worth of capture time rather than for the life of the + :class:`~pcapkit.foundation.extraction.Extractor`. Reassembly is also unavailable on some engines rather than merely slower, which is worth knowing before benchmarking against them: ``pyshark``, ``pypcap`` and diff --git a/pcapkit/foundation/engines/dpkt.py b/pcapkit/foundation/engines/dpkt.py index 08d6d2c230..19387b0a71 100644 --- a/pcapkit/foundation/engines/dpkt.py +++ b/pcapkit/foundation/engines/dpkt.py @@ -175,15 +175,15 @@ def read_frame(self) -> 'DPKTPacket': # record fragments if ext._flag_r: if ext._ipv4: - data_ipv4 = ipv4_reassembly(packet, count=ext._frnum) + data_ipv4 = ipv4_reassembly(packet, timestamp, count=ext._frnum) if data_ipv4 is not None: ext._reasm.ipv4(data_ipv4) if ext._ipv6: - data_ipv6 = ipv6_reassembly(packet, count=ext._frnum) + data_ipv6 = ipv6_reassembly(packet, timestamp, count=ext._frnum) if data_ipv6 is not None: ext._reasm.ipv6(data_ipv6) if ext._tcp: - data_tcp = tcp_reassembly(packet, count=ext._frnum) + data_tcp = tcp_reassembly(packet, timestamp, count=ext._frnum) if data_tcp is not None: ext._reasm.tcp(data_tcp) diff --git a/pcapkit/foundation/extraction.py b/pcapkit/foundation/extraction.py index ca41d7c79a..f5ec9b289f 100644 --- a/pcapkit/foundation/extraction.py +++ b/pcapkit/foundation/extraction.py @@ -710,8 +710,10 @@ def __init__(self, files: 'bool' = False, nofile: 'bool' = False, verbose: 'bool | VerboseHandler' = False, # output settings # pylint: disable=line-too-long engine: 'Optional[Engines]' = None, layer: 'Optional[Layers]' = None, protocol: 'Optional[Protocols]' = None, # extraction settings # pylint: disable=line-too-long reassembly: 'bool' = False, reasm_strict: 'bool' = True, reasm_store: 'bool' = True, # reassembly settings # pylint: disable=line-too-long + reasm_timeout: 'Optional[float]' = None, # reassembly settings # pylint: disable=line-too-long trace: 'bool' = False, trace_fout: 'Optional[str]' = None, trace_format: 'Optional[Formats]' = None, # trace settings # pylint: disable=line-too-long trace_byteorder: 'Literal["big", "little"]' = sys.byteorder, trace_nanosecond: 'bool' = False, # trace settings # pylint: disable=line-too-long + trace_bidirectional: 'bool' = True, # trace settings # pylint: disable=line-too-long ip: 'bool' = False, ipv4: 'bool' = False, ipv6: 'bool' = False, tcp: 'bool' = False, # reassembly/trace settings # pylint: disable=line-too-long buffer_size: 'int' = io.DEFAULT_BUFFER_SIZE, buffer_save: 'bool' = False, buffer_path: 'Optional[str]' = None, # buffer settings # pylint: disable=line-too-long no_eof: 'bool' = False, # EOF settings # pylint: disable=line-too-long @@ -741,12 +743,22 @@ def __init__(self, reassembly: if perform reassembly reasm_strict: if set strict flag for reassembly reasm_store: if store reassembled datagrams + reasm_timeout: reassembly timeout in seconds, measured on the + *capture's* own clock rather than the host's, since an offline + parser has no other notion of time passing; :data:`None` selects + each protocol's own default -- 60 seconds for IPv4 + (:rfc:`1122#section-3.3.2`) and IPv6 (:rfc:`8200#section-4.5`), + disabled for TCP, which no specification gives a deadline. Pass + :data:`math.inf` to disable it everywhere trace: if trace TCP traffic flows trace_fout: path name for flow tracer if necessary trace_format: output file format of flow tracer trace_byteorder: output file byte order trace_nanosecond: output nanosecond-resolution file flag + trace_bidirectional: whether both halves of a conversation are + traced as one flow, which is the default; :data:`False` restores + the older behaviour of a flow per direction ip: if record data for IPv4 & IPv6 reassembly (must be used with ``reassembly=True``) ipv4: if perform IPv4 reassembly (must be used with ``reassembly=True``) @@ -841,7 +853,8 @@ def __init__(self, if isinstance(reasm_cls_ipv4, ModuleDescriptor): reasm_cls_ipv4 = reasm_cls_ipv4.klass self.__reassembly__['ipv4'] = reasm_cls_ipv4 # update mapping upon import - reasm_obj_ipv4 = cast('IPv4_Reassembly', reasm_cls_ipv4(strict=reasm_strict, store=reasm_store)) + reasm_obj_ipv4 = cast('IPv4_Reassembly', reasm_cls_ipv4(strict=reasm_strict, store=reasm_store, + timeout=reasm_timeout)) if self._ipv6: logger.debug('IPv6 reassembly enabled') @@ -849,7 +862,8 @@ def __init__(self, if isinstance(reasm_cls_ipv6, ModuleDescriptor): reasm_cls_ipv6 = reasm_cls_ipv6.klass self.__reassembly__['ipv6'] = reasm_cls_ipv6 # update mapping upon import - reasm_obj_ipv6 = cast('IPv6_Reassembly', reasm_cls_ipv6(strict=reasm_strict, store=reasm_store)) + reasm_obj_ipv6 = cast('IPv6_Reassembly', reasm_cls_ipv6(strict=reasm_strict, store=reasm_store, + timeout=reasm_timeout)) if self._tcp: logger.debug('TCP reassembly enabled') @@ -857,7 +871,8 @@ def __init__(self, if isinstance(reasm_cls_tcp, ModuleDescriptor): reasm_cls_tcp = reasm_cls_tcp.klass self.__reassembly__['tcp'] = reasm_cls_tcp # update mapping upon import - reasm_obj_tcp = cast('TCP_Reassembly', reasm_cls_tcp(strict=reasm_strict, store=reasm_store)) + reasm_obj_tcp = cast('TCP_Reassembly', reasm_cls_tcp(strict=reasm_strict, store=reasm_store, + timeout=reasm_timeout)) self._reasm = ReassemblyManager( ipv4=reasm_obj_ipv4, @@ -903,7 +918,8 @@ def __init__(self, trace_cls_tcp = trace_cls_tcp.klass self.__traceflow__['tcp'] = trace_cls_tcp # update mapping upon import trace_obj_tcp = cast('TCP_TraceFlow', trace_cls_tcp(fout=trace_fout, format=trace_format, - byteorder=trace_byteorder, nanosecond=trace_nanosecond)) + byteorder=trace_byteorder, nanosecond=trace_nanosecond, + bidirectional=trace_bidirectional)) self._trace = TraceFlowManager( tcp=trace_obj_tcp, diff --git a/pcapkit/foundation/reassembly/data/__init__.py b/pcapkit/foundation/reassembly/data/__init__.py index 6737e6df40..ea4182de67 100644 --- a/pcapkit/foundation/reassembly/data/__init__.py +++ b/pcapkit/foundation/reassembly/data/__init__.py @@ -2,7 +2,8 @@ """data models for reassembly""" # shared -from pcapkit.foundation.reassembly.data.data import Deferred, DeferredPacket, ReassemblyData +from pcapkit.foundation.reassembly.data.data import (Completion, Deferred, DeferredPacket, + ReassemblyData) # IP reassembly from pcapkit.foundation.reassembly.data.ip import Buffer as IP_Buffer @@ -21,7 +22,7 @@ from pcapkit.foundation.reassembly.data.tcp import Packet as TCP_Packet __all__ = [ - 'ReassemblyData', 'Deferred', 'DeferredPacket', + 'ReassemblyData', 'Completion', 'Deferred', 'DeferredPacket', 'IP_Packet', 'IP_DatagramID', 'IP_Datagram', 'IP_Buffer', 'IP_BufferID', diff --git a/pcapkit/foundation/reassembly/data/data.py b/pcapkit/foundation/reassembly/data/data.py index 3abda6f4ce..e26a4a4dbd 100644 --- a/pcapkit/foundation/reassembly/data/data.py +++ b/pcapkit/foundation/reassembly/data/data.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- """shared data models for reassembly""" +import enum from typing import TYPE_CHECKING from pcapkit.corekit.infoclass import Info, info_final -__all__ = ['ReassemblyData', 'Deferred', 'DeferredPacket'] +__all__ = ['ReassemblyData', 'Completion', 'Deferred', 'DeferredPacket'] if TYPE_CHECKING: from typing import Callable, Optional @@ -18,6 +19,58 @@ from pcapkit.protocols.protocol import ProtocolBase as Protocol +class Completion(enum.Enum): + """How completely a datagram was reassembled, and why it stopped. + + This is the value of + :attr:`Datagram.completed `. + That field used to be a plain :obj:`bool`, and this enumeration is a widening + of it rather than a second channel beside it: reassembly now has *three* + outcomes to report, not two, since a buffer abandoned under the :rfc:`791` / + :rfc:`8200` reassembly timeout is a different event from one that simply had + not finished when the capture did. Telling them apart is the whole point of + having a timeout at all -- an expired datagram says "these fragments are + gone", a partial one says "these fragments had not arrived yet". + + Truthiness is preserved, so ``if datagram.completed:`` reads exactly as it + did while ``completed`` was a :obj:`bool`: :attr:`COMPLETE` is the only + truthy member. Equality against :obj:`True` and :obj:`False` is *not* + preserved -- ``datagram.completed == True`` is now :data:`False` even for a + complete datagram -- so a caller comparing against a boolean has to compare + against a member instead. + + """ + + #: Reassembled in whole: every octet of the datagram was received. + COMPLETE = 'complete' + + #: Fragments were still outstanding when the buffer was flushed -- at the end + #: of the capture, or when the session was torn down (a TCP FIN/RST, or an + #: IPv4 datagram whose identifier was reused by an unfragmented packet). + #: The missing octets may simply not have been captured. + PARTIAL = 'partial' + + #: Reassembly was **abandoned** under the reassembly timeout, i.e. the + #: capture clock advanced past the deadline of + #: :attr:`Reassembly.timeout ` + #: seconds after the first-arriving fragment while the datagram was still + #: incomplete. :rfc:`8200#section-4.5` requires the held fragments be + #: discarded, so no further fragment will ever be added to this datagram. + TIMEOUT = 'timeout' + + def __bool__(self) -> 'bool': + """Whether the datagram was reassembled in whole. + + Only :attr:`COMPLETE` is truthy; both :attr:`PARTIAL` and + :attr:`TIMEOUT` describe an incomplete datagram. + + """ + return self is Completion.COMPLETE + + def __str__(self) -> 'str': + return self.value + + class Deferred: """A postponed analysis of a reassembled payload. diff --git a/pcapkit/foundation/reassembly/data/ip.py b/pcapkit/foundation/reassembly/data/ip.py index d39395a460..349d7f4463 100644 --- a/pcapkit/foundation/reassembly/data/ip.py +++ b/pcapkit/foundation/reassembly/data/ip.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Generic, TypeVar from pcapkit.corekit.infoclass import Info, info_final -from pcapkit.foundation.reassembly.data.data import Deferred, DeferredPacket +from pcapkit.foundation.reassembly.data.data import Completion, Deferred, DeferredPacket from pcapkit.utilities.compat import Tuple __all__ = [ @@ -13,9 +13,9 @@ if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address - from typing import Any, Callable, Optional, overload + from typing import Any, Callable, Optional - from typing_extensions import Literal, TypeAlias + from typing_extensions import TypeAlias from pcapkit.const.reg.transtype import TransType from pcapkit.protocols.protocol import ProtocolBase as Protocol @@ -47,9 +47,14 @@ class Packet(Info, Generic[_AT]): header: 'bytes' #: Raw :obj:`bytearray` type payload. payload: 'bytearray' + #: Capture timestamp of the fragment, in seconds since the Unix epoch. This + #: is the *capture's* clock, not the host's: it is what drives the :rfc:`791` + #: and :rfc:`8200#section-4.5` reassembly timeout, since an offline parser + #: replaying a file has no other notion of time passing. + timestamp: 'float' if TYPE_CHECKING: - def __init__(self, bufid: 'tuple[_AT, _AT, int, TransType]', num: 'int', fo: 'int', ihl: 'int', mf: 'bool', tl: 'int', header: 'bytes', payload: 'bytearray') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, bufid: 'tuple[_AT, _AT, int, TransType]', num: 'int', fo: 'int', ihl: 'int', mf: 'bool', tl: 'int', header: 'bytes', payload: 'bytearray', timestamp: 'float') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -83,8 +88,11 @@ class Datagram(DeferredPacket, Info, Generic[_AT]): #: :meth:`to_dict` and iteration still report the field under its own name. __additional__ = ['packet'] - #: Completed flag. - completed: 'bool' + #: How completely the datagram was reassembled, and why reassembly stopped. + #: Only :attr:`Completion.COMPLETE` is truthy, so ``if datagram.completed:`` + #: still reads as it did while this was a :obj:`bool`; equality against + #: :obj:`True` or :obj:`False` no longer holds. + completed: 'Completion' #: Original packet identifier. id: 'DatagramID[_AT]' #: Packet numbers. @@ -99,13 +107,18 @@ class Datagram(DeferredPacket, Info, Generic[_AT]): packet: 'Optional[Protocol]' if TYPE_CHECKING: - @overload #pylint: disable=used-before-assignment - def __init__(self, completed: 'Literal[True]', id: 'DatagramID[_AT]', index: 'tuple[int, ...]', header: 'bytes', payload: 'bytes', packet: 'Protocol | Deferred') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin - - @overload - def __init__(self, completed: 'Literal[False]', id: 'DatagramID[_AT]', index: 'tuple[int, ...]', header: 'bytes', payload: 'tuple[bytes, ...]', packet: 'None') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin - - def __init__(self, completed: 'bool', id: 'DatagramID[_AT]', index: 'tuple[int, ...]', header: 'bytes', payload: 'bytes | tuple[bytes, ...]', packet: 'Optional[Protocol | Deferred]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + # NOTE: one signature, not a pair of ``@overload``\\ s keyed on + # ``completed``. There used to be two, correlating a complete datagram with + # a ``bytes`` payload and a parsed ``packet``, and an incomplete one with a + # tuple of fragments and ``packet=None``. That correlation does not hold: + # under ``strict=False`` an *incomplete* datagram is reported as one + # contiguous ``bytes`` with its holes zero-filled, and analysed, because + # that is the payload buffer as it stands -- which is what + # :func:`~pcapkit.interface.misc.follow_tcp_stream` reconstructs a stream + # from. Overloads keyed on a literal cannot be selected from a ``completed`` + # computed at runtime anyway, so they only made the reassemblers' own calls + # untypeable while promising a correlation the code does not keep. + def __init__(self, completed: 'Completion', id: 'DatagramID[_AT]', index: 'tuple[int, ...]', header: 'bytes', payload: 'bytes | tuple[bytes, ...]', packet: 'Optional[Protocol | Deferred]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final class Buffer(Info, Generic[_AT]): @@ -122,6 +135,12 @@ class Buffer(Info, Generic[_AT]): header: 'bytes' #: Data buffer, holes set to ``b'\x00'``. datagram: 'bytearray' + #: Capture timestamp of the **first-arriving** fragment of this datagram, in + #: seconds since the Unix epoch. This is the origin of the reassembly timer: + #: :rfc:`8200#section-4.5` counts its 60 seconds "of the reception of the + #: first-arriving fragment", so a later fragment does not extend the + #: deadline and this field is never revised once set. + timestamp: 'float' if TYPE_CHECKING: - def __init__(self, TDL: 'int', RCVBT: 'bytearray', index: 'list[int]', header: 'bytes', datagram: 'bytearray') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, TDL: 'int', RCVBT: 'bytearray', index: 'list[int]', header: 'bytes', datagram: 'bytearray', timestamp: 'float') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin diff --git a/pcapkit/foundation/reassembly/data/tcp.py b/pcapkit/foundation/reassembly/data/tcp.py index 3a0f2b95e7..907c2a77f9 100644 --- a/pcapkit/foundation/reassembly/data/tcp.py +++ b/pcapkit/foundation/reassembly/data/tcp.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Generic, TypeVar from pcapkit.corekit.infoclass import Info, info_final -from pcapkit.foundation.reassembly.data.data import Deferred, DeferredPacket +from pcapkit.foundation.reassembly.data.data import Completion, Deferred, DeferredPacket from pcapkit.utilities.compat import Tuple __all__ = [ @@ -14,9 +14,9 @@ if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address - from typing import Optional, overload + from typing import Optional - from typing_extensions import Literal, TypeAlias + from typing_extensions import TypeAlias from pcapkit.protocols.protocol import ProtocolBase as Protocol @@ -57,9 +57,14 @@ class Packet(Info): header: 'bytes' #: Raw :obj:`bytearray` type payload. payload: 'bytearray' + #: Capture timestamp of the segment, in seconds since the Unix epoch, i.e. + #: the *capture's* clock rather than the host's. It drives the reassembly + #: timeout, which for TCP is off by default -- see + #: :attr:`TCP.__timeout__ `. + timestamp: 'float' if TYPE_CHECKING: - def __init__(self, bufid: 'BufferID', dsn: 'int', ack: 'int', num: 'int', syn: 'bool', fin: 'bool', rst: 'bool', len: 'int', first: 'int', last: 'int', header: 'bytes', payload: 'bytearray') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, bufid: 'BufferID', dsn: 'int', ack: 'int', num: 'int', syn: 'bool', fin: 'bool', rst: 'bool', len: 'int', first: 'int', last: 'int', header: 'bytes', payload: 'bytearray', timestamp: 'float') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -81,8 +86,11 @@ def __init__(self, src: 'tuple[_AT, int]', dst: 'tuple[_AT, int]', ack: 'int') - class Datagram(DeferredPacket, Info, Generic[_AT]): """Data model for :term:`TCP `.""" - #: Completed flag. - completed: 'bool' + #: How completely the datagram was reassembled, and why reassembly stopped; + #: see :class:`~pcapkit.foundation.reassembly.data.data.Completion`. Only + #: :attr:`Completion.COMPLETE` is truthy, so ``if datagram.completed:`` reads + #: as it did while this was a :obj:`bool`. + completed: 'Completion' #: Listing ``packet`` here is what makes it lazy -- see #: :class:`~pcapkit.foundation.reassembly.data.data.DeferredPacket`. __additional__ = ['packet'] @@ -101,13 +109,12 @@ class Datagram(DeferredPacket, Info, Generic[_AT]): packet: 'Optional[Protocol]' if TYPE_CHECKING: - @overload # pylint: disable=used-before-assignment - def __init__(self, completed: 'Literal[True]', id: 'DatagramID[_AT]', index: 'tuple[int, ...]', header: 'bytes', payload: 'bytes', packet: 'Protocol | Deferred') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin - - @overload - def __init__(self, completed: 'Literal[False]', id: 'DatagramID[_AT]', index: 'tuple[int, ...]', header: 'bytes', payload: 'tuple[bytes, ...]', packet: 'None') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin - - def __init__(self, completed: 'bool', id: 'DatagramID[_AT]', index: 'tuple[int, ...]', header: 'bytes', payload: 'bytes | tuple[bytes, ...]', packet: 'Optional[Protocol | Deferred]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + # NOTE: one signature rather than a pair of ``@overload``\\ s keyed on + # ``completed`` -- for the reason given on + # :class:`~pcapkit.foundation.reassembly.data.ip.Datagram`, which applies + # here identically: ``strict=False`` reports an incomplete payload buffer as + # one contiguous ``bytes`` and analyses it. + def __init__(self, completed: 'Completion', id: 'DatagramID[_AT]', index: 'tuple[int, ...]', header: 'bytes', payload: 'bytes | tuple[bytes, ...]', packet: 'Optional[Protocol | Deferred]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin @info_final @@ -164,6 +171,10 @@ class Buffer(Info): hdr: 'bytes' #: ACK list. ack: 'dict[int, Fragment]' + #: Capture timestamp of the **first** segment buffered under this buffer ID, + #: in seconds since the Unix epoch. Origin of the reassembly timer, and never + #: revised: a later segment does not extend the deadline. + timestamp: 'float' if TYPE_CHECKING: - def __init__(self, hdl: 'list[HoleDescriptor]', hdr: 'bytes', ack: 'dict[int, Fragment]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin + def __init__(self, hdl: 'list[HoleDescriptor]', hdr: 'bytes', ack: 'dict[int, Fragment]', timestamp: 'float') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long,redefined-builtin diff --git a/pcapkit/foundation/reassembly/ip.py b/pcapkit/foundation/reassembly/ip.py index 1f76a20191..a188aadba6 100644 --- a/pcapkit/foundation/reassembly/ip.py +++ b/pcapkit/foundation/reassembly/ip.py @@ -16,6 +16,7 @@ """ from typing import TYPE_CHECKING, Generic +from pcapkit.foundation.reassembly.data.data import Completion from pcapkit.foundation.reassembly.data.ip import (_AT, Buffer, BufferID, Datagram, DatagramID, Deferred, Packet) from pcapkit.foundation.reassembly.reassembly import ReassemblyBase as Reassembly @@ -38,6 +39,8 @@ class IP(Reassembly[Packet[_AT], Datagram[_AT], BufferID, Buffer[_AT]], Generic[ store: if store reassembled datagram in memory, i.e., :attr:`self._dtgram ` (if not, datagram will be discarded after callback) + timeout: reassembly timeout in seconds, on the capture's own clock; + :data:`None` selects the protocol's :attr:`__timeout__` default Important: This class is not intended to be instantiated directly, @@ -87,6 +90,13 @@ def reassembly(self, info: 'Packet[_AT]') -> 'None': IHL = info.ihl # Internet Header Length MF = info.mf # More Fragments flag TL = info.tl # Total Length + TS = info.timestamp # Capture timestamp, i.e. the only clock we have + + # This fragment's arrival is the evidence that capture time has reached + # ``TS``, so it is the moment to abandon whatever the deadline has now + # passed for -- including, deliberately, buffers this fragment does not + # belong to. + self._dtgram.extend(self.expire(TS)) # when non-fragmented (possibly discarded) packet received if not FO and not MF: @@ -107,6 +117,7 @@ def reassembly(self, info: 'Packet[_AT]') -> 'None': index=[], # index record header=header, # header buffer datagram=bytearray(65535), # data buffer + timestamp=TS, # first-arriving fragment's clock reading ) else: # put header into header buffer @@ -141,13 +152,20 @@ def reassembly(self, info: 'Packet[_AT]') -> 'None': ) def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]', # type: ignore[override] # pylint: disable=arguments-differ - checked: 'bool' = False) -> 'list[Datagram[_AT]]': + checked: 'bool' = False, timeout: 'bool' = False) -> 'list[Datagram[_AT]]': """Submit reassembled payload. Arguments: buf: buffer dict of reassembled packets bufid: buffer identifier checked: buffer consistency checked flag + timeout: whether this buffer is being submitted because + :meth:`~pcapkit.foundation.reassembly.reassembly.ReassemblyBase.expire` + abandoned it under the reassembly timeout, which is what + separates + :attr:`Completion.TIMEOUT ` + from + :attr:`Completion.PARTIAL ` Returns: Reassembled packets. @@ -164,6 +182,12 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]' flag = checked or (TDL > 0 and all(RCVBT[start:stop])) ret = [] # type: list[Datagram[_AT]] + # How completely this datagram came out, and why it stopped. Derived once, + # so the two branches below cannot disagree about it. + completion = Completion.COMPLETE if flag else ( + Completion.TIMEOUT if timeout else Completion.PARTIAL + ) + # if datagram is not implemented if not flag and self._flag_s: data = [] # type: list[bytes] @@ -181,7 +205,7 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]' # strip empty packets if data or header: packet = Datagram( - completed=False, + completed=completion, id=DatagramID( src=bufid[0], dst=bufid[1], @@ -194,11 +218,21 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]' packet=None, ) ret.append(packet) - # if datagram is reassembled in whole + # if datagram is reassembled in whole -- or if it is not, and ``strict`` + # asked for one contiguous payload rather than the received runs else: - payload = bytes(datagram[:TDL]) + # NOTE: ``max(TDL, 0)``, not ``TDL``. ``TDL`` is still its initial + # ``-1`` until the fragment with **MF** clear arrives, so a datagram + # whose final fragment never came reached this branch under + # ``strict=False`` and sliced ``datagram[:-1]`` -- handing back 65534 + # octets of the preallocated buffer, almost all of them zeros the + # sender never sent, and calling it complete. The length of such a + # datagram is simply not known, so there is nothing honest to report + # but an empty payload; ``strict=True``, the default, reports the runs + # that did arrive instead. + payload = bytes(datagram[:max(TDL, 0)]) packet = Datagram( - completed=True, + completed=completion, id=DatagramID( src=bufid[0], dst=bufid[1], diff --git a/pcapkit/foundation/reassembly/ipv4.py b/pcapkit/foundation/reassembly/ipv4.py index 467d7a8b09..ad1e7cd1d3 100644 --- a/pcapkit/foundation/reassembly/ipv4.py +++ b/pcapkit/foundation/reassembly/ipv4.py @@ -28,6 +28,8 @@ class IPv4(IP): store: if store reassembled datagram in memory, i.e., :attr:`self._dtgram ` (if not, datagram will be discarded after callback) + timeout: reassembly timeout in seconds, on the capture's own clock; + :data:`None` selects :attr:`__timeout__` Example: >>> from pcapkit.foundation.reassembly import IPv4 @@ -48,3 +50,29 @@ class IPv4(IP): __protocol_name__ = 'IPv4' #: Protocol of current reassembly object. __protocol_type__ = IPv4_Protocol + + #: float: Default reassembly timeout, in seconds, on the capture's clock. + #: + #: :rfc:`1122#section-3.3.2` is the governing text and it is emphatic on both + #: halves: "There MUST be a reassembly timeout. The reassembly timeout value + #: SHOULD be a fixed value, **not set from the remaining TTL**. It is + #: recommended that the value lie between 60 seconds and 120 seconds." Its + #: DISCUSSION explains why it overrode :rfc:`791`: gateways came to treat TTL + #: as a hop count rather than elapsed seconds, so a TTL-derived deadline + #: discards datagrams that were merely slow. + #: + #: :rfc:`791#section-3.2` is therefore *not* followed to the letter. Its + #: scheme is ``TIMER <- MAX(TIMER,TTL)`` seeded from a "Timer Lower Bound", + #: for which "the current recommendation for the initial timer setting is 15 + #: seconds" -- an initial lower bound that a later fragment's TTL raises, + #: up to the 4.25-minute TTL ceiling, not a 15-second deadline. Two things + #: rule it out here: the recommendation is superseded, and TTL is not in + #: :class:`~pcapkit.foundation.reassembly.data.ip.Packet` at all, so the + #: fragment model would have to grow a field to express a scheme the current + #: requirement asks implementations not to use. + #: + #: 60 seconds is the low end of RFC 1122's range, and it makes IPv4 agree + #: with the 60 seconds :rfc:`8200#section-4.5` mandates for IPv6 -- so one + #: timestamp-driven eviction path serves both, which is the whole reason the + #: buffer models carry a timestamp. + __timeout__ = 60.0 diff --git a/pcapkit/foundation/reassembly/ipv6.py b/pcapkit/foundation/reassembly/ipv6.py index 58297e1aa2..6959461252 100644 --- a/pcapkit/foundation/reassembly/ipv6.py +++ b/pcapkit/foundation/reassembly/ipv6.py @@ -80,6 +80,8 @@ class IPv6(IP): store: if store reassembled datagram in memory, i.e., :attr:`self._dtgram ` (if not, datagram will be discarded after callback) + timeout: reassembly timeout in seconds, on the capture's own clock; + :data:`None` selects :attr:`__timeout__` Example: >>> from pcapkit.foundation.reassembly import IPv6 @@ -101,6 +103,25 @@ class IPv6(IP): #: Protocol of current reassembly object. __protocol_type__ = IPv6_Protocol + #: float: Default reassembly timeout, in seconds, on the capture's clock. + #: + #: :rfc:`8200#section-4.5` is unambiguous and mandatory: "If insufficient + #: fragments are received to complete reassembly of a packet within 60 + #: seconds of the reception of the first-arriving fragment of that packet, + #: reassembly of that packet must be abandoned and all the fragments that + #: have been received for that packet must be discarded." + #: + #: Note "first-arriving", not "first" -- the deadline is counted from + #: whichever fragment opened the buffer, not from the one at fragment offset + #: zero, which is why + #: :attr:`Buffer.timestamp ` + #: is set once and never revised. The offset-zero fragment matters only to + #: the second half of the rule, the ICMP Time Exceeded that a live stack + #: "should" send and that an offline parser cannot -- see + #: :meth:`~pcapkit.foundation.reassembly.reassembly.ReassemblyBase.expire`, + #: which reports the condition in its log record instead. + __timeout__ = 60.0 + ########################################################################## # Methods. ########################################################################## diff --git a/pcapkit/foundation/reassembly/reassembly.py b/pcapkit/foundation/reassembly/reassembly.py index 7be2ef42f8..ddb3df7c3b 100644 --- a/pcapkit/foundation/reassembly/reassembly.py +++ b/pcapkit/foundation/reassembly/reassembly.py @@ -12,11 +12,12 @@ """ import abc +import math from typing import TYPE_CHECKING, Generic, Type, TypeVar, cast from pcapkit.protocols import __proto__ as protocol_registry from pcapkit.protocols.misc.raw import Raw -from pcapkit.utilities.exceptions import UnsupportedCall +from pcapkit.utilities.exceptions import FieldValueError, UnsupportedCall from pcapkit.utilities.logging import get_logger # NB: declared above the ``TYPE_CHECKING`` block, not below it, so that @@ -88,6 +89,9 @@ class ReassemblyBase(Generic[_PT, _DT, _IT, _BT], metaclass=ReassemblyMeta): store: if store reassembled datagram in memory, i.e., :attr:`self._dtgram <_dtgram>` (if not, datagram will be discarded after callback) + timeout: reassembly timeout in seconds, measured on the + *capture's* clock; :data:`None` selects the protocol's + own :attr:`__timeout__` default Note: This class is for internal use only. For customisation, please use @@ -106,14 +110,37 @@ class ReassemblyBase(Generic[_PT, _DT, _IT, _BT], metaclass=ReassemblyMeta): _flag_s: 'bool' _flag_d: 'bool' _flag_n: 'bool' + _timeout: 'float' # Internal data storage for cached properties. __cached__: 'dict[str, Any]' + ########################################################################## + # Defaults. + ########################################################################## + + #: float: Default reassembly timeout, in seconds, for this protocol -- + #: overridden per protocol, e.g. + #: :attr:`IPv6.__timeout__ `. + #: :data:`math.inf` means "never expire", which is the base default because + #: nothing here knows what a protocol's specification asks for. + __timeout__: 'float' = math.inf + ########################################################################## # Properties. ########################################################################## + @property + def timeout(self) -> 'float': + """Reassembly timeout, in seconds, of the current reassembly object. + + A buffer whose first-arriving fragment is older than this many seconds + **on the capture's own clock** is abandoned rather than held for the life + of the object -- see :meth:`expire`. :data:`math.inf` disables expiry. + + """ + return self._timeout + @property def name(self) -> 'str': """Protocol name of current reassembly object. @@ -197,9 +224,77 @@ def submit(self, buf: '_BT', **kwargs: 'Any') -> 'list[_DT]': Arguments: buf: buffer dict of reassembled packets - **kwargs: arbitrary keyword arguments + **kwargs: arbitrary keyword arguments; implementations accept + ``timeout``, set when the buffer is being submitted because + :meth:`expire` abandoned it rather than because it completed or + the capture ended + + """ + + # abandon timed-out buffers + def expire(self, timestamp: 'float') -> 'list[_DT]': + """Abandon every buffer whose reassembly timeout has elapsed. + + Arguments: + timestamp: Current time on the capture's clock, in seconds since the + Unix epoch -- i.e. the capture timestamp of the packet just + handed to :meth:`reassembly`. + + Returns: + Datagrams of the buffers abandoned, reported with + :attr:`Completion.TIMEOUT `. + Empty when nothing expired, which is the overwhelmingly common case. + + A buffer expires when more than :attr:`timeout` seconds separate + ``timestamp`` from the capture timestamp of its **first-arriving** + fragment, which is the deadline :rfc:`8200#section-4.5` states ("within + 60 seconds of the reception of the first-arriving fragment") and which + :rfc:`815` suggests implementing by reading "the clock when each first + fragment arrives". A later fragment therefore does not extend the + deadline. + + Note: + The clock only advances when *this* reassembly object is fed, since + a packet handed to it is the only evidence an offline parser has that + capture time has moved on. For IPv4 and TCP that is nearly every + frame of the relevant protocol; for IPv6 it is only the fragments, + so an IPv6 buffer that stalls and is followed by no further IPv6 + fragment is reported as + :attr:`Completion.PARTIAL ` + at the end of the capture. That is the honest answer: the capture + never shows that the deadline passed. A caller with an outside + source of time may call this method itself to advance the clock. """ + if math.isinf(self._timeout) or not self._buffer: + return [] + + # NOTE: The deadline is compared against the buffer's own origin rather + # than an elapsed count decremented per packet, so the answer depends + # only on the timestamps in the capture file and not on how the frames + # were handed over. That is what keeps a replay deterministic. + deadline = timestamp - self._timeout + expired = [ + bufid for (bufid, buffer) in self._buffer.items() + if cast('Any', buffer).timestamp < deadline + ] + + ret = [] # type: list[_DT] + for bufid in expired: + buffer = self._buffer.pop(bufid) + + # NOTE: :rfc:`8200#section-4.5` and :rfc:`1122#section-3.3.2` both + # ask for an ICMP Time Exceeded here, gated on the offset-zero + # fragment having been received. An offline parser sends nothing, so + # the condition is reported instead: the header buffer is non-empty + # exactly when that fragment arrived. + owed_icmp = bool(getattr(buffer, 'header', None) or getattr(buffer, 'hdr', None)) + logger.debug('%s: abandoning buffer %s after %.6fs > %.6fs timeout ' + '(ICMP Time Exceeded owed: %s)', self.name, bufid, + timestamp - cast('Any', buffer).timestamp, self._timeout, owed_icmp) + + ret.extend(self.submit(buffer, bufid=bufid, timeout=True)) + return ret # fetch datagram def fetch(self) -> 'tuple[_DT, ...]': @@ -303,7 +398,8 @@ def __new__(cls, *args: 'Any', **kwargs: 'Any') -> 'Self': # pylint: disable=un return self - def __init__(self, *, strict: 'bool' = True, store: 'bool' = True) -> 'None': + def __init__(self, *, strict: 'bool' = True, store: 'bool' = True, + timeout: 'Optional[float]' = None) -> 'None': """Initialise packet reassembly. Args: @@ -312,6 +408,13 @@ def __init__(self, *, strict: 'bool' = True, store: 'bool' = True) -> 'None': store: if store reassembled datagram in memory, i.e., :attr:`self._dtgram <_dtgram>` (if not, datagram will be discarded after callback) + timeout: reassembly timeout in seconds, measured on the capture's + own clock rather than the host's; :data:`None` selects this + protocol's :attr:`__timeout__` default, and + :data:`math.inf` disables expiry entirely + + Raises: + FieldValueError: If ``timeout`` is negative. """ #: bool: Strict mode flag. If set to :data:`True`, all @@ -328,6 +431,15 @@ def __init__(self, *, strict: 'bool' = True, store: 'bool' = True) -> 'None': #: :attr:`self._dtgram <_dtgram>` will be repopulated. self._flag_n = False + if timeout is None: + timeout = self.__timeout__ + elif timeout < 0: + raise FieldValueError(f'{type(self).__name__}: reassembly timeout must not be ' + f'negative, got {timeout!r}') + #: float: Reassembly timeout in seconds, on the capture's clock. + #: :data:`math.inf` disables expiry. + self._timeout = float(timeout) + #: dict[_IT, _BT]: Dict buffer field. This field is used to #: store reassembled packets in the form of ``{bufid: buffer}``. self._buffer = {} # type: dict[_IT, _BT] @@ -335,8 +447,8 @@ def __init__(self, *, strict: 'bool' = True, store: 'bool' = True) -> 'None': #: to store reassembled datagrams. self._dtgram = [] # type: list[_DT] - logger.debug('%s reassembly initialised (strict=%s, store=%s)', - self.name, strict, store) + logger.debug('%s reassembly initialised (strict=%s, store=%s, timeout=%s)', + self.name, strict, store, self._timeout) def __call__(self, packet: '_PT') -> 'None': """Call packet reassembly. @@ -379,6 +491,8 @@ class MyProtocol(Reassembly, protocol='my_protocol'): store: if store reassembled datagram in memory, i.e., :attr:`self._dtgram <_dtgram>` (if not, datagram will be discarded after callback) + timeout: reassembly timeout in seconds, on the capture's own clock; + :data:`None` selects the protocol's :attr:`__timeout__` default """ diff --git a/pcapkit/foundation/reassembly/tcp.py b/pcapkit/foundation/reassembly/tcp.py index 0fb8434404..519e674865 100644 --- a/pcapkit/foundation/reassembly/tcp.py +++ b/pcapkit/foundation/reassembly/tcp.py @@ -9,10 +9,11 @@ which reconstructs fragmented TCP packets back to origin. """ +import math import sys from typing import TYPE_CHECKING -from pcapkit.foundation.reassembly.data.data import Deferred +from pcapkit.foundation.reassembly.data.data import Completion, Deferred from pcapkit.foundation.reassembly.data.tcp import (Buffer, BufferID, Datagram, DatagramID, Fragment, HoleDescriptor, Packet) from pcapkit.foundation.reassembly.reassembly import ReassemblyBase as Reassembly @@ -33,6 +34,9 @@ class TCP(Reassembly[Packet, Datagram, BufferID, Buffer]): store: if store reassembled datagram in memory, i.e., :attr:`self._dtgram ` (if not, datagram will be discarded after callback) + timeout: reassembly timeout in seconds, on the capture's own clock; + :data:`None` selects :attr:`__timeout__`, which for TCP disables + expiry Example: >>> from pcapkit.foundation.reassembly import TCP @@ -74,6 +78,23 @@ class TCP(Reassembly[Packet, Datagram, BufferID, Buffer]): #: Protocol of current reassembly object. __protocol_type__ = TCP_Protocol + #: float: Default reassembly timeout -- **disabled**, unlike IPv4 and IPv6. + #: + #: No specification gives TCP stream reassembly a deadline the way + #: :rfc:`1122#section-3.3.2` and :rfc:`8200#section-4.5` give IP + #: fragmentation one, and the numbers that look like candidates are not + #: reassembly timeouts: the Maximum Segment Lifetime of + #: :rfc:`9293#section-3.4.1` bounds how long a *segment* may linger in the + #: network, and the user timeout of :rfc:`9293#section-3.8.3` aborts a + #: connection whose data goes unacknowledged. Picking either as a default + #: would silently discard buffered stream data on captures that are merely + #: idle -- a long-lived connection with a two-minute lull is ordinary, while + #: a 60-second gap between fragments of one IP datagram is pathological. + #: + #: So the mechanism is available and the default is off: pass ``timeout`` to + #: ask for one, e.g. ``2 * 120`` for 2·MSL if that is the policy wanted. + __timeout__ = math.inf + ########################################################################## # Methods. ########################################################################## @@ -95,6 +116,12 @@ def reassembly(self, info: 'Packet') -> 'None': FIN = info.fin # Finish Flag (Termination) RST = info.rst # Reset Connection Flag (Termination) SYN = info.syn # Synchronise Flag (Establishment) + TS = info.timestamp # Capture timestamp, i.e. the only clock we have + + # This segment's arrival is the evidence that capture time has reached + # ``TS``. Off by default for TCP -- see ``__timeout__`` -- in which case + # this returns immediately. + self._dtgram.extend(self.expire(TS)) # Sequence number of the first octet of this segment's payload. A SYN # occupies a sequence number of its own (:rfc:`793`), so payload sent @@ -134,6 +161,7 @@ def reassembly(self, info: 'Packet') -> 'None': raw=info.payload, ), }, + timestamp=TS, ) else: # initialise buffer with ACK @@ -219,12 +247,16 @@ def reassembly(self, info: 'Packet') -> 'None': self.submit(self._buffer.pop(BUFID), bufid=BUFID) ) - def submit(self, buf: 'Buffer', *, bufid: 'BufferID') -> 'list[Datagram]': # type: ignore[override] # pylint: disable=arguments-differ + def submit(self, buf: 'Buffer', *, bufid: 'BufferID', # type: ignore[override] # pylint: disable=arguments-differ + timeout: 'bool' = False) -> 'list[Datagram]': """Submit reassembled payload. Arguments: buf: :term:`buffer ` dict of reassembled packets bufid: buffer identifier + timeout: whether this buffer is being submitted because + :meth:`~pcapkit.foundation.reassembly.reassembly.ReassemblyBase.expire` + abandoned it under the reassembly timeout Returns: Reassembled :term:`packets `. @@ -253,6 +285,12 @@ def submit(self, buf: 'Buffer', *, bufid: 'BufferID') -> 'list[Datagram]': # ty holes.append((max(start, 0), min(stop, length))) holes.sort() + # How completely this buffer came out, and why it stopped. Derived + # once per buffer, so the two branches cannot disagree about it. + completion = Completion.COMPLETE if not holes else ( + Completion.TIMEOUT if timeout else Completion.PARTIAL + ) + # if this buffer is not implemented # go through every hole and extract received payload if holes and self._flag_s: @@ -268,7 +306,7 @@ def submit(self, buf: 'Buffer', *, bufid: 'BufferID') -> 'list[Datagram]': # ty data.append(bytes(byte)) if data: # strip empty buffer packet = Datagram( - completed=False, + completed=completion, id=DatagramID( src=(bufid[0], bufid[1]), dst=(bufid[2], bufid[3]), @@ -281,13 +319,20 @@ def submit(self, buf: 'Buffer', *, bufid: 'BufferID') -> 'list[Datagram]': # ty ) datagram.append(packet) - # if this buffer is implemented - # export payload data & convert into bytes + # if this buffer is implemented -- or if it is not, and ``strict`` + # asked for one contiguous payload rather than the received runs + # + # NOTE: ``strict=False`` deliberately keeps reporting the whole + # payload buffer with its holes zero-filled, which is what + # :func:`~pcapkit.interface.misc.follow_tcp_stream` wants of a stream + # it is reconstructing best-effort. What changes is only that + # ``completed`` now says so: this branch used to report + # :attr:`Completion.COMPLETE` for a buffer it knew had holes in it. else: payload = buffer.raw if payload: # strip empty buffer packet = Datagram( - completed=True, + completed=completion, id=DatagramID( src=(bufid[0], bufid[1]), dst=(bufid[2], bufid[3]), diff --git a/pcapkit/foundation/traceflow/data/tcp.py b/pcapkit/foundation/traceflow/data/tcp.py index c5441daf1b..8aa1554356 100644 --- a/pcapkit/foundation/traceflow/data/tcp.py +++ b/pcapkit/foundation/traceflow/data/tcp.py @@ -20,7 +20,18 @@ _AT = TypeVar('_AT', 'IPv4Address', 'IPv6Address') -#: Buffer ID. +#: Buffer ID, i.e. ``(address, port, address, port)``. +#: +#: A plain :obj:`tuple` rather than an :class:`~pcapkit.corekit.infoclass.Info` +#: **deliberately**: :class:`~pcapkit.corekit.infoclass.Info` inherits +#: :class:`collections.abc.Mapping`, which sets ``__hash__ = None``, so an +#: :class:`~pcapkit.corekit.infoclass.Info` cannot be a :obj:`dict` key at all. +#: +#: When tracing bidirectionally -- the default -- the two endpoints are ordered +#: canonically rather than as (source, destination), so that both halves of one +#: conversation produce the same key; see +#: :meth:`TCP.make_bufid `. The +#: shape is unchanged either way. BufferID: 'TypeAlias' = Tuple[_AT, int, _AT, int] @@ -61,7 +72,7 @@ def __init__(self, protocol: 'Enum_LinkType', index: 'int', frame: 'Data_Frame | @info_final -class Buffer(Info): +class Buffer(Info, Generic[_AT]): """Data structure for **TCP flow tracing**. See Also: @@ -72,14 +83,33 @@ class Buffer(Info): #: Output dumper object. fpout: 'Dumper' - #: List of frame index. + #: List of frame index, **both directions**, in capture order. This is the + #: authoritative ordering; :attr:`forward` and :attr:`reverse` are + #: subsequences of it. index: 'list[int]' #: Flow label generated from ``BUFID``. label: 'str' + #: ``(address, port)`` of the endpoint whose packet opened this flow. It + #: defines what "forward" means for the flow, and it is the endpoint the + #: :attr:`label` names first. + origin: 'tuple[_AT, int]' + #: List of frame index sent **by** :attr:`origin`, in capture order. + forward: 'list[int]' + #: List of frame index sent **to** :attr:`origin`, in capture order. Always + #: empty when tracing unidirectionally, since the reverse half of the + #: conversation is then a flow of its own. + reverse: 'list[int]' + #: Endpoints observed to have sent a TCP **FIN**. A bidirectional flow is a + #: whole connection, and a connection closes only once *both* halves have + #: finished (:rfc:`9293#section-3.6`), so the set has to be tracked rather + #: than a single flag: submitting on the first FIN would cut the peer's FIN + #: and the final acknowledgement out of the flow. + fin: 'set[tuple[_AT, int]]' if TYPE_CHECKING: - def __init__(self, fpout: 'Dumper', - index: 'list[int]', label: 'str') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements + def __init__(self, fpout: 'Dumper', index: 'list[int]', label: 'str', # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long + origin: 'tuple[_AT, int]', forward: 'list[int]', reverse: 'list[int]', + fin: 'set[tuple[_AT, int]]') -> 'None': ... @info_final @@ -95,11 +125,21 @@ class Index(Info): #: Output filename if exists. fpout: 'Optional[str]' - #: Tuple of frame index. + #: Tuple of frame index, **both directions**, in capture order. index: 'tuple[int, ...]' #: Flow label generated from ``BUFID``. label: 'str' + #: Frame index of the packets travelling in the direction that opened the + #: flow, in capture order. That endpoint is the one the :attr:`label` names + #: first, so ``frame_number in index.forward`` answers "which way did this + #: packet go" without having to take the label apart. + forward: 'tuple[int, ...]' + #: Frame index of the packets travelling the other way, in capture order. + #: Empty when tracing unidirectionally, in which case + #: :attr:`forward` ``==`` :attr:`index`. + reverse: 'tuple[int, ...]' if TYPE_CHECKING: - def __init__(self, fpout: 'Optional[str]', index: 'tuple[int, ...]', - label: 'str') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements + def __init__(self, fpout: 'Optional[str]', index: 'tuple[int, ...]', # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long + label: 'str', forward: 'tuple[int, ...]', + reverse: 'tuple[int, ...]') -> 'None': ... diff --git a/pcapkit/foundation/traceflow/tcp.py b/pcapkit/foundation/traceflow/tcp.py index 75a96cde2d..00ebaab229 100644 --- a/pcapkit/foundation/traceflow/tcp.py +++ b/pcapkit/foundation/traceflow/tcp.py @@ -27,7 +27,7 @@ logger = get_logger(__name__) -class TCP(TraceFlow[BufferID, Buffer, Index, Packet[_AT]], Generic[_AT]): +class TCP(TraceFlow[BufferID, 'Buffer[_AT]', Index, Packet[_AT]], Generic[_AT]): """Trace TCP flows. Args: @@ -35,9 +35,32 @@ class TCP(TraceFlow[BufferID, Buffer, Index, Packet[_AT]], Generic[_AT]): format: output format byteorder: output file byte order nanosecond: output nanosecond-resolution file flag + bidirectional: trace both halves of a conversation as one flow *args: Arbitrary positional arguments. **kwargs: Arbitrary keyword arguments. + Note: + A TCP connection has two halves, and by default they are traced as **one + flow** -- which is what "following a TCP stream" means everywhere else, + and what this module's own title claims to do. Keying a flow on + (source, destination) instead put a client's packets and the server's + replies in separate flows, separate labels and separate output files, + leaving a caller to pair them up by inspecting the labels. + + Two consequences of the change are worth knowing: + + * The reverse half of a conversation no longer produces a flow of its + own, so a capture of *n* connections yields *n* flows rather than + ``2n``, and one output file each rather than two. + * A flow closes when **both** halves have sent a FIN rather than on the + first FIN seen, since a connection is not over while one direction is + still sending (:rfc:`9293#section-3.6`). A conversation whose reverse + half was never captured therefore stays open until + :meth:`submit` flushes it -- correctly, since nothing in the capture + shows the connection closing. + + Pass ``bidirectional=False`` for the older per-direction behaviour. + """ ########################################################################## @@ -66,6 +89,40 @@ def dump(self, packet: 'Packet[_AT]') -> 'None': # dump files output(packet.frame, name=f'Frame {packet.index}') # pylint: disable=not-callable + def make_bufid(self, packet: 'Packet[_AT]') -> 'BufferID': + """Derive the buffer ID a packet belongs to. + + Arguments: + packet: a flow packet (:term:`trace.tcp.packet`) + + Returns: + Buffer ID, i.e. ``(address, port, address, port)``. + + Tracing bidirectionally means both halves of a conversation have to land + on the *same* key, so the two endpoints are ordered canonically -- the + lesser ``(address, port)`` pair first -- rather than as (source, + destination). Sorting is what makes the key direction-independent: the + client's ``A→B`` and the server's ``B→A`` both reduce to + ``min(A, B), max(A, B)``. + + The result stays a plain :obj:`tuple` of the same shape, which is + deliberate and not merely convenient: it is a :obj:`dict` key, and an + :class:`~pcapkit.corekit.infoclass.Info` cannot be one -- inheriting + :class:`collections.abc.Mapping` sets ``__hash__`` to :data:`None`. + + Note: + Both endpoints of a connection are of the same address family, so the + comparison never has to order an :class:`~ipaddress.IPv4Address` + against an :class:`~ipaddress.IPv6Address` -- which raises + :exc:`TypeError`. + + """ + near_addr, near_port = packet.src, packet.srcport + far_addr, far_port = packet.dst, packet.dstport + if self._bidir and (far_addr, far_port) < (near_addr, near_port): + return (far_addr, far_port, near_addr, near_port) + return (near_addr, near_port, far_addr, far_port) + @overload def trace(self, packet: 'Packet[_AT]', *, output: 'Literal[True]' = ...) -> 'Dumper': ... @overload @@ -91,14 +148,22 @@ def trace(self, packet: 'Packet[_AT]', *, output: 'bool' = False) -> 'Dumper | s f'{packet.src}_{packet.srcport}-{packet.dst}_{info.dstport}-{packet.timestamp}' + It is built from the packet that **opened** the flow, not from the + canonical buffer ID, so the label still names the initiator first and + reads the way it always did. The reverse half of a bidirectional + conversation joins that flow rather than minting a label of its own. + """ # clear cache self.__cached__['submit'] = None - # Buffer Identifier - BUFID = (packet.src, packet.srcport, packet.dst, packet.dstport) # type: BufferID + # Buffer Identifier -- canonical, hence direction-independent, when + # tracing bidirectionally + BUFID = self.make_bufid(packet) # SYN = packet.syn # Synchronise Flag (Establishment) FIN = packet.fin # Finish Flag (Termination) + # the half of the conversation this packet was sent by + END = (packet.src, packet.srcport) # type: tuple[_AT, int] # # when SYN is set, reset buffer of this seesion # if SYN and BUFID in self._buffer: @@ -109,33 +174,58 @@ def trace(self, packet: 'Packet[_AT]', *, output: 'bool' = False) -> 'Dumper | s # initialise buffer with BUFID if BUFID not in self._buffer: - if packet.src.version == 4: - label = f'{packet.src}_{packet.srcport}-{packet.dst}_{packet.dstport}-{packet.timestamp}' - else: - label = f'{packet.src}_{packet.srcport}-{packet.dst}_{packet.dstport}-{packet.timestamp}'.replace(':', '.') + label = f'{packet.src}_{packet.srcport}-{packet.dst}_{packet.dstport}-{packet.timestamp}' + if packet.src.version != 4: + # ``:`` is a path separator on Windows and a drive separator + # elsewhere in the tooling, so an IPv6 label cannot carry it + label = label.replace(':', '.') logger.debug('new TCP flow %s', label) self._buffer[BUFID] = Buffer( fpout=self._foutio(fname=f'{self._fproot}/{label}{self._fdpext or ""}', protocol=packet.protocol, byteorder=self._endian, nanosecond=self._nnsecd), index=[], label=label, + origin=END, + forward=[], + reverse=[], + fin=set(), ) # trace frame record - self._buffer[BUFID].index.append(packet.index) - fpout = self._buffer[BUFID].fpout - label = self._buffer[BUFID].label - - # when FIN is set, submit buffer of this session + buffer = self._buffer[BUFID] + buffer.index.append(packet.index) + # ... and again per direction, so the merged ordering above stays + # authoritative while each half remains recoverable on its own + if END == buffer.origin: + buffer.forward.append(packet.index) + else: + buffer.reverse.append(packet.index) if FIN: + buffer.fin.add(END) + fpout = buffer.fpout + label = buffer.label + + # when the session is over, submit its buffer + # + # NOTE: A bidirectional flow is a whole connection, and a connection is + # not finished while either direction still is -- so it takes a FIN from + # both endpoints, not the first FIN seen. Closing on the first would cut + # the peer's FIN and the final acknowledgement out of the flow, and they + # would then open a *second* flow under the same buffer ID: precisely the + # split this is here to remove. + closed = len(buffer.fin) >= 2 if self._bidir else FIN + if closed: buf = self._buffer.pop(BUFID) # fpout, label = buf['fpout'], buf['label'] - logger.debug('TCP flow %s closed after %d frame(s)', label, len(buf.index)) + logger.debug('TCP flow %s closed after %d frame(s) (%d forward, %d reverse)', + label, len(buf.index), len(buf.forward), len(buf.reverse)) index = Index( fpout=f'{self._fproot}/{label}{self._fdpext}' if self._fdpext is not None else None, index=tuple(buf.index), label=label, + forward=tuple(buf.forward), + reverse=tuple(buf.reverse), ) for callback in self.__callback_fn__: callback(index) @@ -158,7 +248,9 @@ def submit(self) -> 'tuple[Index, ...]': for buf in self._buffer.values(): ret.append(Index(fpout=f"{self._fproot}/{buf.label}{self._fdpext}" if self._fdpext else None, index=tuple(buf.index), - label=buf.label,)) + label=buf.label, + forward=tuple(buf.forward), + reverse=tuple(buf.reverse),)) ret.extend(self._stream) ret_submit = tuple(ret) diff --git a/pcapkit/foundation/traceflow/traceflow.py b/pcapkit/foundation/traceflow/traceflow.py index fb4a868094..384f5a060d 100644 --- a/pcapkit/foundation/traceflow/traceflow.py +++ b/pcapkit/foundation/traceflow/traceflow.py @@ -90,6 +90,7 @@ class TraceFlowBase(Generic[_DT, _BT, _IT, _PT], metaclass=TraceFlowMeta): format: output format byteorder: output file byte order nanosecond: output nanosecond-resolution file flag + bidirectional: trace both halves of a conversation as one flow Note: This class is for internal use only. For customisation, please use @@ -302,7 +303,7 @@ def __new__(cls, *args: 'Any', **kwargs: 'Any') -> 'Self': # pylint: disable=un def __init__(self, fout: 'Optional[str]', format: 'Optional[str]', # pylint: disable=redefined-builtin byteorder: 'Literal["little", "big"]' = sys.byteorder, - nanosecond: bool = False) -> 'None': + nanosecond: bool = False, bidirectional: 'bool' = True) -> 'None': """Initialise instance. Arguments: @@ -310,6 +311,11 @@ def __init__(self, fout: 'Optional[str]', format: 'Optional[str]', # pylint: di format: output format byteorder: output file byte order nanosecond: output nanosecond-resolution file flag + bidirectional: whether the two halves of a conversation are one flow. + :data:`True` -- the default -- keys a flow on the *pair* of + endpoints rather than on (source, destination), so a connection + is traced as the one thing it is; pass :data:`False` for the + older per-direction behaviour. """ if fout is None: @@ -329,6 +335,10 @@ def __init__(self, fout: 'Optional[str]', format: 'Optional[str]', # pylint: di self._endian = byteorder #: bool: Output nanosecond-resolution file flag. self._nnsecd = nanosecond + #: bool: Bidirectional tracing flag. If set to :data:`True`, both halves + #: of a conversation share one buffer entry, one label and one output + #: file; otherwise each direction is a flow of its own. + self._bidir = bidirectional # dump I/O object fio, ext = self.make_fout(fout, format) @@ -338,7 +348,8 @@ def __init__(self, fout: 'Optional[str]', format: 'Optional[str]', # pylint: di self._fdpext = ext logger.debug('%s flow tracing initialised (root=%s, format=%s, byteorder=%s, ' - 'nanosecond=%s)', self.name, fout, format, byteorder, nanosecond) + 'nanosecond=%s, bidirectional=%s)', self.name, fout, format, + byteorder, nanosecond, bidirectional) def __call__(self, packet: '_PT') -> 'None': """Dump frame to output files. @@ -379,6 +390,7 @@ class MyProtocol(TraceFlow, protocol='my_protocol'): format: output format byteorder: output file byte order nanosecond: output nanosecond-resolution file flag + bidirectional: trace both halves of a conversation as one flow """ diff --git a/pcapkit/interface/core.py b/pcapkit/interface/core.py index d3272e1493..4dacd50dcd 100644 --- a/pcapkit/interface/core.py +++ b/pcapkit/interface/core.py @@ -76,8 +76,10 @@ def extract(fin: 'Optional[str | IO[bytes]]' = None, fout: 'Optional[str]' = Non engine: 'Optional[Engines]' = None, layer: 'Optional[Layers] | Type[Protocol]' = None, # extraction settings # pylint: disable=line-too-long protocol: 'Optional[Protocols]' = None, # extraction settings # pylint: disable=line-too-long reassembly: 'bool' = False, reasm_strict: 'bool' = True, reasm_store: 'bool' = True, # reassembly settings # pylint: disable=line-too-long + reasm_timeout: 'Optional[float]' = None, # reassembly settings # pylint: disable=line-too-long trace: 'bool' = False, trace_fout: 'Optional[str]' = None, trace_format: 'Optional[Formats]' = None, # trace settings # pylint: disable=line-too-long trace_byteorder: 'Literal["big", "little"]' = sys.byteorder, trace_nanosecond: 'bool' = False, # trace settings # pylint: disable=line-too-long + trace_bidirectional: 'bool' = True, # trace settings # pylint: disable=line-too-long ip: 'bool' = False, ipv4: 'bool' = False, ipv6: 'bool' = False, tcp: 'bool' = False, # reassembly/trace settings # pylint: disable=line-too-long buffer_size: 'int' = io.DEFAULT_BUFFER_SIZE, buffer_save: 'bool' = False, buffer_path: 'Optional[str]' = None, # buffer settings # pylint: disable=line-too-long no_eof: 'bool' = False, # EOF settings # pylint: disable=line-too-long @@ -107,12 +109,17 @@ def extract(fin: 'Optional[str | IO[bytes]]' = None, fout: 'Optional[str]' = Non reassembly: if perform reassembly reasm_strict: if set strict flag for reassembly reasm_store: if store reassembled datagrams + reasm_timeout: reassembly timeout in seconds, on the capture's own + clock; :data:`None` selects each protocol's default (60 seconds for + IPv4 and IPv6, disabled for TCP) trace: if trace TCP traffic flows trace_fout: path name for flow tracer if necessary trace_format: output file format of flow tracer trace_byteorder: output file byte order trace_nanosecond: output nanosecond-resolution file flag + trace_bidirectional: whether both halves of a conversation are traced as + one flow, which is the default ip: if record data for IPv4 & IPv6 reassembly (must be used with ``reassembly=True``) ipv4: if perform IPv4 reassembly (must be used with ``reassembly=True``) @@ -151,18 +158,23 @@ def extract(fin: 'Optional[str | IO[bytes]]' = None, fout: 'Optional[str]' = Non engine=engine, layer=layer, protocol=protocol, # type: ignore[arg-type] ip=ip, ipv4=ipv4, ipv6=ipv6, tcp=tcp, reassembly=reassembly, reasm_store=reasm_store, reasm_strict=reasm_strict, + reasm_timeout=reasm_timeout, trace=trace, trace_fout=trace_fout, trace_format=trace_format, trace_byteorder=trace_byteorder, trace_nanosecond=trace_nanosecond, + trace_bidirectional=trace_bidirectional, buffer_size=buffer_size, buffer_path=buffer_path, buffer_save=buffer_save, no_eof=no_eof, context=context) -def reassemble(protocol: 'str | Type[Protocol]', strict: 'bool' = False) -> 'Reassembly': +def reassemble(protocol: 'str | Type[Protocol]', strict: 'bool' = False, + timeout: 'Optional[float]' = None) -> 'Reassembly': """Reassemble fragmented datagrams. Arguments: protocol: protocol to be reassembled strict: if return all datagrams (including those not implemented) when submit + timeout: reassembly timeout in seconds, on the capture's own clock; + :data:`None` selects the protocol's own default Returns: A :class:`~pcapkit.foundation.reassembly.reassembly.Reassembly` object of corresponding protocol. @@ -175,18 +187,18 @@ def reassemble(protocol: 'str | Type[Protocol]', strict: 'bool' = False) -> 'Rea protocol = protocol.id()[0] if protocol == 'IPv4': - return IPv4_Reassembly(strict=strict) + return IPv4_Reassembly(strict=strict, timeout=timeout) if protocol == 'IPv6': - return IPv6_Reassembly(strict=strict) + return IPv6_Reassembly(strict=strict, timeout=timeout) if protocol == 'TCP': - return TCP_Reassembly(strict=strict) + return TCP_Reassembly(strict=strict, timeout=timeout) raise FormatError(f'Unsupported reassembly protocol: {protocol}') def trace(protocol: 'str | Type[Protocol]', fout: 'Optional[str]', format: 'Optional[str]', # pylint: disable=redefined-builtin byteorder: 'Literal["little", "big"]' = sys.byteorder, - nanosecond: bool = False) -> 'TraceFlow': + nanosecond: bool = False, bidirectional: 'bool' = True) -> 'TraceFlow': """Trace flows. Arguments: @@ -195,6 +207,8 @@ def trace(protocol: 'str | Type[Protocol]', fout: 'Optional[str]', format: output format byteorder: output file byte order nanosecond: output nanosecond-resolution file flag + bidirectional: whether both halves of a conversation are traced as one + flow, which is the default Returns: A :class:`~pcapkit.foundation.traceflow.traceflow.TraceFlow` object. @@ -207,5 +221,6 @@ def trace(protocol: 'str | Type[Protocol]', fout: 'Optional[str]', protocol = protocol.id()[0] if protocol == 'TCP': - return TCP_TraceFlow(fout=fout, format=format, byteorder=byteorder, nanosecond=nanosecond) + return TCP_TraceFlow(fout=fout, format=format, byteorder=byteorder, + nanosecond=nanosecond, bidirectional=bidirectional) raise FormatError(f'Unsupported flow tracing protocol: {protocol}') diff --git a/pcapkit/interface/misc.py b/pcapkit/interface/misc.py index e540fcce36..80a070b59b 100644 --- a/pcapkit/interface/misc.py +++ b/pcapkit/interface/misc.py @@ -9,6 +9,7 @@ generally provided per user's requests. """ +import functools import sys from typing import TYPE_CHECKING, cast @@ -75,7 +76,8 @@ def __init__(self, filename: 'Optional[str]', packets: 'tuple[Packet, ...]', con def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, # Extrator options extension: 'bool' = True, engine: 'Optional[Engines]' = None, fout: 'Optional[str]' = None, format: 'Optional[Formats]' = None, # TraceFlow options # pylint: disable=redefined-builtin - byteorder: 'ByteOrder' = sys.byteorder, nanosecond: 'bool' = False) -> 'tuple[Stream, ...]': + byteorder: 'ByteOrder' = sys.byteorder, nanosecond: 'bool' = False, + trace_bidirectional: 'bool' = True) -> 'tuple[Stream, ...]': """Follow TCP streams. Arguments: @@ -88,6 +90,11 @@ def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, format: output file format of flow tracer byteorder: output file byte order nanosecond: output nanosecond-resolution file flag + trace_bidirectional: whether both halves of a conversation are followed + as one stream, which is the default -- a stream then holds the + frames and the reassembled payload of *both* directions, which is + what "following a TCP stream" means elsewhere. :data:`False` + restores one stream per direction. Returns: List of extracted TCP streams. @@ -130,7 +137,8 @@ def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, store=True, files=False, nofile=True, verbose=verbose, engine=engine, layer=None, protocol=None, ip=False, ipv4=False, ipv6=False, tcp=True, reassembly=False, trace=True, trace_fout=fout, trace_format=format, - trace_byteorder=byteorder, trace_nanosecond=nanosecond) # type: ignore[var-annotated] + trace_byteorder=byteorder, trace_nanosecond=nanosecond, + trace_bidirectional=trace_bidirectional) # type: ignore[var-annotated] # NOTE: ``Extractor.engine`` returns the running engine *instance* (see # :meth:`Extractor.engine `), @@ -157,7 +165,21 @@ def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, tcp_reassembly, pass_count = cast('ReassemblyAdapter', tk_pcapng.tcp_reassembly), False elif isinstance(exeng, DPKT_Engine): from pcapkit.toolkit import dpkt as tk_dpkt # isort: skip # pylint: disable=import-outside-toplevel - tcp_reassembly = cast('ReassemblyAdapter', tk_dpkt.tcp_reassembly) + # NOTE: DPKT is the one adapter that cannot read a frame's capture + # timestamp off the frame -- DPKT hands ``(timestamp, bytes)`` back from + # its reader and keeps the two apart, and + # :class:`~pcapkit.foundation.engines.dpkt.DPKT` stores only the packet -- + # so the reassembly adapter takes it as an argument. Nothing here has one + # to give: the frames were stored during an extraction that has already + # finished. Binding zero is safe rather than merely convenient, because + # the reassembler below is constructed here and TCP reassembly has no + # timeout by default (see + # :attr:`TCP.__timeout__ `), + # so no deadline is computed from it. Enabling one for this call would + # first mean having the DPKT engine record each frame's timestamp beside + # the frame. + tcp_reassembly = cast('ReassemblyAdapter', + functools.partial(tk_dpkt.tcp_reassembly, timestamp=0.0)) elif isinstance(exeng, Scapy_Engine): from pcapkit.toolkit import scapy as tk_scapy # isort: skip # pylint: disable=import-outside-toplevel tcp_reassembly = cast('ReassemblyAdapter', tk_scapy.tcp_reassembly) diff --git a/pcapkit/toolkit/dpkt.py b/pcapkit/toolkit/dpkt.py index d0c5cfdbd1..9619066115 100644 --- a/pcapkit/toolkit/dpkt.py +++ b/pcapkit/toolkit/dpkt.py @@ -109,11 +109,15 @@ def wrapper(packet: 'Packet') -> 'dict[str, Any]': } -def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Address] | None': +def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, + count: 'int' = -1) -> 'IP_Packet[IPv4Address] | None': """Make data for IPv4 reassembly. Args: packet: DPKT packet. + timestamp: Capture timestamp of the packet, which drives the reassembly + timeout. DPKT hands it back beside the packet rather than on it, so + it has to be passed in -- as :func:`tcp_traceflow` already does. count: Packet index. If not provided, default to ``-1``. Returns: @@ -154,16 +158,20 @@ def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Ad tl=ipv4.len, # total length, header includes header=ipv4.pack()[:ihl], # raw bytes type header payload=bytearray(ipv4.pack()[ihl:]), # raw bytearray type payload + timestamp=timestamp, # capture timestamp ) return data return None -def ipv6_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv6Address] | None': +def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, + count: 'int' = -1) -> 'IP_Packet[IPv6Address] | None': """Make data for IPv6 reassembly. Args: packet: DPKT packet. + timestamp: Capture timestamp of the packet, which drives the reassembly + timeout. count: Packet index. If not provided, default to ``-1``. Returns: @@ -217,16 +225,20 @@ def ipv6_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv6Ad tl=hdr_len + len(payload), # total length, header includes header=ipv6.pack()[:hdr_len], # raw bytes type header before IPv6-Frag payload=bytearray(payload), # raw bytearray type payload after IPv6-Frag + timestamp=timestamp, # capture timestamp ) return data return None -def tcp_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'TCP_Packet | None': +def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, + count: 'int' = -1) -> 'TCP_Packet | None': """Make data for TCP reassembly. Args: packet: DPKT packet. + timestamp: Capture timestamp of the packet, which drives the reassembly + timeout. count: Packet index. If not provided, default to ``-1``. Returns: @@ -273,6 +285,7 @@ def tcp_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'TCP_Packet | None first=tcp.seq, # first sequence number of payload last=tcp.seq + raw_len - 1, # last sequence number of payload len=raw_len, # payload length, header excludes + timestamp=timestamp, # capture timestamp ) return data return None diff --git a/pcapkit/toolkit/pcap.py b/pcapkit/toolkit/pcap.py index 4e30a35677..ad2bc1c221 100644 --- a/pcapkit/toolkit/pcap.py +++ b/pcapkit/toolkit/pcap.py @@ -70,6 +70,7 @@ def ipv4_reassembly(frame: 'Frame') -> 'IP_Packet[IPv4Address] | None': tl=ipv4_info.len, # total length, header includes header=ipv4.packet.header, # raw bytes type header payload=bytearray(ipv4.packet.payload), # raw bytearray type payload + timestamp=float(frame.info.time_epoch), # capture timestamp ) return data return None @@ -133,6 +134,7 @@ def ipv6_reassembly(frame: 'Frame') -> 'IP_Packet[IPv6Address] | None': tl=hdr_len + len(payload), # total length, header includes header=ipv6_info.fragment.header[:hdr_len], # raw bytes type header before IPv6-Frag payload=payload, # raw bytearray type payload after IPv6-Frag + timestamp=float(frame.info.time_epoch), # capture timestamp ) return data return None @@ -181,6 +183,7 @@ def tcp_reassembly(frame: 'Frame') -> 'TCP_Packet | None': first=tcp_info.seq, # first sequence number of payload last=tcp_info.seq + raw_len - 1, # last sequence number of payload len=raw_len, # payload length, header excludes + timestamp=float(frame.info.time_epoch), # capture timestamp ) return data return None diff --git a/pcapkit/toolkit/pcapng.py b/pcapkit/toolkit/pcapng.py index 9050e11eb1..d49828b6c7 100644 --- a/pcapkit/toolkit/pcapng.py +++ b/pcapkit/toolkit/pcapng.py @@ -73,6 +73,7 @@ def ipv4_reassembly(frame: 'PCAPNG') -> 'IP_Packet[IPv4Address] | None': tl=ipv4_info.len, # total length, header includes header=ipv4.packet.header, # raw bytes type header payload=bytearray(ipv4.packet.payload), # raw bytearray type payload + timestamp=float(frame_info.timestamp_epoch), # capture timestamp ) return data return None @@ -137,6 +138,7 @@ def ipv6_reassembly(frame: 'PCAPNG') -> 'IP_Packet[IPv6Address] | None': tl=hdr_len + len(payload), # total length, header includes header=ipv6_info.fragment.header[:hdr_len], # raw bytes type header before IPv6-Frag payload=payload, # raw bytearray type payload after IPv6-Frag + timestamp=float(frame_info.timestamp_epoch), # capture timestamp ) return data return None @@ -187,6 +189,7 @@ def tcp_reassembly(frame: 'PCAPNG') -> 'TCP_Packet | None': first=tcp_info.seq, # first sequence number of payload last=tcp_info.seq + raw_len - 1, # last sequence number of payload len=raw_len, # payload length, header excludes + timestamp=float(frame_info.timestamp_epoch), # capture timestamp ) return data return None diff --git a/pcapkit/toolkit/pypcapfile.py b/pcapkit/toolkit/pypcapfile.py index 26a7a512dc..437a655600 100644 --- a/pcapkit/toolkit/pypcapfile.py +++ b/pcapkit/toolkit/pypcapfile.py @@ -329,6 +329,7 @@ def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Ad tl=ipv4.len, # total length, header includes header=header, # raw bytes type header payload=bytearray(ipv4.payload), # raw bytearray type payload + timestamp=packet2timestamp(packet), # capture timestamp ) @@ -403,6 +404,7 @@ def tcp_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'TCP_Packet | None first=tcp.seqnum, # this sequence number last=tcp.seqnum + len(payload), # next (wanted) sequence number len=len(payload), # payload length, header excludes + timestamp=packet2timestamp(packet), # capture timestamp ) diff --git a/pcapkit/toolkit/scapy.py b/pcapkit/toolkit/scapy.py index 02b2dbb417..e9d2f79e29 100644 --- a/pcapkit/toolkit/scapy.py +++ b/pcapkit/toolkit/scapy.py @@ -35,7 +35,6 @@ """ import ipaddress -import time from typing import TYPE_CHECKING, cast from pcapkit.const.reg.linktype import LinkType as Enum_LinkType @@ -164,6 +163,7 @@ def ipv4_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv4Ad tl=ipv4.len, # total length, header includes header=bytes(ipv4)[:ipv4.ihl * 4], # raw bytes type header payload=bytearray(bytes(ipv4.payload)), # raw bytearray type payload + timestamp=float(packet.time), # capture timestamp ) return data return None @@ -232,6 +232,7 @@ def ipv6_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'IP_Packet[IPv6Ad tl=hdr_len + len(payload), # total length, header includes header=bytes(ipv6)[:hdr_len], # raw bytes type header before IPv6-Frag payload=payload, # raw bytearray type payload after IPv6-Frag + timestamp=float(packet.time), # capture timestamp ) return data return None @@ -285,6 +286,7 @@ def tcp_reassembly(packet: 'Packet', *, count: 'int' = -1) -> 'TCP_Packet | None first=tcp.seq, # first sequence number of payload last=tcp.seq + raw_len - 1, # last sequence number of payload len=raw_len, # payload length, header excludes + timestamp=float(packet.time), # capture timestamp ) return data return None @@ -323,7 +325,13 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | No dst=ipaddress.ip_address(ip.dst), # destination IP srcport=tcp.sport, # TCP source port dstport=tcp.dport, # TCP destination port - timestamp=time.time(), # timestamp + # NOTE: the *capture's* clock, not the host's. This read + # ``time.time()``, which put the moment of parsing into every flow + # label -- so the same capture traced twice produced different label + # strings and different output filenames. Scapy carries the record's + # own timestamp on ``Packet.time``, which is what every other + # engine's adapter reports. + timestamp=float(packet.time), # capture timestamp ) return data return None diff --git a/tests/foundation/reassembly/data/test_models.py b/tests/foundation/reassembly/data/test_models.py index 8e5ea8f877..1ebca66913 100644 --- a/tests/foundation/reassembly/data/test_models.py +++ b/tests/foundation/reassembly/data/test_models.py @@ -17,29 +17,32 @@ def setUp(self) -> None: def test_ip_data_models_and_package_aliases(self) -> None: from pcapkit.const.reg.transtype import TransType - from pcapkit.foundation.reassembly.data import (IP_Buffer, IP_Datagram, IP_DatagramID, - IP_Packet, ReassemblyData) + from pcapkit.foundation.reassembly.data import (Completion, IP_Buffer, IP_Datagram, + IP_DatagramID, IP_Packet, ReassemblyData) from pcapkit.foundation.reassembly.data.ip import Buffer, Datagram, DatagramID, Packet src = ip_address('192.0.2.1') dst = ip_address('198.51.100.2') bufid = (src, dst, 123, TransType.UDP) - packet = Packet(bufid, 7, 8, 20, True, 28, b'ip-header', bytearray(b'payload')) + packet = Packet(bufid, 7, 8, 20, True, 28, b'ip-header', bytearray(b'payload'), 1000.0) self.assertIsInstance(packet, IP_Packet) self.assertEqual(packet.bufid, bufid) self.assertEqual(packet.payload, bytearray(b'payload')) + self.assertEqual(packet.timestamp, 1000.0) datagram_id = DatagramID(src, dst, 123, TransType.UDP) - datagram = Datagram(False, datagram_id, (7,), b'ip-header', (b'payload',), None) + datagram = Datagram(Completion.PARTIAL, datagram_id, (7,), b'ip-header', (b'payload',), None) self.assertIsInstance(datagram.id, IP_DatagramID) self.assertIsInstance(datagram, IP_Datagram) self.assertFalse(datagram.completed) + self.assertIs(datagram.completed, Completion.PARTIAL) self.assertEqual(datagram.to_dict()['payload'], (b'payload',)) - buffer = Buffer(-1, bytearray(b'\x01'), [7], b'ip-header', bytearray(b'payload')) + buffer = Buffer(-1, bytearray(b'\x01'), [7], b'ip-header', bytearray(b'payload'), 1000.0) self.assertIsInstance(buffer, IP_Buffer) self.assertEqual(buffer.index, [7]) + self.assertEqual(buffer.timestamp, 1000.0) storage = ReassemblyData((datagram,), (), ()) self.assertEqual(storage.ipv4, (datagram,)) @@ -47,9 +50,9 @@ def test_ip_data_models_and_package_aliases(self) -> None: self.assertEqual(storage.tcp, ()) def test_tcp_data_models_and_package_aliases(self) -> None: - from pcapkit.foundation.reassembly.data import (TCP_Buffer, TCP_Datagram, TCP_DatagramID, - TCP_Fragment, TCP_HoleDescriptor, - TCP_Packet) + from pcapkit.foundation.reassembly.data import (Completion, TCP_Buffer, TCP_Datagram, + TCP_DatagramID, TCP_Fragment, + TCP_HoleDescriptor, TCP_Packet) from pcapkit.foundation.reassembly.data.tcp import (Buffer, Datagram, DatagramID, Fragment, HoleDescriptor, Packet) @@ -58,24 +61,28 @@ def test_tcp_data_models_and_package_aliases(self) -> None: bufid = (src, 12345, dst, 443) packet = Packet(bufid, 100, 200, 3, True, False, False, 5, 0, 4, - b'tcp-header', bytearray(b'hello')) + b'tcp-header', bytearray(b'hello'), 1000.0) self.assertIsInstance(packet, TCP_Packet) self.assertEqual(packet.first, 0) self.assertTrue(packet.syn) + self.assertEqual(packet.timestamp, 1000.0) datagram_id = DatagramID((src, 12345), (dst, 443), 200) - datagram = Datagram(True, datagram_id, (3,), b'tcp-header', b'hello', {'parsed': True}) + datagram = Datagram(Completion.COMPLETE, datagram_id, (3,), b'tcp-header', b'hello', + {'parsed': True}) self.assertIsInstance(datagram.id, TCP_DatagramID) self.assertIsInstance(datagram, TCP_Datagram) self.assertTrue(datagram.completed) + self.assertIs(datagram.completed, Completion.COMPLETE) hole = HoleDescriptor(5, 10) fragment = Fragment([3], 100, 5, bytearray(b'hello')) - buffer = Buffer([hole], b'tcp-header', {200: fragment}) + buffer = Buffer([hole], b'tcp-header', {200: fragment}, 1000.0) self.assertIsInstance(hole, TCP_HoleDescriptor) self.assertIsInstance(fragment, TCP_Fragment) self.assertIsInstance(buffer, TCP_Buffer) self.assertEqual(buffer.ack[200].raw, bytearray(b'hello')) + self.assertEqual(buffer.timestamp, 1000.0) if __name__ == '__main__': diff --git a/tests/foundation/reassembly/test_ip.py b/tests/foundation/reassembly/test_ip.py index 0f428b86fa..2e0b6d4e1d 100644 --- a/tests/foundation/reassembly/test_ip.py +++ b/tests/foundation/reassembly/test_ip.py @@ -16,14 +16,14 @@ def setUp(self) -> None: purge_modules(['pcapkit']) def _packet(self, *, num: int, fo: int, mf: bool, payload: bytes, - header: bytes = b'ip-header'): + header: bytes = b'ip-header', timestamp: float = 1000.0): from pcapkit.const.reg.transtype import TransType from pcapkit.foundation.reassembly.data.ip import Packet src = ip_address('192.0.2.1') dst = ip_address('198.51.100.2') return Packet((src, dst, 42, TransType.UDP), num, fo, 20, mf, - 20 + len(payload), header, bytearray(payload)) + 20 + len(payload), header, bytearray(payload), timestamp) def test_complete_fragmented_datagram_is_submitted_and_analyzed(self) -> None: from pcapkit.const.reg.transtype import TransType @@ -103,7 +103,7 @@ class TestIP(IP): dst = ip_address('198.51.100.2') self.assertEqual( empty.submit( - Buffer(-1, bytearray(b'\x00\x00'), [], b'', bytearray(b'')), + Buffer(-1, bytearray(b'\x00\x00'), [], b'', bytearray(b''), 1000.0), bufid=(src, dst, 42, TransType.UDP), ), [], @@ -148,7 +148,7 @@ class TestIP(IP): dst = ip_address('198.51.100.2') reasm = TestIP() reasm(Packet((src, dst, 42, TransType.UDP), 1, 0, 20, False, 25, - b'ip-header', bytearray(b'hello'))) + b'ip-header', bytearray(b'hello'), 1000.0)) datagram, = reasm.datagram return datagram diff --git a/tests/foundation/reassembly/test_ipv6.py b/tests/foundation/reassembly/test_ipv6.py index a6a13504b1..7f666c2d66 100644 --- a/tests/foundation/reassembly/test_ipv6.py +++ b/tests/foundation/reassembly/test_ipv6.py @@ -192,7 +192,7 @@ def _packet(self, *, num: int, fo: int, mf: bool, payload: bytes, header: bytes) src = ip_address('2001:db8::1') dst = ip_address('2001:db8::2') return Packet((src, dst, 4321, TransType.UDP), num, fo, len(header), mf, - len(header) + len(payload), header, bytearray(payload)) + len(header) + len(payload), header, bytearray(payload), 1000.0) def test_the_reassembled_datagram_does_not_advertise_a_fragment_header(self) -> None: from pcapkit.foundation.reassembly.ipv6 import IPv6 diff --git a/tests/foundation/reassembly/test_tcp.py b/tests/foundation/reassembly/test_tcp.py index 5bbb16dde2..1c5e4d77c9 100644 --- a/tests/foundation/reassembly/test_tcp.py +++ b/tests/foundation/reassembly/test_tcp.py @@ -29,7 +29,7 @@ def _bufid(self): def _packet(self, *, num: int, dsn: int, ack: int = 500, payload: bytes = b'', syn: bool = False, fin: bool = False, rst: bool = False, first: int | None = None, last: int | None = None, - header: bytes = b'tcp-header'): + header: bytes = b'tcp-header', timestamp: float = 1000.0): """Build a reassembly packet the way the engine toolkits build one. ``first`` defaults to ``dsn`` and ``last`` to ``dsn + len(payload) - 1`` @@ -45,7 +45,7 @@ def _packet(self, *, num: int, dsn: int, ack: int = 500, payload: bytes = b'', if last is None: last = first + len(payload) - 1 return Packet(self._bufid(), dsn, ack, num, syn, fin, rst, len(payload), - first, last, header, bytearray(payload)) + first, last, header, bytearray(payload), timestamp) def test_complete_stream_submits_on_fin_and_analyzes_payload(self) -> None: from pcapkit.foundation.reassembly.tcp import TCP @@ -120,6 +120,7 @@ class TestTCP(TCP): { 500: Fragment([1], 10, 10, bytearray(b'0123456789')), }, + 1000.0, ) reasm(self._packet(num=2, dsn=25, payload=b'after-gap', first=10, last=18)) @@ -144,12 +145,14 @@ class TestTCP(TCP): [HoleDescriptor(50, sys.maxsize)], b'', {500: Fragment([1], 10, 5, bytearray(b'world'))}, + 1000.0, ) before_gap(self._packet(num=2, dsn=0, payload=b'hello', first=40, last=44)) self.assertEqual(before_gap._buffer[bufid].ack[500].raw, bytearray(b'hello\x00\x00\x00\x00\x00world')) def test_submit_incomplete_strict_complete_strict_false_and_empty_buffers(self) -> None: + from pcapkit.foundation.reassembly.data.data import Completion from pcapkit.foundation.reassembly.data.tcp import Buffer, Fragment, HoleDescriptor from pcapkit.foundation.reassembly.tcp import TCP @@ -171,11 +174,15 @@ class TestTCP(TCP): [HoleDescriptor(2, 3), HoleDescriptor(7, 8), HoleDescriptor(99, 100)], b'tcp-header', {500: Fragment([1, 2], 0, 10, bytearray(b'abcdefghij'))}, + 1000.0, ), bufid=bufid, ) datagram, = incomplete self.assertFalse(datagram.completed) + # PARTIAL rather than TIMEOUT: this buffer was handed to ``submit`` directly, + # not abandoned by ``expire`` + self.assertIs(datagram.completed, Completion.PARTIAL) self.assertEqual(datagram.payload, (bytearray(b'ab'), bytearray(b'efg'), bytearray(b'j'))) self.assertIsNone(datagram.packet) @@ -187,26 +194,37 @@ class TestTCP(TCP): 500: Fragment([], 0, 0, bytearray()), 501: Fragment([9], 0, 9, bytearray(b'abcdefghi')), }, + 1000.0, ), bufid=bufid, ) self.assertEqual(len(mixed), 1) self.assertEqual(mixed[0].payload, (bytearray(b'bcd'), bytearray(b'g'), b'i')) + # ``strict=False`` reports the payload buffer as one contiguous blob with + # its holes zero-filled -- which is what + # :func:`~pcapkit.interface.misc.follow_tcp_stream` reconstructs a stream + # from -- rather than the runs that arrived. What it must *not* do is call + # that blob complete: the holes are still holes, so ``completed`` reads + # PARTIAL even though the payload is whole-looking. loose = TestTCP(strict=False) completed = loose.submit( Buffer( [HoleDescriptor(2, 3), HoleDescriptor(7, 8), HoleDescriptor(99, 100)], b'tcp-header', {500: Fragment([3], 0, 3, bytearray(b'abc'))}, + 1000.0, ), bufid=bufid, ) - self.assertTrue(completed[0].completed) + self.assertFalse(completed[0].completed) + self.assertIs(completed[0].completed, Completion.PARTIAL) + self.assertEqual(completed[0].payload, bytearray(b'abc')) self.assertEqual(completed[0].packet, b'abc') self.assertEqual(Analyzer.calls[-1], ((12345, 443), b'abc')) - self.assertEqual(loose.submit(Buffer([], b'', {500: Fragment([], 0, 0, bytearray())}), + self.assertEqual(loose.submit(Buffer([], b'', {500: Fragment([], 0, 0, bytearray())}, + 1000.0), bufid=bufid), []) @@ -267,12 +285,14 @@ class TestTCP(TCP): return TestTCP(**kwargs) def _segment(self, *, num: int, seq: int, payload: bytes = b'', ack: int = 1000, - syn: bool = False, fin: bool = False, rst: bool = False): + syn: bool = False, fin: bool = False, rst: bool = False, + timestamp: float = 1000.0): """One segment, described the way every :mod:`pcapkit.toolkit` describes it.""" from pcapkit.foundation.reassembly.data.tcp import Packet return Packet(self._bufid(), seq, ack, num, syn, fin, rst, len(payload), - seq, seq + len(payload) - 1, b'tcp-header', bytearray(payload)) + seq, seq + len(payload) - 1, b'tcp-header', bytearray(payload), + timestamp) def _stream(self, *, segments, isn: int = ISN, syn: bool = True, syn_ack: int = 0, teardown: str | None = 'fin'): @@ -350,8 +370,10 @@ def test_the_answer_does_not_depend_on_the_initial_sequence_number(self) -> None long; anywhere else the datagram disappeared entirely. """ + from pcapkit.foundation.reassembly.data.data import Completion + segments = [(0, b'A' * 10), (20, b'C' * 10), (40, b'E' * 10)] - expected = (False, (b'A' * 10, b'C' * 10, b'E' * 10)) + expected = (Completion.PARTIAL, (b'A' * 10, b'C' * 10, b'E' * 10)) for isn in (0, 1, 0x1000, ISN, 0xFFFF0000): with self.subTest(isn=isn): diff --git a/tests/foundation/reassembly/test_timeout.py b/tests/foundation/reassembly/test_timeout.py new file mode 100644 index 0000000000..701f2bc3d1 --- /dev/null +++ b/tests/foundation/reassembly/test_timeout.py @@ -0,0 +1,272 @@ +"""The RFC reassembly timeout, on an offline parser's only clock. + +A live stack runs its reassembly timer off the wall clock. A parser replaying a +capture file cannot: wall-clock time says nothing about the capture, and keying on +it would make the same file produce different answers on every run. The clock here +is therefore the **capture's own timestamps**, and a fragment arriving is the only +evidence that capture time has moved on -- which is why expiry is checked when a +packet is handed over rather than on a timer. + +Every timestamp below is a literal, so every expectation is exact. +""" + +from __future__ import annotations + +import importlib.util +from ipaddress import ip_address +import math +import unittest + +from tests._support import purge_modules + +RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') +HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) + +#: Arbitrary but fixed epoch the synthetic captures below start from. +T0 = 1_600_000_000.0 + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class IPReassemblyTimeoutTests(unittest.TestCase): + def setUp(self) -> None: + purge_modules(['pcapkit']) + + def _packet(self, *, num: int, fo: int, mf: bool, payload: bytes, timestamp: float, + id: int = 42): # pylint: disable=redefined-builtin + from pcapkit.const.reg.transtype import TransType + from pcapkit.foundation.reassembly.data.ip import Packet + + src = ip_address('192.0.2.1') + dst = ip_address('198.51.100.2') + return Packet((src, dst, id, TransType.UDP), num, fo, 20, mf, + 20 + len(payload), b'ip-header', bytearray(payload), timestamp) + + def test_the_default_deadline_is_the_one_the_rfcs_ask_for(self) -> None: + """60 seconds for both IPv4 and IPv6, and no deadline at all for TCP. + + :rfc:`8200#section-4.5` mandates 60 seconds for IPv6. + :rfc:`1122#section-3.3.2` requires a reassembly timeout for IPv4, says it + SHOULD be a fixed value rather than derived from the remaining TTL, and + recommends 60 to 120 seconds -- so 60 is the low end of the range and + agrees with IPv6. :rfc:`791`'s 15 seconds is an *initial* setting that + ``MAX(TIMER,TTL)`` then raises, and RFC 1122 supersedes it. + + """ + from pcapkit.foundation.reassembly.ipv4 import IPv4 + from pcapkit.foundation.reassembly.ipv6 import IPv6 + from pcapkit.foundation.reassembly.tcp import TCP + + self.assertEqual(IPv4.__timeout__, 60.0) + self.assertEqual(IPv6.__timeout__, 60.0) + self.assertEqual(IPv4().timeout, 60.0) + self.assertEqual(IPv6().timeout, 60.0) + + self.assertTrue(math.isinf(TCP.__timeout__)) + self.assertTrue(math.isinf(TCP().timeout)) + + def test_a_stalled_datagram_is_abandoned_once_the_capture_clock_passes_it(self) -> None: + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + reasm = IPv4() + # a first fragment that is never completed + reasm(self._packet(num=1, fo=0, mf=True, payload=b'abcdefgh', timestamp=T0)) + self.assertEqual(len(reasm._buffer), 1) + self.assertEqual(reasm._dtgram, []) + + # an unrelated datagram, more than 60 seconds later, advances the clock + reasm(self._packet(num=2, fo=0, mf=False, payload=b'later', timestamp=T0 + 60.5, id=99)) + + self.assertEqual(len(reasm._buffer), 0, 'the stalled buffer was not released') + expired = reasm._dtgram[0] + self.assertIs(expired.completed, Completion.TIMEOUT) + self.assertFalse(expired.completed, 'an abandoned datagram is not complete') + self.assertEqual(expired.index, (1,)) + self.assertEqual(expired.payload, (b'abcdefgh',)) + self.assertEqual(expired.id.id, 42) + + def test_timeout_is_told_apart_from_merely_unfinished(self) -> None: + """The two incomplete outcomes must be distinguishable. + + ``PARTIAL`` says "these fragments had not arrived yet"; ``TIMEOUT`` says + "these fragments are gone, and no further fragment will ever be added". + Both are falsy, so ``if datagram.completed`` reads as it did while the + field was a :obj:`bool`. + + """ + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + # left unfinished when the capture ended + unfinished = IPv4() + unfinished(self._packet(num=1, fo=0, mf=True, payload=b'abcdefgh', timestamp=T0)) + datagram, = unfinished.datagram + self.assertIs(datagram.completed, Completion.PARTIAL) + + # abandoned under the timeout + abandoned = IPv4() + abandoned(self._packet(num=1, fo=0, mf=True, payload=b'abcdefgh', timestamp=T0)) + abandoned(self._packet(num=2, fo=0, mf=True, payload=b'ijklmnop', timestamp=T0 + 61, + id=99)) + datagram, = (dtgram for dtgram in abandoned.datagram + if dtgram.completed is Completion.TIMEOUT) + self.assertEqual(datagram.index, (1,)) + + self.assertFalse(Completion.PARTIAL) + self.assertFalse(Completion.TIMEOUT) + self.assertTrue(Completion.COMPLETE) + + def test_the_deadline_is_inclusive_and_counted_from_the_first_fragment(self) -> None: + """"within 60 seconds of the reception of the first-arriving fragment". + + Two things follow from that wording, and both are asserted here: at + exactly 60 seconds the datagram is still *within* the limit, and a later + fragment does not restart the clock -- otherwise a slow trickle of + fragments could hold a buffer open indefinitely, which is the resource + exhaustion the timeout exists to bound. + + """ + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + # exactly at the limit: still within it + boundary = IPv4() + boundary(self._packet(num=1, fo=0, mf=True, payload=b'abcdefgh', timestamp=T0)) + boundary(self._packet(num=2, fo=0, mf=False, payload=b'x', timestamp=T0 + 60.0, id=99)) + self.assertEqual(len(boundary._buffer), 1) + + # one tick past it: abandoned + boundary(self._packet(num=3, fo=0, mf=False, payload=b'x', timestamp=T0 + 60.001, id=98)) + self.assertEqual(len(boundary._buffer), 0) + + # a second fragment of the same datagram does not extend the deadline + trickle = IPv4() + trickle(self._packet(num=1, fo=0, mf=True, payload=b'abcdefgh', timestamp=T0)) + trickle(self._packet(num=2, fo=16, mf=True, payload=b'ijklmnop', timestamp=T0 + 40)) + bufid, = trickle._buffer + self.assertEqual(trickle._buffer[bufid].timestamp, T0, + 'the buffer timer was restarted by a later fragment') + trickle(self._packet(num=3, fo=0, mf=False, payload=b'x', timestamp=T0 + 61, id=99)) + self.assertEqual(len(trickle._buffer), 0) + expired, = (dtgram for dtgram in trickle.datagram + if dtgram.completed is Completion.TIMEOUT) + self.assertEqual(expired.index, (1, 2)) + + def test_a_datagram_that_completes_in_time_is_untouched(self) -> None: + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + reasm = IPv4() + reasm(self._packet(num=1, fo=0, mf=True, payload=b'A' * 8, timestamp=T0)) + reasm(self._packet(num=2, fo=8, mf=False, payload=b'B' * 8, timestamp=T0 + 59.999)) + + datagram, = reasm.datagram + self.assertIs(datagram.completed, Completion.COMPLETE) + self.assertEqual(datagram.payload, b'A' * 8 + b'B' * 8) + + def test_an_infinite_timeout_disables_expiry(self) -> None: + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + reasm = IPv4(timeout=math.inf) + reasm(self._packet(num=1, fo=0, mf=True, payload=b'abcdefgh', timestamp=T0)) + reasm(self._packet(num=2, fo=0, mf=False, payload=b'x', timestamp=T0 + 86_400, id=99)) + self.assertEqual(len(reasm._buffer), 1) + + def test_an_explicit_timeout_overrides_the_protocol_default(self) -> None: + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + reasm = IPv4(timeout=5.0) + self.assertEqual(reasm.timeout, 5.0) + reasm(self._packet(num=1, fo=0, mf=True, payload=b'abcdefgh', timestamp=T0)) + reasm(self._packet(num=2, fo=0, mf=False, payload=b'x', timestamp=T0 + 6, id=99)) + self.assertEqual(len(reasm._buffer), 0) + + def test_a_negative_timeout_is_refused(self) -> None: + from pcapkit.foundation.reassembly.ipv4 import IPv4 + from pcapkit.utilities.exceptions import FieldValueError + + with self.assertRaises(FieldValueError): + IPv4(timeout=-1.0) + + def test_expire_on_an_empty_buffer_is_a_no_op(self) -> None: + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + self.assertEqual(IPv4().expire(T0), []) + self.assertEqual(IPv4(timeout=math.inf).expire(T0), []) + + def test_the_answer_depends_only_on_the_capture_timestamps(self) -> None: + """Replaying the same timestamps twice gives the same answer. + + This is the property that makes the feature usable on a file at all: no + part of it consults :func:`time.time` or :meth:`datetime.datetime.now`, so + a capture reassembles identically today, tomorrow and on another machine. + + """ + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + def run() -> list: + reasm = IPv4() + for num, (fo, mf, ts, id_) in enumerate([ + (0, True, T0, 42), + (0, True, T0 + 10, 43), + (0, False, T0 + 75, 44), + (0, False, T0 + 200, 45), + ], start=1): + reasm(self._packet(num=num, fo=fo, mf=mf, payload=b'payload', + timestamp=ts, id=id_)) + return [(dtgram.completed, dtgram.index) for dtgram in reasm.datagram] + + first, second = run(), run() + self.assertEqual(first, second) + # and it really did expire something, so the equality above is not vacuous + self.assertTrue(any(not completed for completed, _ in first)) + + +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class TCPReassemblyTimeoutTests(unittest.TestCase): + def setUp(self) -> None: + purge_modules(['pcapkit']) + + def _segment(self, *, num: int, seq: int, payload: bytes, timestamp: float, + fin: bool = False): + from pcapkit.foundation.reassembly.data.tcp import Packet + + bufid = (ip_address('192.0.2.1'), 12345, ip_address('198.51.100.2'), 443) + return Packet(bufid, seq, 500, num, False, fin, False, len(payload), + seq, seq + len(payload) - 1, b'tcp-header', bytearray(payload), + timestamp) + + def test_tcp_holds_its_buffer_by_default(self) -> None: + """An idle connection is ordinary, so nothing evicts it unasked.""" + from pcapkit.foundation.reassembly.tcp import TCP + + reasm = TCP() + reasm(self._segment(num=1, seq=0, payload=b'A' * 10, timestamp=T0)) + reasm(self._segment(num=2, seq=30, payload=b'C' * 10, timestamp=T0 + 86_400)) + self.assertEqual(len(reasm._buffer), 1) + + def test_tcp_honours_a_timeout_when_one_is_asked_for(self) -> None: + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.tcp import TCP + + reasm = TCP(timeout=120.0) + self.assertEqual(reasm.timeout, 120.0) + # a gap at sequence 10..29 that is never filled + reasm(self._segment(num=1, seq=0, payload=b'A' * 10, timestamp=T0)) + reasm(self._segment(num=2, seq=30, payload=b'C' * 10, timestamp=T0 + 1)) + self.assertEqual(len(reasm._buffer), 1) + + # a segment of the same connection past the deadline; the buffer it would + # have joined is abandoned first, so the segment opens a fresh buffer + reasm(self._segment(num=3, seq=100, payload=b'D' * 4, timestamp=T0 + 121)) + expired, = (dtgram for dtgram in reasm._dtgram + if dtgram.completed is Completion.TIMEOUT) + self.assertEqual(expired.index, (1, 2)) + self.assertEqual(expired.payload, (b'A' * 10, b'C' * 10)) + self.assertEqual(len(reasm._buffer), 1) + bufid, = reasm._buffer + self.assertEqual(reasm._buffer[bufid].timestamp, T0 + 121) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/foundation/traceflow/data/test_models.py b/tests/foundation/traceflow/data/test_models.py index 884cee444e..49232d532e 100644 --- a/tests/foundation/traceflow/data/test_models.py +++ b/tests/foundation/traceflow/data/test_models.py @@ -30,13 +30,21 @@ def test_tcp_traceflow_data_models_and_package_aliases(self) -> None: self.assertEqual(packet.frame, frame) dumper = object() - buffer = Buffer(dumper, [1, 2], '2001_db8_1-12345_2001_db8_2-443') + origin = (src, 12345) + buffer = Buffer(dumper, [1, 2], '2001_db8_1-12345_2001_db8_2-443', + origin, [1], [2], {origin}) self.assertIsInstance(buffer, TCP_Buffer) self.assertEqual(buffer.fpout, dumper) + self.assertEqual(buffer.origin, origin) + self.assertEqual((buffer.forward, buffer.reverse), ([1], [2])) + self.assertEqual(buffer.fin, {origin}) - index = Index('/tmp/flow.json', (1, 2), buffer.label) + index = Index('/tmp/flow.json', (1, 2), buffer.label, (1,), (2,)) self.assertIsInstance(index, TCP_Index) self.assertEqual(index.index, (1, 2)) + # each direction stays recoverable, so a caller can still ask which way a + # given frame went without taking the label apart + self.assertEqual((index.forward, index.reverse), ((1,), (2,))) storage = TraceFlowData((index,)) self.assertEqual(storage.tcp, (index,)) diff --git a/tests/foundation/traceflow/test_tcp.py b/tests/foundation/traceflow/test_tcp.py index 17bdda7f30..73c87a2527 100644 --- a/tests/foundation/traceflow/test_tcp.py +++ b/tests/foundation/traceflow/test_tcp.py @@ -29,6 +29,13 @@ def _packet(self, *, index: int, src: str = '192.0.2.1', dst: str = '198.51.100. srcport, dstport, timestamp) def test_tcp_trace_ipv4_fin_submit_cache_callback_and_dump(self) -> None: + """One direction, traced with ``bidirectional=False``. + + This is the per-direction behaviour flow tracing had before conversations + became one flow, and it is still reachable on request -- so a single FIN + closes the flow, since in that mode a flow *is* one direction. + + """ from pcapkit.dumpkit.null import NotImplementedIO from pcapkit.foundation.traceflow.tcp import TCP @@ -37,7 +44,7 @@ def test_tcp_trace_ipv4_fin_submit_cache_callback_and_dump(self) -> None: TCP.register_callback(callbacks.append) with tempfile.TemporaryDirectory() as tempdir: - trace = TCP(tempdir, 'unit-null') + trace = TCP(tempdir, 'unit-null', bidirectional=False) first = self._packet(index=1, syn=True) label = trace.trace(first) self.assertEqual(label, '192.0.2.1_12345-198.51.100.2_443-1.25') @@ -56,6 +63,101 @@ def test_tcp_trace_ipv4_fin_submit_cache_callback_and_dump(self) -> None: self.assertEqual(final_index.label, label) self.assertEqual(final_index.fpout, f'{tempdir}/{label}.unit') self.assertEqual(callbacks[-1], final_index) + # every frame is "forward" when a flow is a single direction + self.assertEqual(final_index.forward, (1, 2, 3, 4)) + self.assertEqual(final_index.reverse, ()) + + def _reply(self, **kwargs): + """A packet travelling the other way down the same connection.""" + return self._packet(src='198.51.100.2', dst='192.0.2.1', + srcport=443, dstport=12345, **kwargs) + + def test_both_halves_of_a_connection_are_one_flow(self) -> None: + """The two directions share a buffer, a label and an output file. + + Keyed on (source, destination) they were two flows with two labels and two + dump files, leaving a caller to pair them up by reading the labels. + + """ + from pcapkit.dumpkit.null import NotImplementedIO + from pcapkit.foundation.traceflow.tcp import TCP + + TCP.register_dumper('unit-null', NotImplementedIO, '.unit') + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unit-null') + + # the client opens the conversation, so its direction is "forward" + label = trace.trace(self._packet(index=1, syn=True)) + self.assertEqual(label, '192.0.2.1_12345-198.51.100.2_443-1.25') + + # ... and the server's reply joins that flow rather than minting a + # label of its own + self.assertEqual(trace.trace(self._reply(index=2, syn=True, timestamp=1.5)), label) + self.assertEqual(len(trace._buffer), 1) + + flow, = trace.index + self.assertEqual(flow.index, (1, 2)) + self.assertEqual(flow.forward, (1,)) + self.assertEqual(flow.reverse, (2,)) + + def test_a_conversation_closes_only_once_both_halves_have_finished(self) -> None: + """One FIN is half a teardown, so it must not close the flow. + + Closing on the first FIN would cut the peer's FIN and the final + acknowledgement out of the flow -- and they would then open a *second* + flow under the same buffer ID, which is the split bidirectional tracing + exists to remove. + + """ + from pcapkit.dumpkit.null import NotImplementedIO + from pcapkit.foundation.traceflow.tcp import TCP + + TCP.register_dumper('unit-null', NotImplementedIO, '.unit') + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unit-null') + trace.trace(self._packet(index=1, syn=True)) + trace.trace(self._reply(index=2, timestamp=1.5)) + + # the client finishes; the server has not + trace.trace(self._packet(index=3, fin=True, timestamp=1.75)) + self.assertEqual(len(trace._buffer), 1, 'flow closed on a half teardown') + self.assertEqual(trace._stream, []) + + # a retransmitted FIN from the same endpoint is still one endpoint + trace.trace(self._packet(index=4, fin=True, timestamp=1.8)) + self.assertEqual(trace._stream, []) + + # now the server finishes too + trace.trace(self._reply(index=5, fin=True, timestamp=2.0)) + self.assertEqual(len(trace._buffer), 0) + flow, = trace._stream + self.assertEqual(flow.index, (1, 2, 3, 4, 5)) + self.assertEqual(flow.forward, (1, 3, 4)) + self.assertEqual(flow.reverse, (2, 5)) + + def test_the_buffer_id_is_canonical_and_stays_a_tuple(self) -> None: + """Both directions reduce to the same key, whichever is seen first. + + The key has to stay a plain :obj:`tuple`: it is a :obj:`dict` key, and an + :class:`~pcapkit.corekit.infoclass.Info` cannot be one, because inheriting + :class:`collections.abc.Mapping` sets ``__hash__`` to :data:`None`. + + """ + from pcapkit.foundation.traceflow.tcp import TCP + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unknown-unit-format') + forward = trace.make_bufid(self._packet(index=1)) + reverse = trace.make_bufid(self._reply(index=2)) + self.assertEqual(forward, reverse) + self.assertIs(type(forward), tuple) + hash(forward) # a key that cannot be hashed is not a key + + oneway = TCP(tempdir, 'unknown-unit-format', bidirectional=False) + self.assertNotEqual(oneway.make_bufid(self._packet(index=1)), + oneway.make_bufid(self._reply(index=2))) def test_tcp_trace_ipv6_label_and_no_extension_output_path(self) -> None: from pcapkit.foundation.traceflow.tcp import TCP diff --git a/tests/integration/test_reassembly_engine_parity.py b/tests/integration/test_reassembly_engine_parity.py index dff288c320..07a6078138 100644 --- a/tests/integration/test_reassembly_engine_parity.py +++ b/tests/integration/test_reassembly_engine_parity.py @@ -162,10 +162,14 @@ def adapter_packets(self) -> 'dict[str, list]': import dpkt from pcapkit.toolkit import dpkt as dpkt_toolkit + # ``pcap_frames`` hands back the record octets alone, so the capture + # timestamp has to be supplied here -- one value for every fragment, + # both because they are one datagram and because the fields compared + # below must not depend on which fragment is being looked at packets['dpkt'] = [ data for number, frame in enumerate(frames, start=1) if (data := dpkt_toolkit.ipv6_reassembly( - dpkt.ethernet.Ethernet(frame), count=number)) is not None + dpkt.ethernet.Ethernet(frame), 0.0, count=number)) is not None ] if HAS_SCAPY: diff --git a/tests/integration/test_traceflow_end_to_end.py b/tests/integration/test_traceflow_end_to_end.py index 07e9987ceb..1b1b2bff8f 100644 --- a/tests/integration/test_traceflow_end_to_end.py +++ b/tests/integration/test_traceflow_end_to_end.py @@ -15,13 +15,15 @@ from tests._support import sample_path from tests.integration._helpers import HAS_RUNTIME, EndToEndTestCase, read_json -#: The four directional flows of :file:`tcp.pcap`: two concurrent SSH sessions, -#: one over IPv4 and one over IPv6, each traced per direction. +#: The two connections of :file:`tcp.pcap`: concurrent SSH sessions, one over +#: IPv4 and one over IPv6. Tracing keys a flow on the canonical *pair* of +#: endpoints, so a connection is one flow rather than one per direction; the +#: label is that of the direction seen first, and ``forward``/``reverse`` hold +#: that direction's frames and its peer's. Mapped here as +#: ``label: (forward, reverse)``. TCP_PCAP_FLOWS = { - '10.20.30.130_22-10.20.30.131_53406-1500000000.000774': (1, 7), - '10.20.30.131_53406-10.20.30.130_22-1500000000.001585': (2, 6), - 'fe80..a6.87f9.2793.16ee_51774-fe80..1ccd.7c77.bac7.46b7_22-1500000000.002433': (3,), - 'fe80..1ccd.7c77.bac7.46b7_22-fe80..a6.87f9.2793.16ee_51774-1500000000.003318': (4, 5), + '10.20.30.130_22-10.20.30.131_53406-1500000000.000774': ((1, 7), (2, 6)), + 'fe80..a6.87f9.2793.16ee_51774-fe80..1ccd.7c77.bac7.46b7_22-1500000000.002433': ((3,), (4, 5)), } @@ -35,20 +37,29 @@ def trace(self, capture: 'str' = 'tcp.pcap') -> 'tuple': trace_fout=self.out('trace')) return extractor.length, extractor.trace.tcp - def test_every_direction_of_every_connection_becomes_a_flow(self) -> None: + def test_both_directions_of_a_connection_become_one_flow(self) -> None: length, flows = self.trace() self.assertEqual(length, 7) - self.assertEqual({flow.label: flow.index for flow in flows}, TCP_PCAP_FLOWS) + # each connection is a single flow, and both of its halves are still + # reported separately -- as ``forward`` and ``reverse`` of that one flow + self.assertEqual({flow.label: (flow.forward, flow.reverse) for flow in flows}, + TCP_PCAP_FLOWS) + # ... and the flow's own index is the two halves in wire order, so no + # frame is dropped by the merge and none is counted twice + self.assertEqual({flow.label: flow.index for flow in flows}, + {label: tuple(sorted(forward + reverse)) + for label, (forward, reverse) in TCP_PCAP_FLOWS.items()}) def test_flow_labels_carry_both_address_families(self) -> None: _, flows = self.trace() labels = {flow.label for flow in flows} # A label is ``src_port-dst_port-timestamp``, with the dots of an IPv6 - # address doubled so the label stays usable as a file name. - self.assertEqual(sum(1 for label in labels if label.startswith('10.20.30.')), 2) - self.assertEqual(sum(1 for label in labels if label.startswith('fe80..')), 2) + # address doubled so the label stays usable as a file name. One label per + # connection rather than per direction, so one of each family here. + self.assertEqual(sum(1 for label in labels if label.startswith('10.20.30.')), 1) + self.assertEqual(sum(1 for label in labels if label.startswith('fe80..')), 1) def test_each_flow_is_dumped_to_its_own_report(self) -> None: _, flows = self.trace() @@ -99,7 +110,11 @@ def test_every_frame_lands_in_exactly_one_flow(self) -> None: flows = extractor.trace.tcp self.assertEqual(extractor.length, 1117) - self.assertEqual(len(flows), 331) + # 220 connections, not the 331 directions they are made of: 111 of them + # were captured both ways and 109 only one way, and 2 * 111 + 109 == 331, + # so the drop is the two halves of a connection meeting in one flow and + # not frames going missing -- which is what the index check below pins + self.assertEqual(len(flows), 220) indexed = [number for flow in flows for number in flow.index] self.assertEqual(len(indexed), 1117) @@ -113,9 +128,9 @@ def test_one_report_is_written_per_flow(self) -> None: written = {entry.name for entry in self.tmp_path.joinpath('trace').iterdir()} self.assertEqual(written, {f'{flow.label}.json' for flow in flows}) - self.assertEqual(len(written), 331) + self.assertEqual(len(written), 220) - # Spot-check the longest flow rather than re-reading all 331 reports. + # Spot-check the longest flow rather than re-reading all 220 reports. longest = max(flows, key=lambda flow: len(flow.index)) report = read_json(longest.fpout) self.assertEqual(list(report), [f'Frame {number}' for number in longest.index]) diff --git a/tests/interface/test_core.py b/tests/interface/test_core.py index 15e09cb256..4c5ba09d0a 100644 --- a/tests/interface/test_core.py +++ b/tests/interface/test_core.py @@ -88,8 +88,9 @@ def __init__(self, **kwargs): def make_reassembly(name): class Reassembly: - def __init__(self, strict=False): + def __init__(self, strict=False, timeout=None): self.strict = strict + self.timeout = timeout Reassembly.__name__ = name return Reassembly @@ -152,6 +153,10 @@ def id(cls): result = module.reassemble(TCP, strict=True) self.assertEqual(type(result).__name__, 'TCP') self.assertTrue(result.strict) + # ``None`` is passed straight through, so each reassembler picks its own + # RFC default rather than having one imposed here + self.assertIsNone(result.timeout) + self.assertEqual(module.reassemble('IPv4', timeout=30.0).timeout, 30.0) self.assertEqual(type(module.reassemble('IPv4')).__name__, 'IPv4') self.assertEqual(type(module.reassemble('IPv6')).__name__, 'IPv6') diff --git a/tests/interface/test_misc.py b/tests/interface/test_misc.py index b8f7e80416..7228446784 100644 --- a/tests/interface/test_misc.py +++ b/tests/interface/test_misc.py @@ -25,7 +25,20 @@ #: yardstick every other engine is judged by: every engine reaching this module #: is expected to agree on it, either by dissecting the capture itself or by being #: redirected to the default engine. -IN_PCAP_TCP_STREAMS = 3 +#: +#: **Two, not three.** Flow tracing was keyed on (source, destination), so the two +#: halves of one connection came back as two flows; it is now keyed on the pair of +#: endpoints, so a conversation is one flow. ``in.pcap``'s three TCP frames are +#: frame 3 (``123.129.210.135:80`` -> ``192.168.1.100:55232``), frame 4 (the same +#: connection the other way) and frame 5 (``192.168.1.100:55216`` -> +#: ``123.129.210.135:80``, a second connection). Frames 3 and 4 share an address +#: pair and a port pair and differ only in direction, so they are one TCP +#: connection and now one flow -- leaving two conversations, not three. +IN_PCAP_TCP_STREAMS = 2 + +#: The same capture with ``trace_bidirectional=False``, i.e. the per-direction +#: behaviour flow tracing had before conversations became one flow. +IN_PCAP_TCP_FLOWS_PER_DIRECTION = 3 @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') @@ -33,14 +46,16 @@ class FollowTCPStreamTests(unittest.TestCase): """:func:`~pcapkit.interface.misc.follow_tcp_stream` per extraction engine. ``examples/captures/in.pcap`` (committed, so unit-tier safe) holds three TCP - conversations. #399: the reassembly adapter was chosen by comparing the - engine *instance* to a string -- a comparison that is never true -- so every - engine silently used the pcapkit adapter regardless of which engine ran. That - crashed on DPKT frames (``AttributeError: 'dict' object has no attribute - 'packet'``) and returned an empty, misleading result on Scapy frames. + frames making up **two** conversations -- see + :data:`IN_PCAP_TCP_STREAMS` for which frames pair up and why. #399: the + reassembly adapter was chosen by comparing the engine *instance* to a string -- + a comparison that is never true -- so every engine silently used the pcapkit + adapter regardless of which engine ran. That crashed on DPKT frames + (``AttributeError: 'dict' object has no attribute 'packet'``) and returned an + empty, misleading result on Scapy frames. Both third-party engines that dissect this capture -- DPKT and Scapy -- must - therefore agree with the default engine on all three streams. Engines that do + therefore agree with the default engine on every stream. Engines that do no dissection at all (PyPCAP) or have no reassembly adapter (PyShark) are redirected to the default engine instead, and are asserted separately below. @@ -51,7 +66,7 @@ class FollowTCPStreamTests(unittest.TestCase): from :class:`~pcapkit.foundation.engines.scapy.Scapy` importing only :mod:`scapy.sendrecv`, which populates none of Scapy's layer registries, so every frame came back as an undissected ``Raw``. Zero was the polluted - answer and three is the truthful one. + answer and the count here is the truthful one. """ def setUp(self) -> None: @@ -74,14 +89,28 @@ def test_default_engine_finds_every_tcp_stream(self) -> None: streams = self._follow() self.assertEqual(len(streams), IN_PCAP_TCP_STREAMS) # Each detected flow carries the frames traceflow grouped into it, and names - # the file its trace was written to. in.pcap's three TCP flows are one frame - # each and single-segment, so there is no multi-segment payload to - # reassemble -- the empty conversations are a property of this capture, not - # of the reassembly, which is why the parity test below compares them rather - # than requiring them non-empty. + # the file its trace was written to. in.pcap's TCP segments are + # single-segment, so there is no multi-segment payload to reassemble -- the + # empty conversations are a property of this capture, not of the reassembly, + # which is why the parity test below compares them rather than requiring + # them non-empty. for stream in streams: self.assertGreaterEqual(len(stream.packets), 1) self.assertIsNotNone(stream.filename) + # the merged conversation carries both of its frames, which is the point: + # keyed per direction they were two streams of one frame each + self.assertEqual(sorted(len(stream.packets) for stream in streams), [1, 2]) + + def test_per_direction_tracing_still_splits_the_conversation(self) -> None: + """``trace_bidirectional=False`` restores the older per-direction flows. + + The compatibility escape hatch has to keep working, and it is what makes + the merge above measurable rather than merely asserted. + + """ + streams = self._follow(trace_bidirectional=False) + self.assertEqual(len(streams), IN_PCAP_TCP_FLOWS_PER_DIRECTION) + self.assertEqual(sorted(len(stream.packets) for stream in streams), [1, 1, 1]) @unittest.skipUnless(HAS_DPKT, 'dpkt not installed') def test_dpkt_engine_matches_the_default_engine(self) -> None: diff --git a/tests/toolkit/test_dpkt_unit.py b/tests/toolkit/test_dpkt_unit.py index 5c8ebdb6f9..28b55ad7e1 100644 --- a/tests/toolkit/test_dpkt_unit.py +++ b/tests/toolkit/test_dpkt_unit.py @@ -162,7 +162,7 @@ def test_packet_chain_dict_and_ipv4_reassembly_with_real_dpkt_packet(self) -> No fragment = self._make_ipv4_tcp_packet(fragmented=True) with warnings.catch_warnings(): warnings.simplefilter('ignore') - reassembled = toolkit.ipv4_reassembly(fragment, count=4) + reassembled = toolkit.ipv4_reassembly(fragment, 0.0, count=4) self.assertIsNotNone(reassembled) assert reassembled is not None self.assertEqual(reassembled.num, 4) @@ -175,15 +175,16 @@ def test_packet_chain_dict_and_ipv4_reassembly_with_real_dpkt_packet(self) -> No self.assertEqual(bytes(reassembled.payload), fragment.ip.pack()[fragment.ip.hl * 4:]) - self.assertIsNone(toolkit.ipv4_reassembly(types.SimpleNamespace(), count=1)) - self.assertIsNone(toolkit.ipv4_reassembly(self._make_ipv4_tcp_packet(df=True), count=1)) + self.assertIsNone(toolkit.ipv4_reassembly(types.SimpleNamespace(), 0.0, count=1)) + self.assertIsNone(toolkit.ipv4_reassembly(self._make_ipv4_tcp_packet(df=True), 0.0, + count=1)) def test_tcp_reassembly_and_traceflow_accept_dpkt_data_payload_tcp(self) -> None: from pcapkit.const.reg.linktype import LinkType from pcapkit.toolkit import dpkt as toolkit packet = self._make_ipv4_tcp_packet() - tcp = toolkit.tcp_reassembly(packet, count=9) + tcp = toolkit.tcp_reassembly(packet, 50.25, count=9) self.assertIsNotNone(tcp) assert tcp is not None self.assertEqual(tcp.bufid[1], 1234) @@ -206,12 +207,12 @@ def test_tcp_reassembly_and_traceflow_accept_dpkt_data_payload_tcp(self) -> None self.assertFalse(flow.fin) self.assertEqual(flow.timestamp, 50.25) - self.assertIsNone(toolkit.tcp_reassembly(types.SimpleNamespace(), count=1)) + self.assertIsNone(toolkit.tcp_reassembly(types.SimpleNamespace(), 1.0, count=1)) self.assertIsNone(toolkit.tcp_traceflow(types.SimpleNamespace(), 1.0, data_link=LinkType.ETHERNET, count=1)) raw_ip = types.SimpleNamespace(src=b'\x7f\x00\x00\x01', dst=b'\x7f\x00\x00\x01', data=b'not tcp') - self.assertIsNone(toolkit.tcp_reassembly(types.SimpleNamespace(ip=raw_ip), count=1)) + self.assertIsNone(toolkit.tcp_reassembly(types.SimpleNamespace(ip=raw_ip), 1.0, count=1)) self.assertIsNone(toolkit.tcp_traceflow(types.SimpleNamespace(ip=raw_ip), 1.0, data_link=LinkType.ETHERNET, count=1)) @@ -220,7 +221,7 @@ def test_tcp_helpers_cover_ipv6_and_data_payload_fallback(self) -> None: from pcapkit.toolkit import dpkt as toolkit packet = FakeDPKTPacket() - tcp = toolkit.tcp_reassembly(packet, count=12) + tcp = toolkit.tcp_reassembly(packet, 60.5, count=12) self.assertIsNotNone(tcp) assert tcp is not None self.assertEqual(tcp.bufid[0], ip_address('2001:db8::10')) @@ -248,7 +249,7 @@ def test_ipv6_header_length_and_reassembly_with_fragment_fake(self) -> None: self.assertEqual(toolkit.ipv6_hdr_len(ipv6), 48) packet = types.SimpleNamespace(ip6=ipv6) - reassembled = toolkit.ipv6_reassembly(packet, count=5) + reassembled = toolkit.ipv6_reassembly(packet, 0.0, count=5) self.assertIsNotNone(reassembled) assert reassembled is not None self.assertEqual(reassembled.num, 5) @@ -268,8 +269,9 @@ def test_ipv6_header_length_and_reassembly_with_fragment_fake(self) -> None: self.assertEqual(reassembled.ihl, 48) self.assertEqual(bytes(reassembled.payload), b'PAYLOAD') - self.assertIsNone(toolkit.ipv6_reassembly(types.SimpleNamespace(), count=1)) - self.assertIsNone(toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=FakeIPv6()), count=1)) + self.assertIsNone(toolkit.ipv6_reassembly(types.SimpleNamespace(), 0.0, count=1)) + self.assertIsNone(toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=FakeIPv6()), 0.0, + count=1)) # --------------------------------------------------------------------------- @@ -405,7 +407,7 @@ def test_header_ends_at_data_offset_and_len_matches_payload(self) -> None: self.assertGreater(len(tcp.opts), 0) self.assertGreater(tcp.off * 4, tcp.__hdr_len__) - data = toolkit.tcp_reassembly(packet, count=1) + data = toolkit.tcp_reassembly(packet, 0.0, count=1) self.assertIsNotNone(data) assert data is not None @@ -433,7 +435,7 @@ def test_option_only_segment_yields_empty_payload(self) -> None: tcp = packet.ip.data self.assertGreater(len(tcp.opts), 0) - data = toolkit.tcp_reassembly(packet, count=1) + data = toolkit.tcp_reassembly(packet, 0.0, count=1) assert data is not None self.assertEqual(data.len, 0) self.assertEqual(len(data.payload), 0) @@ -465,7 +467,7 @@ def test_every_tcp_frame_of_the_sample_capture_holds_the_invariants(self) -> Non if tcp is None: continue - data = toolkit.tcp_reassembly(packet, count=index) + data = toolkit.tcp_reassembly(packet, 0.0, count=index) assert data is not None with self.subTest(frame=index): self.assertEqual(len(data.header), tcp.off * 4) @@ -502,7 +504,7 @@ def test_header_ends_at_internet_header_length_with_options(self) -> None: with warnings.catch_warnings(): warnings.simplefilter('ignore') - data = toolkit.ipv4_reassembly(packet, count=1) + data = toolkit.ipv4_reassembly(packet, 0.0, count=1) self.assertIsNotNone(data) assert data is not None @@ -526,7 +528,7 @@ def test_fragment_offset_is_reported_in_octets(self) -> None: with warnings.catch_warnings(): warnings.simplefilter('ignore') - data = toolkit.ipv4_reassembly(packet, count=1) + data = toolkit.ipv4_reassembly(packet, 0.0, count=1) assert data is not None self.assertEqual(ipv4.offset, offset_units) @@ -546,6 +548,11 @@ def test_reassembly_reconstructs_a_fragmented_datagram(self) -> None: body = bytes(range(256)) * 6 # 1536 octets, a multiple of 8 first, second = body[:1024], body[1024:] + # both fragments carry the one capture timestamp: they belong to a single + # datagram, and a spread wider than ``IPv4.__timeout__`` would expire the + # buffer rather than complete it, which is a different test + timestamp = 1.0 + reasm = IPv4(strict=True) with warnings.catch_warnings(): warnings.simplefilter('ignore') @@ -554,7 +561,7 @@ def test_reassembly_reconstructs_a_fragmented_datagram(self) -> None: (len(first) // 8, False, second), ), start=1): packet = _make_ipv4_fragment(offset_units=offset_units, mf=mf, body=chunk) - data = toolkit.ipv4_reassembly(packet, count=index) + data = toolkit.ipv4_reassembly(packet, timestamp, count=index) assert data is not None reasm(data) @@ -597,7 +604,7 @@ def test_reads_the_real_fragment_header_attributes(self) -> None: self.assertEqual(frag.frag_off, frag._frag_off_resv_m >> 3) self.assertNotEqual(frag.frag_off, frag._frag_off_resv_m) - data = toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=ipv6), count=3) + data = toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=ipv6), 0.0, count=3) self.assertIsNotNone(data) assert data is not None @@ -633,7 +640,7 @@ def test_offset_is_scaled_once_into_octets(self) -> None: ipv6 = dpkt.ip6.IP6(_ipv6_fragment_bytes( offset_units=offset_units, mf=True, body=b'C' * 64, )) - data = toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=ipv6), count=1) + data = toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=ipv6), 0.0, count=1) assert data is not None self.assertEqual(data.fo, 1448) # 181 units of 8 octets @@ -659,7 +666,7 @@ def test_buffer_identifier_is_keyed_on_the_fragment_identification(self) -> None self.assertEqual(ipv6.flow, flow) self.assertEqual(ipv6.extension_hdrs[44].id, ident) - data = toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=ipv6), count=1) + data = toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=ipv6), 0.0, count=1) assert data is not None self.assertEqual(data.bufid[2], ident) @@ -682,6 +689,10 @@ def test_two_datagrams_sharing_a_flow_label_stay_separate(self) -> None: first, second = b'E' * 64, b'F' * 64 third, fourth = b'G' * 64, b'H' * 64 + # one capture timestamp across all four, so that what separates the two + # datagrams is the identification alone and not the reassembly timeout + timestamp = 1.0 + reasm = IPv6(strict=True) for index, (ident, offset_units, mf, chunk) in enumerate(( (1000, 0, True, first), @@ -692,7 +703,7 @@ def test_two_datagrams_sharing_a_flow_label_stay_separate(self) -> None: ipv6 = dpkt.ip6.IP6(_ipv6_fragment_bytes( offset_units=offset_units, mf=mf, body=chunk, ident=ident, flow=0x12345, )) - data = toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=ipv6), count=index) + data = toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=ipv6), timestamp, count=index) assert data is not None reasm(data) @@ -712,6 +723,10 @@ def test_reassembly_reconstructs_a_fragmented_datagram(self) -> None: body = bytes(range(256)) * 6 # 1536 octets, a multiple of 8 first, second = body[:1024], body[1024:] + # both fragments carry the one capture timestamp, so the datagram + # completes rather than expiring under ``IPv6.__timeout__`` + timestamp = 1.0 + reasm = IPv6(strict=True) for index, (offset_units, mf, chunk) in enumerate(( (0, True, first), @@ -720,7 +735,7 @@ def test_reassembly_reconstructs_a_fragmented_datagram(self) -> None: ipv6 = dpkt.ip6.IP6(_ipv6_fragment_bytes( offset_units=offset_units, mf=mf, body=chunk, )) - data = toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=ipv6), count=index) + data = toolkit.ipv6_reassembly(types.SimpleNamespace(ip6=ipv6), timestamp, count=index) assert data is not None reasm(data) diff --git a/tests/toolkit/test_scapy_unit.py b/tests/toolkit/test_scapy_unit.py index 08925d6dc1..d62632c319 100644 --- a/tests/toolkit/test_scapy_unit.py +++ b/tests/toolkit/test_scapy_unit.py @@ -198,8 +198,11 @@ def test_tcp_reassembly_and_traceflow(self) -> None: self.assertEqual(v6_tcp.bufid[0], ip_address('2001:db8::1')) self.assertEqual(v6_tcp.bufid[3], 443) - with mock.patch('pcapkit.toolkit.scapy.time.time', return_value=77.25): - flow = toolkit.tcp_traceflow(packet, count=8) + # the flow label carries the *capture's* clock, read off ``Packet.time``, + # rather than the moment of parsing -- so the deterministic value is + # pinned on the packet itself rather than by patching ``time.time`` + packet.time = 77.25 + flow = toolkit.tcp_traceflow(packet, count=8) self.assertIsNotNone(flow) assert flow is not None self.assertEqual(flow.protocol, LinkType.ETHERNET) From 7f7c6a774bdf73001fd577d4b69d94df01a4676b Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 17 Sep 2026 12:11:59 -0400 Subject: [PATCH 2/8] traceflow: keep the __init__ stubs on one line, where their pylint disable reaches ``# pylint: disable`` is line-scoped, and ``unused-argument`` is reported against the ``def`` -- so wrapping a signature leaves every parameter on a continuation line outside the disable's reach. Splitting Buffer's and Index's stubs to fit the new fields therefore leaked seven unused-argument messages, and the shorter form already there was leaking three of its own plus two super-init-not-called. Both stubs go back on one line, as every other data model in pcapkit writes them, with a note saying why the long line is deliberate. Pylint: 4689 messages against 4706 on the branch point. --- pcapkit/foundation/traceflow/data/tcp.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pcapkit/foundation/traceflow/data/tcp.py b/pcapkit/foundation/traceflow/data/tcp.py index 8aa1554356..5067f4eb78 100644 --- a/pcapkit/foundation/traceflow/data/tcp.py +++ b/pcapkit/foundation/traceflow/data/tcp.py @@ -107,9 +107,13 @@ class Buffer(Info, Generic[_AT]): fin: 'set[tuple[_AT, int]]' if TYPE_CHECKING: - def __init__(self, fpout: 'Dumper', index: 'list[int]', label: 'str', # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long - origin: 'tuple[_AT, int]', forward: 'list[int]', reverse: 'list[int]', - fin: 'set[tuple[_AT, int]]') -> 'None': ... + # NOTE: one line, however long. ``# pylint: disable`` is *line*-scoped and + # ``unused-argument`` is reported against the ``def``, so wrapping the + # signature leaves every parameter on a continuation line outside the + # disable's reach -- which is why the shorter form this replaces leaked + # three ``unused-argument`` messages of its own. Every other data model in + # :mod:`pcapkit` writes these stubs on one line for the same reason. + def __init__(self, fpout: 'Dumper', index: 'list[int]', label: 'str', origin: 'tuple[_AT, int]', forward: 'list[int]', reverse: 'list[int]', fin: 'set[tuple[_AT, int]]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long @info_final @@ -140,6 +144,5 @@ class Index(Info): reverse: 'tuple[int, ...]' if TYPE_CHECKING: - def __init__(self, fpout: 'Optional[str]', index: 'tuple[int, ...]', # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long - label: 'str', forward: 'tuple[int, ...]', - reverse: 'tuple[int, ...]') -> 'None': ... + # NOTE: on one line, for the reason given on :class:`Buffer` above. + def __init__(self, fpout: 'Optional[str]', index: 'tuple[int, ...]', label: 'str', forward: 'tuple[int, ...]', reverse: 'tuple[int, ...]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long From f5fd367a5a1e8a5d1964b0991aac2fac5ddd2951 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 17 Sep 2026 14:08:12 -0400 Subject: [PATCH 3/8] traceflow: a teardown is not the end of a flow; correct the IP loose-mode claim Two findings from the review of #435. ## A flow closed one packet too early, and reused ports merged (blocking) `closed = len(buffer.fin) >= 2` submitted a bidirectional flow on the second FIN of its four-way close. The close is FIN, ACK, FIN, ACK, so the final ACK arrived after the buffer had been popped and opened a fresh buffer under the same canonical BUFID -- which a later connection reusing those endpoints then merged into. The NOTE above that line predicted exactly this defect for closing on the *first* FIN and was only half-applied. Reproduced on the reviewed head with a four-way close plus reuse: frames (6, 7, 8) came back as one flow, mixing connection one's final ACK with connection two. It was visible in the corpus all along. http.pcap traced 220 flows, of which **109 were single-frame stray tails** beside 109 nine-frame flows -- each conversation split in two. It now traces 111, every one two-way, none of one frame, all 1117 frames still in exactly one flow. My earlier report read those 109 as one-directional conversations; they were the bug. The fix is not a later close condition but a different question. Observing a teardown is not knowing that nothing more will arrive: duplicates of the final ACK can follow, so no rule naming the last packet of the exchange can hold. So a teardown -- FIN from both endpoints, or RST from either -- is *recorded* and finalises nothing; the flow keeps the packets that belong to it. It is finalised only by proof that no more can come: a new connection's SYN on the same endpoints, or the end of the capture, via the new `TraceFlow.finish()` that `Extractor._cleanup` calls. Callbacks fire there, so they see the whole conversation. `submit()` still reports an unfinalised flow, so reading `index` mid-capture cannot strand the rest of a conversation in a second flow. Telling that SYN from the peer's SYN-ACK is what the recorded teardown is for -- a SYN-ACK cannot follow a completed teardown. That also closes the RST gap `pep.rst` documents: `rst` is now on the traceflow packet model and reported by all six adapters that build one. `bidirectional=False` still closes on FIN, ignores RST, and reproduces main exactly at 355 flows. ## The IP loose-mode payload *did* change (correcting the earlier claim) The previous commit message said of both `strict=False` fixes that "the payload shape is unchanged, so follow_tcp_stream still reconstructs what it did". That is true of the TCP fix and **not** of the IP one, where the payload changed. Stated properly: - **TCP**: payload unchanged -- the buffer with its holes zero-filled, which is what `follow_tcp_stream` reconstructs a stream from. Only `completed` stopped claiming a holed datagram was complete. - **IP, final fragment received** (`TDL > 0`): payload unchanged, holes zero-filled, as TCP does. Only `completed` became honest. - **IP, final fragment never received** (`TDL` still `-1`): the payload changed. It was `datagram[:-1]`, 65534 octets of preallocated buffer reported complete. It is now the **contiguous prefix** -- offset zero to the first hole -- rather than the `b''` an earlier draft of this branch returned, because those octets really did arrive and a blob cannot convey the offset of a run that starts after an unmeasured gap. `strict=True`, the default, still lists the runs. Both branches now have tests; the IP one had none. Suite 901 passed / 17 skipped / 848 subtests, 0 failed. All 15 sample captures still give byte-identical tree, json and reassembly output against the branch point, and still regenerate byte-identically; 1479 datagrams, all COMPLETE. mypy 127 against 128; pylint 4686 against 4706. --- .../pcapkit/foundation/traceflow/tcp.rst | 15 ++ .../foundation/traceflow/traceflow.rst | 1 + docs/source/pep.rst | 44 ++++- pcapkit/foundation/extraction.py | 12 ++ pcapkit/foundation/reassembly/ip.py | 47 ++++- pcapkit/foundation/traceflow/data/tcp.py | 15 +- pcapkit/foundation/traceflow/tcp.py | 171 +++++++++++++---- pcapkit/foundation/traceflow/traceflow.py | 22 +++ pcapkit/toolkit/dpkt.py | 1 + pcapkit/toolkit/pcap.py | 1 + pcapkit/toolkit/pcapng.py | 1 + pcapkit/toolkit/pypcapfile.py | 1 + pcapkit/toolkit/pyshark.py | 1 + pcapkit/toolkit/scapy.py | 1 + tests/foundation/reassembly/test_timeout.py | 102 ++++++++++ tests/foundation/test_extraction.py | 5 +- .../foundation/traceflow/data/test_models.py | 6 +- tests/foundation/traceflow/test_tcp.py | 177 ++++++++++++++++-- .../integration/test_traceflow_end_to_end.py | 29 ++- tests/toolkit/test_pyshark_unit.py | 4 +- 20 files changed, 571 insertions(+), 85 deletions(-) diff --git a/docs/source/pcapkit/foundation/traceflow/tcp.rst b/docs/source/pcapkit/foundation/traceflow/tcp.rst index 85973e1eb6..f00ccb3e45 100644 --- a/docs/source/pcapkit/foundation/traceflow/tcp.rst +++ b/docs/source/pcapkit/foundation/traceflow/tcp.rst @@ -16,6 +16,7 @@ TCP flows from a series of packets and connections. .. automethod:: dump .. automethod:: make_bufid .. automethod:: trace + .. automethod:: finish .. automethod:: submit .. autoattribute:: __protocol_name__ @@ -39,6 +40,7 @@ Terminology frame=frame.info, # extracted frame info syn=tcp.flags.syn, # TCP synchronise (SYN) flag fin=tcp.flags.fin, # TCP finish (FIN) flag + rst=tcp.flags.rst, # TCP reset (RST) flag src=ip.src, # source IP dst=ip.dst, # destination IP srcport=tcp.srcport, # TCP source port @@ -71,6 +73,7 @@ Terminology | |--> 'forward': (list) frame index sent by ``origin`` | |--> 'reverse': (list) frame index sent to ``origin`` | |--> 'fin': (set) endpoints seen to have sent a FIN + | |--> 'reset': (bool) whether a RST has been seen |--> (tuple) BUFID ... When tracing bidirectionally -- the default -- ``BUFID`` orders the two @@ -80,6 +83,18 @@ Terminology :class:`~pcapkit.corekit.infoclass.Info` cannot be one -- :class:`collections.abc.Mapping` sets its ``__hash__`` to :data:`None`. + A teardown -- a FIN from each endpoint, or a RST from either -- is recorded + in ``fin`` and ``reset`` but does **not** finalise the flow. The four-way + close of :rfc:`9293#section-3.6` is FIN, ACK, FIN, ACK, so the + acknowledgement that completes it arrives after the second FIN; finalising + on that FIN would drop the ACK from the flow and let it open a fresh buffer + under the same ``BUFID``, which a later connection reusing those endpoints + would then merge into. The flow is finalised instead by proof that nothing + more can arrive -- a new connection's SYN on the same endpoints, or + :meth:`TCP.finish ` at the end + of the capture. Telling that SYN from the peer's SYN-ACK is what the + recorded teardown is for. + .. seealso:: :class:`pcapkit.foundation.traceflow.data.tcp.Buffer` trace.tcp.index diff --git a/docs/source/pcapkit/foundation/traceflow/traceflow.rst b/docs/source/pcapkit/foundation/traceflow/traceflow.rst index 578c7652bd..47aca49463 100644 --- a/docs/source/pcapkit/foundation/traceflow/traceflow.rst +++ b/docs/source/pcapkit/foundation/traceflow/traceflow.rst @@ -44,6 +44,7 @@ which is an abstract base class for all flow tracing classes. .. automethod:: dump .. automethod:: trace + .. automethod:: finish .. automethod:: submit .. autoattribute:: __output__ diff --git a/docs/source/pep.rst b/docs/source/pep.rst index b14bc4d20e..d9603ddd33 100644 --- a/docs/source/pep.rst +++ b/docs/source/pep.rst @@ -594,26 +594,54 @@ Two smaller items in the same subsystem: * **Flow tracing is TCP only**, and blocked on the same generalisation -- :class:`~pcapkit.foundation.traceflow.TraceFlowManager` holds a single field, - so UDP, SCTP and IP conversation tracing have nowhere to go. The TCP tracer - also closes a flow on FIN but never on RST, which is not in - :class:`~pcapkit.foundation.traceflow.data.tcp.Packet` at all -- every toolkit - adapter would have to report the flag before the tracer could act on it. + so UDP, SCTP and IP conversation tracing have nowhere to go. It no longer *treats each direction of a connection as a separate flow*, - though. :meth:`TCP.make_bufid + though, and RST is no longer missing from + :class:`~pcapkit.foundation.traceflow.data.tcp.Packet`. :meth:`TCP.make_bufid ` orders the two endpoints canonically, so both halves of a conversation reduce to one buffer ID, one label and one output file, and :class:`~pcapkit.foundation.traceflow.data.tcp.Index` reports ``forward`` and ``reverse`` alongside ``index`` so per-direction ordering stays recoverable. - A flow now closes only once **both** halves have sent a FIN, since a - connection is not over while one direction is still sending. This is the - default; ``bidirectional=False`` (``trace_bidirectional=False`` on + This is the default; ``bidirectional=False`` + (``trace_bidirectional=False`` on :class:`~pcapkit.foundation.extraction.Extractor`, :func:`~pcapkit.interface.core.extract` and :func:`~pcapkit.interface.misc.follow_tcp_stream`) restores the older per-direction behaviour. + What ends a bidirectional flow is worth stating, because the obvious answer is + wrong. A teardown -- a FIN from each endpoint, or a RST from either -- is + *recorded* but does not finalise the flow: the four-way close of + :rfc:`9293#section-3.6` is FIN, ACK, FIN, ACK, so the final acknowledgement + arrives after the second FIN, and finalising on that FIN drops the ACK from the + flow and lets it open a fresh buffer under the same canonical buffer ID -- which + a later connection reusing those endpoints then merges into. Duplicates of that + ACK defeat any rule that tries to name the last packet of the exchange. So the + flow is finalised only by proof that nothing more can arrive: a new connection's + SYN on the same endpoints, or the end of the capture, via + :meth:`TraceFlow.finish + `. Distinguishing + that SYN from the peer's SYN-ACK is what the recorded teardown is for. + + What is still wanted here is **wiring the application layer into flow + tracing**. Reassembly analyses a datagram's payload lazily through + :class:`~pcapkit.foundation.reassembly.data.data.Deferred`; flow tracing + analyses nothing, because it buffers no payload at all -- its + :class:`~pcapkit.foundation.traceflow.data.tcp.Buffer` holds a dumper, frame + indices and a label. So this is not a parse to postpone but a capability to + add, and it needs a decision first: whether the tracer grows a payload buffer + per direction, or delegates to + :class:`~pcapkit.foundation.reassembly.tcp.TCP` the way + :func:`~pcapkit.interface.misc.follow_tcp_stream` already does. + + One case remains undecided rather than solved: a capture that *starts* in the + middle of a connection, sees no teardown, and then has its endpoints reused. The + reuse is indistinguishable from a continuation without the ACK flag on + :class:`~pcapkit.foundation.traceflow.data.tcp.Packet`, which would make + ``syn and not ack`` a definitive new-connection test on its own. + What is still wanted here is **wiring the application layer into flow tracing**. Reassembly analyses a datagram's payload lazily through :class:`~pcapkit.foundation.reassembly.data.data.Deferred`; flow tracing diff --git a/pcapkit/foundation/extraction.py b/pcapkit/foundation/extraction.py index f5ec9b289f..024f527e7b 100644 --- a/pcapkit/foundation/extraction.py +++ b/pcapkit/foundation/extraction.py @@ -1045,10 +1045,22 @@ def _cleanup(self) -> 'None': sets :attr:`self._flag_e ` as :data:`True` and closes the input file (if necessary). + It also tells the flow tracer the capture has ended, via + :meth:`TraceFlow.finish `. + That is the point at which a traced flow nothing has superseded can be + said to be over, so it is where such a flow is finalised and its callbacks + run. This method can be reached twice for one extraction -- the EOF path in + :meth:`_read_frame` and again from :meth:`run` -- so ``finish`` is required + to be idempotent rather than guarded here. + """ # pylint: disable=attribute-defined-outside-init logger.debug('cleaning up after %d frame(s) from %s', self._frnum, self._ifnm) self._flag_e = True + + if self._flag_t and self._tcp: + self._trace.tcp.finish() + if isinstance(self._ifile, SeekableReader): self._ifile.close() elif not self._flag_s: diff --git a/pcapkit/foundation/reassembly/ip.py b/pcapkit/foundation/reassembly/ip.py index a188aadba6..72ea6f061b 100644 --- a/pcapkit/foundation/reassembly/ip.py +++ b/pcapkit/foundation/reassembly/ip.py @@ -221,16 +221,43 @@ def submit(self, buf: 'Buffer[_AT]', *, bufid: 'tuple[_AT, _AT, int, TransType]' # if datagram is reassembled in whole -- or if it is not, and ``strict`` # asked for one contiguous payload rather than the received runs else: - # NOTE: ``max(TDL, 0)``, not ``TDL``. ``TDL`` is still its initial - # ``-1`` until the fragment with **MF** clear arrives, so a datagram - # whose final fragment never came reached this branch under - # ``strict=False`` and sliced ``datagram[:-1]`` -- handing back 65534 - # octets of the preallocated buffer, almost all of them zeros the - # sender never sent, and calling it complete. The length of such a - # datagram is simply not known, so there is nothing honest to report - # but an empty payload; ``strict=True``, the default, reports the runs - # that did arrive instead. - payload = bytes(datagram[:max(TDL, 0)]) + # ``TDL`` is the datagram's total length, and it is only known once the + # fragment with **MF** clear has arrived -- until then it is still its + # initial ``-1``. Which case this is decides how much of the buffer + # there is to report. + if TDL > 0: + # The length is known. Report it, holes and all: the gaps read as + # zeros, exactly as they do in the TCP reassembler's loose mode. + # This is unchanged behaviour, and the reason ``strict=False`` + # exists -- a caller who wants the gaps *marked* rather than + # zero-filled uses ``strict=True`` and gets the runs. + stop = TDL + else: + # The length is not known, and this is the case that used to slice + # ``datagram[:-1]`` -- handing back 65534 octets of the + # preallocated buffer, almost all of them zeros the sender never + # sent, and calling the result complete. + # + # Reporting nothing at all would be the other extreme, and it + # discards data that really did arrive. So report the **contiguous + # prefix**: every octet from offset zero up to the first hole. That + # is the longest run whose extent is known without knowing the + # total length, it is all genuinely received, and it is the part a + # caller asking for one contiguous payload can actually use -- + # a parser reading a payload from its start cannot use a run that + # begins after an unmeasured gap anyway. Anything past the first + # hole is still reported by ``strict=True``, which lists the runs + # precisely because their offsets cannot be conveyed in a blob. + # + # ``RCVBT`` records receipt in 8-octet units, so the prefix ends at + # the first clear bit. + received = 0 + for bit in RCVBT: + if not bit: + break + received += 1 + stop = received * 8 + payload = bytes(datagram[:stop]) packet = Datagram( completed=completion, id=DatagramID( diff --git a/pcapkit/foundation/traceflow/data/tcp.py b/pcapkit/foundation/traceflow/data/tcp.py index 5067f4eb78..d81de38bba 100644 --- a/pcapkit/foundation/traceflow/data/tcp.py +++ b/pcapkit/foundation/traceflow/data/tcp.py @@ -55,6 +55,12 @@ class Packet(Info, Generic[_AT]): syn: 'bool' #: TCP finish (FIN) flag. fin: 'bool' + #: TCP reset (RST) flag. A connection can end abruptly as well as politely + #: (:rfc:`9293#section-3.5.2`), and the tracer cannot notice that unless the + #: flag reaches it -- which it did not, so a reset connection used to look + #: merely idle and a later connection reusing the same endpoints merged into + #: it. + rst: 'bool' #: Source IP. src: '_AT' #: Destination IP. @@ -67,8 +73,7 @@ class Packet(Info, Generic[_AT]): timestamp: 'float' if TYPE_CHECKING: - def __init__(self, protocol: 'Enum_LinkType', index: 'int', frame: 'Data_Frame | dict[str, Any]', syn: 'bool', fin: 'bool', src: '_AT', dst: '_AT', - srcport: 'int', dstport: 'int', timestamp: 'float') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long + def __init__(self, protocol: 'Enum_LinkType', index: 'int', frame: 'Data_Frame | dict[str, Any]', syn: 'bool', fin: 'bool', rst: 'bool', src: '_AT', dst: '_AT', srcport: 'int', dstport: 'int', timestamp: 'float') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long @info_final @@ -105,6 +110,10 @@ class Buffer(Info, Generic[_AT]): #: than a single flag: submitting on the first FIN would cut the peer's FIN #: and the final acknowledgement out of the flow. fin: 'set[tuple[_AT, int]]' + #: Whether a TCP **RST** has been seen on this flow. A reset ends the + #: connection at once (:rfc:`9293#section-3.5.2`), where a polite close needs + #: a FIN from each side, so it is tracked as a flag rather than per endpoint. + reset: 'bool' if TYPE_CHECKING: # NOTE: one line, however long. ``# pylint: disable`` is *line*-scoped and @@ -113,7 +122,7 @@ class Buffer(Info, Generic[_AT]): # disable's reach -- which is why the shorter form this replaces leaked # three ``unused-argument`` messages of its own. Every other data model in # :mod:`pcapkit` writes these stubs on one line for the same reason. - def __init__(self, fpout: 'Dumper', index: 'list[int]', label: 'str', origin: 'tuple[_AT, int]', forward: 'list[int]', reverse: 'list[int]', fin: 'set[tuple[_AT, int]]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long + def __init__(self, fpout: 'Dumper', index: 'list[int]', label: 'str', origin: 'tuple[_AT, int]', forward: 'list[int]', reverse: 'list[int]', fin: 'set[tuple[_AT, int]]', reset: 'bool') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long @info_final diff --git a/pcapkit/foundation/traceflow/tcp.py b/pcapkit/foundation/traceflow/tcp.py index 00ebaab229..ca53768258 100644 --- a/pcapkit/foundation/traceflow/tcp.py +++ b/pcapkit/foundation/traceflow/tcp.py @@ -52,14 +52,23 @@ class TCP(TraceFlow[BufferID, 'Buffer[_AT]', Index, Packet[_AT]], Generic[_AT]): * The reverse half of a conversation no longer produces a flow of its own, so a capture of *n* connections yields *n* flows rather than ``2n``, and one output file each rather than two. - * A flow closes when **both** halves have sent a FIN rather than on the - first FIN seen, since a connection is not over while one direction is - still sending (:rfc:`9293#section-3.6`). A conversation whose reverse - half was never captured therefore stays open until - :meth:`submit` flushes it -- correctly, since nothing in the capture - shows the connection closing. - - Pass ``bidirectional=False`` for the older per-direction behaviour. + * **A teardown does not end a flow.** Seeing a connection close is not the + same as knowing nothing more will arrive on its endpoints: the four-way + close of :rfc:`9293#section-3.6` is FIN, ACK, FIN, ACK, so the + acknowledgement that completes it follows the second FIN, and duplicates + of that acknowledgement can follow in turn. A flow is therefore + finalised only by proof that no more of it can come -- a new + connection's SYN on the same endpoints, or the end of the capture + (:meth:`finish`). What the teardown does is get *recorded*, which is how + that SYN is told from the peer's SYN-ACK; see :meth:`_ended`. + + Finalising a flow is also when its callbacks run, so they run against + the whole conversation rather than a truncated one. + + Pass ``bidirectional=False`` for the older per-direction behaviour: a flow + is one direction, and closes on that direction's FIN. That mode reproduces + what flow tracing did before conversations became one flow, RST included -- + which is to say it ignores RST, as it always did. """ @@ -160,17 +169,27 @@ def trace(self, packet: 'Packet[_AT]', *, output: 'bool' = False) -> 'Dumper | s # Buffer Identifier -- canonical, hence direction-independent, when # tracing bidirectionally BUFID = self.make_bufid(packet) - # SYN = packet.syn # Synchronise Flag (Establishment) + SYN = packet.syn # Synchronise Flag (Establishment) FIN = packet.fin # Finish Flag (Termination) + RST = packet.rst # Reset Flag (Abrupt Termination) # the half of the conversation this packet was sent by END = (packet.src, packet.srcport) # type: tuple[_AT, int] - # # when SYN is set, reset buffer of this seesion - # if SYN and BUFID in self._buffer: - # temp = self._buffer.pop(BUFID) - # temp['fpout'] = (self._fproot, self._fdpext) - # temp['index'] = tuple(temp['index']) - # self._stream.append(Info(temp)) + # A SYN arriving on a flow whose teardown has already been observed is a + # *new* connection reusing the endpoints, and the only signal that proves + # the previous one can receive nothing further. It is what finalises a + # bidirectional flow mid-capture; everything else is flushed by + # :meth:`submit` at the end of the capture. + # + # The state test matters. A SYN alone does not mean "new connection" -- + # the peer's SYN-ACK carries the flag too, and it belongs to the flow the + # client's SYN just opened. Gating on the teardown having been seen tells + # the two apart without needing the ACK flag: a SYN-ACK cannot arrive + # after both endpoints have finished, or after a reset. + if self._bidir and BUFID in self._buffer and SYN and self._ended(self._buffer[BUFID]): + logger.debug('TCP flow %s superseded by a new connection on the same endpoints', + self._buffer[BUFID].label) + self._finalise(BUFID) # initialise buffer with BUFID if BUFID not in self._buffer: @@ -189,6 +208,7 @@ def trace(self, packet: 'Packet[_AT]', *, output: 'bool' = False) -> 'Dumper | s forward=[], reverse=[], fin=set(), + reset=False, ) # trace frame record @@ -202,44 +222,117 @@ def trace(self, packet: 'Packet[_AT]', *, output: 'bool' = False) -> 'Dumper | s buffer.reverse.append(packet.index) if FIN: buffer.fin.add(END) + if RST: + buffer.__update__(reset=True) fpout = buffer.fpout label = buffer.label - # when the session is over, submit its buffer + # A *unidirectional* flow is one direction, and a direction is over when + # its FIN goes out -- so that mode keeps closing on FIN exactly as it did. # - # NOTE: A bidirectional flow is a whole connection, and a connection is - # not finished while either direction still is -- so it takes a FIN from - # both endpoints, not the first FIN seen. Closing on the first would cut - # the peer's FIN and the final acknowledgement out of the flow, and they - # would then open a *second* flow under the same buffer ID: precisely the - # split this is here to remove. - closed = len(buffer.fin) >= 2 if self._bidir else FIN - if closed: - buf = self._buffer.pop(BUFID) - # fpout, label = buf['fpout'], buf['label'] - - logger.debug('TCP flow %s closed after %d frame(s) (%d forward, %d reverse)', - label, len(buf.index), len(buf.forward), len(buf.reverse)) - index = Index( - fpout=f'{self._fproot}/{label}{self._fdpext}' if self._fdpext is not None else None, - index=tuple(buf.index), - label=label, - forward=tuple(buf.forward), - reverse=tuple(buf.reverse), - ) - for callback in self.__callback_fn__: - callback(index) - self._stream.append(index) + # A bidirectional flow is a whole connection, and observing its teardown + # is not the same as knowing nothing more will arrive. The four-way close + # is FIN, ACK, FIN, ACK: the final acknowledgement comes *after* the + # second FIN, so submitting on the second FIN drops it from the flow and + # lets it open a fresh buffer under the same canonical buffer ID -- which + # a later connection reusing those endpoints then merges into. Closing on + # the *first* FIN has the same defect one packet earlier, and duplicate + # acknowledgements after the close would defeat any rule that tries to + # name the last packet of the exchange. + # + # So a teardown does not close the flow here at all: it is recorded, and + # the flow keeps accepting the packets that still belong to it. What + # finalises the flow is proof that no more can come -- a new connection's + # SYN on the same endpoints, handled above, or the end of the capture. + if not self._bidir and FIN: + self._finalise(BUFID) # return label or output object return fpout if output else label + @staticmethod + def _ended(buffer: 'Buffer[_AT]') -> 'bool': + """Whether a flow's connection has been seen to end. + + Arguments: + buffer: a flow buffer (:term:`trace.tcp.buffer`) + + Returns: + Whether a teardown was observed -- a **FIN from both endpoints**, the + polite close of :rfc:`9293#section-3.6`, or a **RST** from either, the + abrupt one of :rfc:`9293#section-3.5.2`. + + This is not the same as "nothing more will arrive on these endpoints": the + acknowledgement that completes a four-way close, and any duplicate of it, + still follow. It is used to tell a new connection's SYN from the peer's + SYN-ACK, which is a question the flag alone cannot answer. + + """ + return buffer.reset or len(buffer.fin) >= 2 + + def _finalise(self, bufid: 'BufferID') -> 'Index': + """Finalise a flow: report it, and stop tracing into it. + + Arguments: + bufid: buffer identifier of the flow to finalise + + Returns: + The flow's :term:`index ` entry. + + Called once per flow, from the one place that can prove a flow is over -- + a new connection superseding it, or :meth:`submit` at the end of the + capture. Registering the flow here rather than at teardown is what keeps + the acknowledgement that completes a close inside the flow it belongs to. + + """ + buf = self._buffer.pop(bufid) + label = buf.label + + logger.debug('TCP flow %s finalised after %d frame(s) (%d forward, %d reverse)', + label, len(buf.index), len(buf.forward), len(buf.reverse)) + index = Index( + fpout=f'{self._fproot}/{label}{self._fdpext}' if self._fdpext is not None else None, + index=tuple(buf.index), + label=label, + forward=tuple(buf.forward), + reverse=tuple(buf.reverse), + ) + for callback in self.__callback_fn__: + callback(index) + self._stream.append(index) + return index + + def finish(self) -> 'None': + """Finalise every flow still being traced. + + The end of the capture is the second of the two things that can prove a + bidirectional flow is over -- the first being a new connection on the same + endpoints. Draining the buffer here is what lets a flow keep the + acknowledgement that completes its close and *still* have its callback + fired, rather than having to choose between the two. + + Idempotent: it drains the buffer, so a second call finds nothing to do. + + """ + for bufid in list(self._buffer): + self._finalise(bufid) + self.__cached__['submit'] = None + def submit(self) -> 'tuple[Index, ...]': """Submit traced TCP flows. Returns: Traced TCP flow (:term:`trace.tcp.index`). + Note: + This reports flows still being traced **without** finalising them, so + that reading + :attr:`TraceFlow.index ` + part-way through a capture cannot disturb the tracing -- popping a + buffer there would strand the rest of its conversation in a second + flow. Such a flow is therefore reported here but has not fired its + callback; :meth:`finish` is what does that, at the end of the capture. + """ if (cached := self.__cached__.get('submit')) is not None: return cached diff --git a/pcapkit/foundation/traceflow/traceflow.py b/pcapkit/foundation/traceflow/traceflow.py index 384f5a060d..b38d0352ca 100644 --- a/pcapkit/foundation/traceflow/traceflow.py +++ b/pcapkit/foundation/traceflow/traceflow.py @@ -288,6 +288,28 @@ def submit(self) -> 'tuple[_IT, ...]': """ + def finish(self) -> 'None': + """Finalise every flow still being traced. + + Called by :meth:`Extractor._cleanup + ` once the capture has + been read to its end, which is the point at which a flow that was never + superseded can be said to be over. + + The base implementation does nothing, so a tracer that has no such notion + -- or an existing third-party subclass that predates this method -- keeps + working unchanged. :meth:`submit` must remain able to report a flow that + was never finalised, since nothing guarantees this is called: a tracer + driven directly rather than through an + :class:`~pcapkit.foundation.extraction.Extractor` never sees an end of + capture. + + Implementations must be **idempotent**: :meth:`Extractor._cleanup + ` can run more than once + for one extraction. + + """ + ########################################################################## # Data models. ########################################################################## diff --git a/pcapkit/toolkit/dpkt.py b/pcapkit/toolkit/dpkt.py index 9619066115..b5761909e9 100644 --- a/pcapkit/toolkit/dpkt.py +++ b/pcapkit/toolkit/dpkt.py @@ -332,6 +332,7 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, frame=packet2dict(packet, timestamp, data_link=data_link), # extracted packet syn=bool(int(flags[6])), # TCP synchronise (SYN) flag fin=bool(int(flags[7])), # TCP finish (FIN) flag + rst=bool(int(flags[5])), # TCP reset (RST) flag src=ipaddress.ip_address(ip.src), # source IP dst=ipaddress.ip_address(ip.dst), # destination IP srcport=tcp.sport, # TCP source port diff --git a/pcapkit/toolkit/pcap.py b/pcapkit/toolkit/pcap.py index ad2bc1c221..9c4e151235 100644 --- a/pcapkit/toolkit/pcap.py +++ b/pcapkit/toolkit/pcap.py @@ -220,6 +220,7 @@ def tcp_traceflow(frame: 'Frame', *, data_link: 'LinkType') -> 'TF_TCP_Packet | frame=frame.info, # extracted frame info syn=tcp_info.flags.syn, # TCP synchronise (SYN) flag fin=tcp_info.flags.fin, # TCP finish (FIN) flag + rst=tcp_info.flags.rst, # TCP reset (RST) flag src=ip_info.src, # source IP dst=ip_info.dst, # destination IP srcport=tcp_info.srcport.port, # TCP source port diff --git a/pcapkit/toolkit/pcapng.py b/pcapkit/toolkit/pcapng.py index d49828b6c7..9ffe05c742 100644 --- a/pcapkit/toolkit/pcapng.py +++ b/pcapkit/toolkit/pcapng.py @@ -228,6 +228,7 @@ def tcp_traceflow(frame: 'PCAPNG', *, nanosecond: 'bool' = False) -> 'TF_TCP_Pac # extracted frame info syn=tcp_info.flags.syn, # TCP synchronise (SYN) flag fin=tcp_info.flags.fin, # TCP finish (FIN) flag + rst=tcp_info.flags.rst, # TCP reset (RST) flag src=ip_info.src, # source IP dst=ip_info.dst, # destination IP srcport=tcp_info.srcport.port, # TCP source port diff --git a/pcapkit/toolkit/pypcapfile.py b/pcapkit/toolkit/pypcapfile.py index 437a655600..08b277ec94 100644 --- a/pcapkit/toolkit/pypcapfile.py +++ b/pcapkit/toolkit/pypcapfile.py @@ -448,6 +448,7 @@ def tcp_traceflow(packet: 'Packet', *, data_link: 'Enum_LinkType', frame=packet2dict(packet, data_link=data_link), # extracted packet syn=bool(tcp.syn), # TCP synchronise (SYN) flag fin=bool(tcp.fin), # TCP finish (FIN) flag + rst=bool(tcp.rst), # TCP reset (RST) flag src=ipaddress.IPv4Address(ipv4.src), # source IP dst=ipaddress.IPv4Address(ipv4.dst), # destination IP srcport=tcp.src_port, # TCP source port diff --git a/pcapkit/toolkit/pyshark.py b/pcapkit/toolkit/pyshark.py index ae5e893bf7..5ced8079f7 100644 --- a/pcapkit/toolkit/pyshark.py +++ b/pcapkit/toolkit/pyshark.py @@ -89,6 +89,7 @@ def tcp_traceflow(packet: 'Packet') -> 'TF_TCP_Packet | None': frame=packet2dict(packet), # extracted packet syn=bool(int(tcp.flags_syn)), # TCP synchronise (SYN) flag fin=bool(int(tcp.flags_fin)), # TCP finish (FIN) flag + rst=bool(int(tcp.flags_reset)), # TCP reset (RST) flag src=ipaddress.ip_address(ip.src), # source IP dst=ipaddress.ip_address(ip.dst), # destination IP srcport=int(tcp.srcport), # TCP source port diff --git a/pcapkit/toolkit/scapy.py b/pcapkit/toolkit/scapy.py index e9d2f79e29..38bf03d473 100644 --- a/pcapkit/toolkit/scapy.py +++ b/pcapkit/toolkit/scapy.py @@ -321,6 +321,7 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | No frame=packet2dict(packet), # extracted packet syn=bool(tcp.flags.S), # TCP synchronise (SYN) flag fin=bool(tcp.flags.F), # TCP finish (FIN) flag + rst=bool(tcp.flags.R), # TCP reset (RST) flag src=ipaddress.ip_address(ip.src), # source IP dst=ipaddress.ip_address(ip.dst), # destination IP srcport=tcp.sport, # TCP source port diff --git a/tests/foundation/reassembly/test_timeout.py b/tests/foundation/reassembly/test_timeout.py index 701f2bc3d1..21fbf5658f 100644 --- a/tests/foundation/reassembly/test_timeout.py +++ b/tests/foundation/reassembly/test_timeout.py @@ -222,6 +222,108 @@ def run() -> list: self.assertTrue(any(not completed for completed, _ in first)) +@unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') +class IPLooseModeTests(unittest.TestCase): + """``strict=False``, where IP reports one contiguous payload, not the runs. + + This branch had no test, and it is the one where the payload -- not merely the + ``completed`` flag -- changed: an unterminated datagram used to come back as + 65534 octets of preallocated buffer, reported *complete*. + """ + + def setUp(self) -> None: + purge_modules(['pcapkit']) + + def _packet(self, *, num: int, fo: int, mf: bool, payload: bytes): + from pcapkit.const.reg.transtype import TransType + from pcapkit.foundation.reassembly.data.ip import Packet + + src = ip_address('192.0.2.1') + dst = ip_address('198.51.100.2') + return Packet((src, dst, 42, TransType.UDP), num, fo, 20, mf, + 20 + len(payload), b'ip-header', bytearray(payload), T0) + + def test_an_unterminated_datagram_reports_the_prefix_that_arrived(self) -> None: + """No final fragment means no known length -- but the prefix is known. + + ``TDL`` is still ``-1`` here, and ``datagram[:-1]`` was 65534 octets of the + preallocated buffer with ``completed=True`` on top. Neither that nor an + empty payload is right: the octets from offset zero to the first hole did + genuinely arrive, so they are what a caller asking for one contiguous + payload gets, and the datagram is reported incomplete. + + """ + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + reasm = IPv4(strict=False) + # 16 octets at offset 0, then nothing -- MF set throughout, so the total + # length is never learnt + reasm(self._packet(num=1, fo=0, mf=True, payload=b'A' * 16)) + + datagram, = reasm.datagram + self.assertIs(datagram.completed, Completion.PARTIAL) + self.assertFalse(datagram.completed) + self.assertEqual(datagram.payload, b'A' * 16) + self.assertNotEqual(len(datagram.payload), 65534, 'the whole buffer came back') + + def test_the_prefix_stops_at_the_first_hole(self) -> None: + """A run after a gap cannot be placed in a blob, so it is not in one. + + Its offset is exactly what a contiguous payload cannot express; ``strict`` + mode lists the runs for that reason. Reporting the later run here would + misrepresent it as starting at the datagram's beginning. + + """ + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + reasm = IPv4(strict=False) + reasm(self._packet(num=1, fo=0, mf=True, payload=b'A' * 16)) + reasm(self._packet(num=2, fo=64, mf=True, payload=b'C' * 16)) + + datagram, = reasm.datagram + self.assertEqual(datagram.payload, b'A' * 16) + + # ... and strict mode, the default, reports both runs instead + strict = IPv4() + strict(self._packet(num=1, fo=0, mf=True, payload=b'A' * 16)) + strict(self._packet(num=2, fo=64, mf=True, payload=b'C' * 16)) + datagram, = strict.datagram + self.assertEqual(datagram.payload, (b'A' * 16, b'C' * 16)) + + def test_a_known_length_still_zero_fills_its_holes(self) -> None: + """With the last fragment in hand the length is known, so nothing changes. + + This is the case ``strict=False`` exists for, and its payload is exactly + what it always was -- only ``completed`` stopped claiming the datagram was + whole. + + """ + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + reasm = IPv4(strict=False) + # a 24-octet datagram missing its middle 8 octets + reasm(self._packet(num=1, fo=0, mf=True, payload=b'A' * 8)) + reasm(self._packet(num=2, fo=16, mf=False, payload=b'C' * 8)) + + datagram, = reasm.datagram + self.assertIs(datagram.completed, Completion.PARTIAL) + self.assertEqual(datagram.payload, b'A' * 8 + bytes(8) + b'C' * 8) + + def test_a_complete_datagram_is_unaffected_by_loose_mode(self) -> None: + from pcapkit.foundation.reassembly.data.data import Completion + from pcapkit.foundation.reassembly.ipv4 import IPv4 + + reasm = IPv4(strict=False) + reasm(self._packet(num=1, fo=0, mf=True, payload=b'A' * 8)) + reasm(self._packet(num=2, fo=8, mf=False, payload=b'B' * 8)) + + datagram, = reasm.datagram + self.assertIs(datagram.completed, Completion.COMPLETE) + self.assertEqual(datagram.payload, b'A' * 8 + b'B' * 8) + + @unittest.skipUnless(HAS_RUNTIME, 'runtime dependencies not installed') class TCPReassemblyTimeoutTests(unittest.TestCase): def setUp(self) -> None: diff --git a/tests/foundation/test_extraction.py b/tests/foundation/test_extraction.py index dc75f68246..5766caa2a2 100644 --- a/tests/foundation/test_extraction.py +++ b/tests/foundation/test_extraction.py @@ -103,8 +103,11 @@ def _bare_extractor(self): ipv6=types.SimpleNamespace(datagram=('ipv6',)), tcp=types.SimpleNamespace(datagram=('tcp',)), ) + # ``finish`` as well as ``index``: _cleanup tells the tracer the capture + # has ended, which is where a flow nothing superseded is finalised, so a + # stand-in for the tracer has to answer to it extractor._trace = types.SimpleNamespace( - tcp=types.SimpleNamespace(index=('trace',)), + tcp=types.SimpleNamespace(index=('trace',), finish=lambda: None), ) extractor._exeng = FakeEngine(extractor) extractor._magic = b'fake' diff --git a/tests/foundation/traceflow/data/test_models.py b/tests/foundation/traceflow/data/test_models.py index 49232d532e..9d29a5892b 100644 --- a/tests/foundation/traceflow/data/test_models.py +++ b/tests/foundation/traceflow/data/test_models.py @@ -24,20 +24,22 @@ def test_tcp_traceflow_data_models_and_package_aliases(self) -> None: src = ip_address('2001:db8::1') dst = ip_address('2001:db8::2') frame = {'frame': 1} - packet = Packet(LinkType.ETHERNET, 1, frame, True, False, src, dst, 12345, 443, 1.5) + packet = Packet(LinkType.ETHERNET, 1, frame, True, False, False, src, dst, 12345, 443, 1.5) self.assertIsInstance(packet, TCP_Packet) self.assertEqual(packet.src, src) self.assertEqual(packet.frame, frame) + self.assertFalse(packet.rst) dumper = object() origin = (src, 12345) buffer = Buffer(dumper, [1, 2], '2001_db8_1-12345_2001_db8_2-443', - origin, [1], [2], {origin}) + origin, [1], [2], {origin}, False) self.assertIsInstance(buffer, TCP_Buffer) self.assertEqual(buffer.fpout, dumper) self.assertEqual(buffer.origin, origin) self.assertEqual((buffer.forward, buffer.reverse), ([1], [2])) self.assertEqual(buffer.fin, {origin}) + self.assertFalse(buffer.reset) index = Index('/tmp/flow.json', (1, 2), buffer.label, (1,), (2,)) self.assertIsInstance(index, TCP_Index) diff --git a/tests/foundation/traceflow/test_tcp.py b/tests/foundation/traceflow/test_tcp.py index 73c87a2527..112937dea1 100644 --- a/tests/foundation/traceflow/test_tcp.py +++ b/tests/foundation/traceflow/test_tcp.py @@ -19,14 +19,15 @@ def setUp(self) -> None: def _packet(self, *, index: int, src: str = '192.0.2.1', dst: str = '198.51.100.2', srcport: int = 12345, dstport: int = 443, syn: bool = False, - fin: bool = False, timestamp: float = 1.25, frame: object | None = None): + fin: bool = False, rst: bool = False, timestamp: float = 1.25, + frame: object | None = None): from pcapkit.const.reg.linktype import LinkType from pcapkit.foundation.traceflow.data.tcp import Packet if frame is None: frame = {'frame': index} - return Packet(LinkType.ETHERNET, index, frame, syn, fin, ip_address(src), ip_address(dst), - srcport, dstport, timestamp) + return Packet(LinkType.ETHERNET, index, frame, syn, fin, rst, ip_address(src), + ip_address(dst), srcport, dstport, timestamp) def test_tcp_trace_ipv4_fin_submit_cache_callback_and_dump(self) -> None: """One direction, traced with ``bidirectional=False``. @@ -101,13 +102,16 @@ def test_both_halves_of_a_connection_are_one_flow(self) -> None: self.assertEqual(flow.forward, (1,)) self.assertEqual(flow.reverse, (2,)) - def test_a_conversation_closes_only_once_both_halves_have_finished(self) -> None: - """One FIN is half a teardown, so it must not close the flow. + def test_a_four_way_close_keeps_its_final_acknowledgement(self) -> None: + """The whole close belongs to the flow, the last ACK included. - Closing on the first FIN would cut the peer's FIN and the final - acknowledgement out of the flow -- and they would then open a *second* - flow under the same buffer ID, which is the split bidirectional tracing - exists to remove. + A four-way close is FIN, ACK, FIN, ACK, so the final acknowledgement + arrives *after* the second FIN. Submitting the flow on the second FIN -- + or, worse, on the first -- drops that ACK from it and lets the ACK open a + fresh buffer under the same canonical buffer ID, which + :meth:`test_a_reused_port_pair_does_not_join_the_closed_connection` shows + a later connection then merges into. So a teardown records itself and + finalises nothing. """ from pcapkit.dumpkit.null import NotImplementedIO @@ -129,14 +133,159 @@ def test_a_conversation_closes_only_once_both_halves_have_finished(self) -> None trace.trace(self._packet(index=4, fin=True, timestamp=1.8)) self.assertEqual(trace._stream, []) - # now the server finishes too + # the server's FIN completes the exchange, but not the conversation: + # its acknowledgement is still to come, so nothing is finalised yet trace.trace(self._reply(index=5, fin=True, timestamp=2.0)) - self.assertEqual(len(trace._buffer), 0) - flow, = trace._stream - self.assertEqual(flow.index, (1, 2, 3, 4, 5)) - self.assertEqual(flow.forward, (1, 3, 4)) + self.assertEqual(len(trace._buffer), 1) + self.assertEqual(trace._stream, []) + + # ... and here it is, in the flow where it belongs + trace.trace(self._packet(index=6, timestamp=2.25)) + # ... as is a duplicate of it, which no rule naming "the last packet + # of the exchange" could have accommodated + trace.trace(self._packet(index=7, timestamp=2.5)) + + flow, = trace.index + self.assertEqual(flow.index, (1, 2, 3, 4, 5, 6, 7)) + self.assertEqual(flow.forward, (1, 3, 4, 6, 7)) self.assertEqual(flow.reverse, (2, 5)) + def test_a_reused_port_pair_does_not_join_the_closed_connection(self) -> None: + """A new connection on the same endpoints is a second flow. + + This is the defect the review found. With the flow submitted on the second + FIN, the final ACK arrived after the buffer had been popped and opened a + stray one-packet buffer under the same canonical buffer ID; the next + connection to reuse the endpoints then merged into that stray buffer, and + two unrelated connections came back as one flow. + + A SYN is what proves the previous connection can receive nothing further, + so it is what finalises the old flow and starts a new one. + + """ + from pcapkit.dumpkit.null import NotImplementedIO + from pcapkit.foundation.traceflow.tcp import TCP + + TCP.register_dumper('unit-null', NotImplementedIO, '.unit') + callbacks = [] + TCP.register_callback(callbacks.append) + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unit-null') + + # connection one: handshake, data, and a full four-way close + trace.trace(self._packet(index=1, syn=True, timestamp=1.0)) + trace.trace(self._reply(index=2, syn=True, timestamp=1.1)) + trace.trace(self._packet(index=3, fin=True, timestamp=1.2)) + trace.trace(self._reply(index=4, timestamp=1.3)) + trace.trace(self._reply(index=5, fin=True, timestamp=1.4)) + trace.trace(self._packet(index=6, timestamp=1.5)) + + # connection two: the very same address and port pair, reused. Its SYN + # is what finalises the first flow, so that is where the first flow's + # callback fires -- carrying the whole conversation, final ACK + # included, rather than a truncated one. + fired = len(callbacks) + trace.trace(self._packet(index=7, syn=True, timestamp=9.0)) + self.assertEqual(len(callbacks), fired + 1) + self.assertEqual(callbacks[-1].index, (1, 2, 3, 4, 5, 6)) + + trace.trace(self._reply(index=8, syn=True, timestamp=9.1)) + + # ``finish`` is what Extractor._cleanup calls at the end of a capture; + # after it every flow has been finalised, so ``index`` reports them in + # the order they closed rather than open-buffers-first + trace.finish() + first, second = trace.index + self.assertEqual(first.index, (1, 2, 3, 4, 5, 6), + 'the second connection joined the first') + self.assertEqual(second.index, (7, 8)) + self.assertNotEqual(first.label, second.label) + + def test_the_peers_syn_ack_joins_the_flow_rather_than_splitting_it(self) -> None: + """A SYN-ACK carries SYN too, and must not be read as a new connection. + + Which is why the rule is gated on the teardown having been seen: a SYN-ACK + cannot arrive after both endpoints have finished, or after a reset. Without + that gate every connection whose handshake was captured would split in two + at its second frame. + + """ + from pcapkit.foundation.traceflow.tcp import TCP + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unknown-unit-format') + trace.trace(self._packet(index=1, syn=True, timestamp=1.0)) + trace.trace(self._reply(index=2, syn=True, timestamp=1.1)) + + flow, = trace.index + self.assertEqual(flow.index, (1, 2)) + + def test_a_reset_ends_the_connection_as_a_close_does(self) -> None: + """RST is the other way a connection ends, and was not modelled at all. + + Without the flag reaching the tracer a reset connection looked merely + idle, so a later connection reusing the endpoints merged into it -- the + same contamination as the FIN case, by a different route. + + """ + from pcapkit.foundation.traceflow.tcp import TCP + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unknown-unit-format') + trace.trace(self._packet(index=1, syn=True, timestamp=1.0)) + trace.trace(self._reply(index=2, rst=True, timestamp=1.1)) + + # the reset is recorded but does not itself finalise the flow + self.assertEqual(len(trace._buffer), 1) + bufid, = trace._buffer + self.assertTrue(trace._buffer[bufid].reset) + + # a new connection on the same endpoints is a second flow + trace.trace(self._packet(index=3, syn=True, timestamp=9.0)) + trace.finish() + first, second = trace.index + self.assertEqual(first.index, (1, 2)) + self.assertEqual(second.index, (3,)) + + def test_a_half_open_connection_is_still_reported_at_end_of_capture(self) -> None: + """A conversation with no teardown must not simply vanish. + + Nothing supersedes it, so ``finish`` -- which + :meth:`Extractor._cleanup ` + calls at the end of the capture -- is what finalises it and fires its + callback. ``submit`` reports it either way, so that reading ``index`` + part-way through a capture is not destructive. + + """ + from pcapkit.foundation.traceflow.tcp import TCP + + callbacks = [] + TCP.register_callback(callbacks.append) + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unknown-unit-format') + trace.trace(self._packet(index=1, syn=True, timestamp=1.0)) + trace.trace(self._reply(index=2, timestamp=1.1)) + + # reported, but not finalised -- and reading it changed nothing + flow, = trace.index + self.assertEqual(flow.index, (1, 2)) + self.assertEqual(len(trace._buffer), 1) + self.assertEqual(trace._stream, []) + + before = len(callbacks) + trace.finish() + self.assertEqual(len(trace._buffer), 0) + flow, = trace.index + self.assertEqual(flow.index, (1, 2)) + self.assertEqual(len(callbacks), before + 1) + + # idempotent: Extractor._cleanup can run twice for one extraction + trace.finish() + self.assertEqual(len(callbacks), before + 1) + self.assertEqual(len(trace.index), 1) + def test_the_buffer_id_is_canonical_and_stays_a_tuple(self) -> None: """Both directions reduce to the same key, whichever is seen first. diff --git a/tests/integration/test_traceflow_end_to_end.py b/tests/integration/test_traceflow_end_to_end.py index 1b1b2bff8f..7e6652c07d 100644 --- a/tests/integration/test_traceflow_end_to_end.py +++ b/tests/integration/test_traceflow_end_to_end.py @@ -110,15 +110,30 @@ def test_every_frame_lands_in_exactly_one_flow(self) -> None: flows = extractor.trace.tcp self.assertEqual(extractor.length, 1117) - # 220 connections, not the 331 directions they are made of: 111 of them - # were captured both ways and 109 only one way, and 2 * 111 + 109 == 331, - # so the drop is the two halves of a connection meeting in one flow and - # not frames going missing -- which is what the index check below pins - self.assertEqual(len(flows), 220) + # 111 connections, not the 331 directions they are made of. Every one of + # them was captured both ways, so keying a flow on the pair of endpoints + # rather than on (source, destination) halves the count; the odd frame is + # the two connections that carry more than the usual ten. + # + # 220 would be the answer if a flow were submitted on the second FIN of + # its four-way close: the final acknowledgement then arrives after the + # buffer has been popped and forms a flow of its own, so each conversation + # comes back as a nine-frame flow plus a one-frame tail -- 109 of them + # here, which is what the review caught. The assertions below are what + # would notice it again: a stray tail is a single-frame flow, and it is a + # flow a later connection reusing those endpoints can be absorbed into. + self.assertEqual(len(flows), 111) + self.assertEqual(sum(1 for flow in flows if len(flow.index) == 1), 0, + 'a flow of one frame is a stray tail, not a conversation') + self.assertEqual(sum(1 for flow in flows if not flow.reverse), 0) indexed = [number for flow in flows for number in flow.index] self.assertEqual(len(indexed), 1117) self.assertEqual(sorted(indexed), list(range(1, 1118))) + # and each flow's own index is its two halves merged, so the split into + # directions cannot lose or duplicate a frame either + for flow in flows: + self.assertEqual(flow.index, tuple(sorted(flow.forward + flow.reverse))) def test_one_report_is_written_per_flow(self) -> None: extractor = self.extract(fin=sample_path('http.pcap'), nofile=True, store=False, @@ -128,9 +143,9 @@ def test_one_report_is_written_per_flow(self) -> None: written = {entry.name for entry in self.tmp_path.joinpath('trace').iterdir()} self.assertEqual(written, {f'{flow.label}.json' for flow in flows}) - self.assertEqual(len(written), 220) + self.assertEqual(len(written), 111) - # Spot-check the longest flow rather than re-reading all 220 reports. + # Spot-check the longest flow rather than re-reading all 111 reports. longest = max(flows, key=lambda flow: len(flow.index)) report = read_json(longest.fpout) self.assertEqual(list(report), [f'Frame {number}' for number in longest.index]) diff --git a/tests/toolkit/test_pyshark_unit.py b/tests/toolkit/test_pyshark_unit.py index 1a276f19ad..1dccdc76c8 100644 --- a/tests/toolkit/test_pyshark_unit.py +++ b/tests/toolkit/test_pyshark_unit.py @@ -26,8 +26,10 @@ def __init__(self, *, ipv6: bool = False, tcp: bool = True, self.layers = [ FakeLayer('ethernet', src='aa:aa:aa:aa:aa:aa'), FakeLayer('ip', src='192.0.2.1', dst='198.51.100.1'), + # ``flags_reset`` is PyShark's spelling of Wireshark's + # ``tcp.flags.reset``, the field the RST flag comes from FakeLayer('tcp', srcport='1234', dstport='80', - flags_syn='1', flags_fin='0'), + flags_syn='1', flags_fin='0', flags_reset='0'), ] self._contains = set() if ip: From 21e61189f041a9ad747a36dcf6ba4b4d7b12ac5b Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 17 Sep 2026 14:21:28 -0400 Subject: [PATCH 4/8] docs: drop backticks from inside the buffer diagram, where they render verbatim A ``code`` span is not markup inside a ``.. code-block:: text``, so the two lines added for 'forward' and 'reverse' were showing their backticks in the rendered page. Nothing else in that diagram uses them. --- docs/source/pcapkit/foundation/traceflow/tcp.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/pcapkit/foundation/traceflow/tcp.rst b/docs/source/pcapkit/foundation/traceflow/tcp.rst index f00ccb3e45..6041b3e67d 100644 --- a/docs/source/pcapkit/foundation/traceflow/tcp.rst +++ b/docs/source/pcapkit/foundation/traceflow/tcp.rst @@ -70,8 +70,8 @@ Terminology | | that opened the flow | |--> 'origin': (tuple) (address, port) of the endpoint | | that opened the flow - | |--> 'forward': (list) frame index sent by ``origin`` - | |--> 'reverse': (list) frame index sent to ``origin`` + | |--> 'forward': (list) frame index sent by 'origin' + | |--> 'reverse': (list) frame index sent to 'origin' | |--> 'fin': (set) endpoints seen to have sent a FIN | |--> 'reset': (bool) whether a RST has been seen |--> (tuple) BUFID ... From a90db24521c6b910fa8a9e8120af85240cab46e8 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 17 Sep 2026 15:15:23 -0400 Subject: [PATCH 5/8] docs: drop a duplicated paragraph from the flow-tracing bullet --- docs/source/pep.rst | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/source/pep.rst b/docs/source/pep.rst index d9603ddd33..53d5918366 100644 --- a/docs/source/pep.rst +++ b/docs/source/pep.rst @@ -625,17 +625,6 @@ Two smaller items in the same subsystem: `. Distinguishing that SYN from the peer's SYN-ACK is what the recorded teardown is for. - What is still wanted here is **wiring the application layer into flow - tracing**. Reassembly analyses a datagram's payload lazily through - :class:`~pcapkit.foundation.reassembly.data.data.Deferred`; flow tracing - analyses nothing, because it buffers no payload at all -- its - :class:`~pcapkit.foundation.traceflow.data.tcp.Buffer` holds a dumper, frame - indices and a label. So this is not a parse to postpone but a capability to - add, and it needs a decision first: whether the tracer grows a payload buffer - per direction, or delegates to - :class:`~pcapkit.foundation.reassembly.tcp.TCP` the way - :func:`~pcapkit.interface.misc.follow_tcp_stream` already does. - One case remains undecided rather than solved: a capture that *starts* in the middle of a connection, sees no teardown, and then has its endpoints reused. The reuse is indistinguishable from a continuation without the ACK flag on From 7ce8cdab773bb9bc073a7017089146d2e30f843a Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 17 Sep 2026 17:13:54 -0400 Subject: [PATCH 6/8] traceflow: wire the application layer in; stop the DPKT engine discarding timestamps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five review comments on #435. ## The DPKT engine was throwing the timestamp away (substantive) The fix was not where the comment sat. DPKT's reader yields ``(timestamp, bytes)`` and only the octets became a packet, so a *stored* frame did not know when it was captured and ``follow_tcp_stream`` had nothing to pass its reassembler but a bound ``functools.partial(timestamp=0.0)``. The timestamp was never unavailable -- it was being discarded at the engine. ``DPKT.read_frame`` now attaches it (``pcapkit.toolkit.dpkt.attach_timestamp``, read back by ``packet2timestamp``, which raises rather than defaulting for a frame that never came through the engine), the partial is gone, and the NOTE that asserted DPKT "cannot read a frame's capture timestamp off the frame" -- the framing that led to the wrong fix -- is corrected here and in the three adapter docstrings that repeated it. **No behaviour change**, as expected: TCP reassembly has no timeout by default, so the value reaches no decision. ``follow_tcp_stream`` on the DPKT engine returns the same streams and conversations as the default engine, which is now pinned by a test; the timestamps attached to ``in.pcap`` are the capture's own (1511106545.471719 …), not zeros. ## The application layer is wired into flow tracing (scope addition) Previously declined on the grounds that traceflow buffers no payload so there is no second parse to postpone. That is true, which is why this is a capability rather than a deferral -- and of the two designs the page posed, the tracer **delegates to** ``reassembly.tcp.TCP`` rather than growing a payload buffer of its own: a buffer concatenating payloads in capture order is silently wrong on the first retransmission or reordered segment, where RFC 815's hole-descriptor algorithm already in the reassembler is not. A test delivers segments out of order and asserts sequence order comes back. So the traceflow packet carries the four segment fields that reassembler needs (``seq``, ``ack``, ``header``, ``payload``) from exactly where each engine's sibling ``tcp_reassembly`` adapter already reads them, each flow owns a reassembler, and ``Index.packet`` postpones twice: reading it flushes the flow's reassembler, and each datagram's own ``packet`` is parsed later still -- the same ``Deferred`` arrangement as the reassembly side, mirrored in ``traceflow/data/data.py`` for the same reason those two modules already mirror each other. **Opt-in** (``analyse=True``, ``trace_analyse=True`` on Extractor, extract() and follow_tcp_stream): buffering every traced payload is a cost tracing does not otherwise pay, and tracing's per-packet cost is something this package has deliberately driven down. Off by default nothing is buffered, reassembled or parsed, and flow counts are unchanged. Refused on the ``pyshark`` engine, which reports dissected fields rather than octets -- the same reason it has no reassembly adapter. ## Completion uses StrEnum (question) ``pcapkit.utilities.compat.StrEnum``, as ``httpv1.Type`` and ``pcapng.TLSKeyLabel`` do. Verified on both the stdlib and aenum paths that it keeps every property the contract needs: only COMPLETE truthy, ``== True`` and ``== False`` both false. It adds ``json.dumps`` support -- a plain Enum raises TypeError, and ``to_dict()`` hands the field straight out -- and ``completed == 'timeout'``. The one cost is documented in a Warning: a non-empty string that tests false, so ``bool(x)`` and ``bool(str(x))`` disagree. The redundant ``__str__`` is gone, StrEnum already giving it. ## Quoting and alignment (questions) ``Buffer[_AT]`` unquoted, matching ``Packet[_AT]`` beside it and the sibling ``reassembly.ip.IP``. Comment columns re-aligned in both reassembly modules, where ``TS = info.timestamp`` had pushed its own comment a column right of the rest. Also silences, on both DeferredPacket mixins, the mypy ``[misc]`` and pylint ``no-member`` false positives about ``super()`` calls undefined on a mixin -- pylint rates those as *errors*. Suite 930 passed / 17 skipped / 929 subtests, 0 failed; 947 collected against 916 on origin/main, the difference being this branch's own tests. All 15 sample captures give byte-identical tree, json and reassembly output against origin/main and still regenerate byte-identically; 1479 datagrams, all COMPLETE. Flow counts unchanged at 122 bidirectional / 355 unidirectional, 1222 traced frames either way. mypy 124 against 128; pylint 4712 against 4723. --- .../pcapkit/foundation/traceflow/tcp.rst | 30 +++- docs/source/pep.rst | 38 +++-- pcapkit/foundation/engines/dpkt.py | 10 +- pcapkit/foundation/extraction.py | 25 +++- pcapkit/foundation/reassembly/data/data.py | 44 ++++-- pcapkit/foundation/reassembly/ip.py | 10 +- pcapkit/foundation/reassembly/tcp.py | 12 +- pcapkit/foundation/traceflow/data/data.py | 130 +++++++++++++++++- pcapkit/foundation/traceflow/data/tcp.py | 46 ++++++- pcapkit/foundation/traceflow/tcp.py | 117 +++++++++++++++- pcapkit/foundation/traceflow/traceflow.py | 17 ++- pcapkit/interface/core.py | 6 +- pcapkit/interface/misc.py | 46 ++++--- pcapkit/toolkit/dpkt.py | 74 +++++++++- pcapkit/toolkit/pcap.py | 4 + pcapkit/toolkit/pcapng.py | 4 + pcapkit/toolkit/pypcapfile.py | 5 + pcapkit/toolkit/pyshark.py | 9 ++ pcapkit/toolkit/scapy.py | 4 + .../foundation/reassembly/data/test_models.py | 33 +++++ .../foundation/traceflow/data/test_models.py | 15 +- tests/foundation/traceflow/test_tcp.py | 90 +++++++++++- tests/toolkit/test_dpkt_unit.py | 68 +++++++++ tests/toolkit/test_pyshark_unit.py | 6 +- 24 files changed, 762 insertions(+), 81 deletions(-) diff --git a/docs/source/pcapkit/foundation/traceflow/tcp.rst b/docs/source/pcapkit/foundation/traceflow/tcp.rst index 6041b3e67d..03408938ef 100644 --- a/docs/source/pcapkit/foundation/traceflow/tcp.rst +++ b/docs/source/pcapkit/foundation/traceflow/tcp.rst @@ -41,6 +41,11 @@ Terminology syn=tcp.flags.syn, # TCP synchronise (SYN) flag fin=tcp.flags.fin, # TCP finish (FIN) flag rst=tcp.flags.rst, # TCP reset (RST) flag + seq=tcp.seq, # TCP sequence number + ack=tcp.ack, # TCP acknowledgement number + header=tcp.packet.header, # raw bytes type header + payload=bytearray( + tcp.packet.payload), # raw bytearray type payload src=ip.src, # source IP dst=ip.dst, # destination IP srcport=tcp.srcport, # TCP source port @@ -74,6 +79,9 @@ Terminology | |--> 'reverse': (list) frame index sent to 'origin' | |--> 'fin': (set) endpoints seen to have sent a FIN | |--> 'reset': (bool) whether a RST has been seen + | |--> 'reassembly': (Optional[TCP]) the flow's own + | reassembler, or None when + | analyse is off |--> (tuple) BUFID ... When tracing bidirectionally -- the default -- ``BUFID`` orders the two @@ -115,7 +123,9 @@ Terminology | |--> 'forward': (tuple) frame index in the direction that | | opened the flow | |--> 'reverse': (tuple) frame index the other way; empty when - | tracing unidirectionally + | | tracing unidirectionally + | |--> 'packet': (Optional[tuple]) one reassembled datagram per + | direction, or None when analyse is off |--> (Info) data ... ``forward`` and ``reverse`` partition ``index``, so @@ -123,6 +133,16 @@ Terminology taking the label apart. ``forward`` is the direction of the packet that opened the flow, whose endpoints the label names first. + ``packet`` is the conversation's application layer: one reassembled datagram + per direction, present only when the tracer was constructed with + ``analyse=True``. It is reassembled on the *first read*, and each datagram's + own :attr:`~pcapkit.foundation.reassembly.data.tcp.Datagram.packet` is + parsed later still, so a caller that wanted only frame numbers pays for + neither. The tracer does not reassemble the stream itself -- it feeds + :class:`~pcapkit.foundation.reassembly.tcp.TCP`, whose :rfc:`815` algorithm + handles the reordering and retransmission that concatenating payloads in + capture order would corrupt. + .. seealso:: :class:`pcapkit.foundation.traceflow.data.tcp.Index` Data Structures @@ -145,6 +165,14 @@ Data Structures :members: :show-inheritance: +.. autoclass:: pcapkit.foundation.traceflow.data.data.Deferred + :members: + :show-inheritance: + +.. autoclass:: pcapkit.foundation.traceflow.data.data.DeferredPacket + :members: + :show-inheritance: + Type Variables -------------- diff --git a/docs/source/pep.rst b/docs/source/pep.rst index 53d5918366..28cc7515f6 100644 --- a/docs/source/pep.rst +++ b/docs/source/pep.rst @@ -631,16 +631,34 @@ Two smaller items in the same subsystem: :class:`~pcapkit.foundation.traceflow.data.tcp.Packet`, which would make ``syn and not ack`` a definitive new-connection test on its own. - What is still wanted here is **wiring the application layer into flow - tracing**. Reassembly analyses a datagram's payload lazily through - :class:`~pcapkit.foundation.reassembly.data.data.Deferred`; flow tracing - analyses nothing, because it buffers no payload at all -- its - :class:`~pcapkit.foundation.traceflow.data.tcp.Buffer` holds a dumper, frame - indices and a label. So this is not a parse to postpone but a capability to - add, and it needs a decision first: whether the tracer grows a payload buffer - per direction, or delegates to - :class:`~pcapkit.foundation.reassembly.tcp.TCP` the way - :func:`~pcapkit.interface.misc.follow_tcp_stream` already does. + **The application layer is wired into flow tracing** as well, though it is a + capability rather than a parse to postpone: flow tracing buffered no payload at + all, so there was no second parse to defer. Of the two ways of getting one, the + tracer **delegates to** + :class:`~pcapkit.foundation.reassembly.tcp.TCP` rather than growing a + per-direction payload buffer of its own. A buffer that concatenated payloads in + capture order would be silently wrong on the first retransmission or reordered + segment, where the :rfc:`815` hole-descriptor algorithm already in the + reassembler is not -- so + :class:`~pcapkit.foundation.traceflow.data.tcp.Packet` carries the four segment + fields (``seq``, ``ack``, ``header``, ``payload``) that reassembler needs, and + the tracer hands each traced segment straight to it. + + :attr:`Index.packet ` then + holds one reassembled datagram per direction, and postpones twice: reading it is + what flushes the flow's reassembler, and each datagram's own + :attr:`~pcapkit.foundation.reassembly.data.tcp.Datagram.packet` is parsed later + still, through the same + :class:`~pcapkit.foundation.reassembly.data.data.Deferred` arrangement the + reassembly side uses. It is **opt-in** -- ``analyse=True``, or + ``trace_analyse=True`` on :class:`~pcapkit.foundation.extraction.Extractor`, + :func:`~pcapkit.interface.core.extract` and + :func:`~pcapkit.interface.misc.follow_tcp_stream` -- because buffering every + traced payload is a cost tracing does not otherwise pay, and tracing's + per-packet cost is something this package has deliberately driven down. It is + unavailable on the ``pyshark`` engine, which reports dissected fields rather + than the octets behind them, and which for the same reason has no reassembly + adapter at all; asking for it there warns and falls back. * **Timing a partial datagram out** is implemented, for IP. :meth:`Reassembly.expire ` abandons a diff --git a/pcapkit/foundation/engines/dpkt.py b/pcapkit/foundation/engines/dpkt.py index 19387b0a71..6349221073 100644 --- a/pcapkit/foundation/engines/dpkt.py +++ b/pcapkit/foundation/engines/dpkt.py @@ -144,8 +144,8 @@ def read_frame(self) -> 'DPKTPacket': for more operational information. """ - from pcapkit.toolkit.dpkt import (ipv4_reassembly, ipv6_reassembly, packet2dict, - tcp_reassembly, tcp_traceflow) + from pcapkit.toolkit.dpkt import (attach_timestamp, ipv4_reassembly, ipv6_reassembly, + packet2dict, tcp_reassembly, tcp_traceflow) ext = self._extractor reader = self._extmp @@ -156,6 +156,12 @@ def read_frame(self) -> 'DPKTPacket': protocol = self._get_protocol(linktype) packet = protocol(pkt) # type: DPKTPacket + # DPKT hands the record's timestamp back beside its octets and only the + # octets become a packet, so the frame would otherwise not know when it was + # captured -- and anything reading a *stored* frame after this loop has + # moved on could not find out. Keep the two together from the start. + attach_timestamp(packet, timestamp) + # verbose output ext._frnum += 1 ext._vfunc(ext, packet) diff --git a/pcapkit/foundation/extraction.py b/pcapkit/foundation/extraction.py index 024f527e7b..45f0934370 100644 --- a/pcapkit/foundation/extraction.py +++ b/pcapkit/foundation/extraction.py @@ -38,8 +38,8 @@ from pcapkit.utilities.exceptions import (CallableError, FileNotFound, FormatError, IterableError, RegistryError, UnsupportedCall, stacklevel) from pcapkit.utilities.logging import get_logger -from pcapkit.utilities.warnings import (EngineWarning, ExtractionWarning, FormatWarning, - RegistryWarning, warn) +from pcapkit.utilities.warnings import (AttributeWarning, EngineWarning, ExtractionWarning, + FormatWarning, RegistryWarning, warn) if TYPE_CHECKING: from io import BufferedReader @@ -713,7 +713,7 @@ def __init__(self, reasm_timeout: 'Optional[float]' = None, # reassembly settings # pylint: disable=line-too-long trace: 'bool' = False, trace_fout: 'Optional[str]' = None, trace_format: 'Optional[Formats]' = None, # trace settings # pylint: disable=line-too-long trace_byteorder: 'Literal["big", "little"]' = sys.byteorder, trace_nanosecond: 'bool' = False, # trace settings # pylint: disable=line-too-long - trace_bidirectional: 'bool' = True, # trace settings # pylint: disable=line-too-long + trace_bidirectional: 'bool' = True, trace_analyse: 'bool' = False, # trace settings # pylint: disable=line-too-long ip: 'bool' = False, ipv4: 'bool' = False, ipv6: 'bool' = False, tcp: 'bool' = False, # reassembly/trace settings # pylint: disable=line-too-long buffer_size: 'int' = io.DEFAULT_BUFFER_SIZE, buffer_save: 'bool' = False, buffer_path: 'Optional[str]' = None, # buffer settings # pylint: disable=line-too-long no_eof: 'bool' = False, # EOF settings # pylint: disable=line-too-long @@ -759,6 +759,11 @@ def __init__(self, trace_bidirectional: whether both halves of a conversation are traced as one flow, which is the default; :data:`False` restores the older behaviour of a flow per direction + trace_analyse: whether each traced flow reassembles its application + layer, so that its ``packet`` can be read. Off by default, + because it buffers every traced payload -- a cost tracing does + not otherwise pay. Unavailable on the ``pyshark`` engine, which + reports dissected fields rather than the octets behind them ip: if record data for IPv4 & IPv6 reassembly (must be used with ``reassembly=True``) ipv4: if perform IPv4 reassembly (must be used with ``reassembly=True``) @@ -910,6 +915,17 @@ def __init__(self, "using 'trace_format=\"json\"' instead", FormatWarning, stacklevel=stacklevel()) trace_format = 'json' + # NOTE: PyShark hands the tracer dissected *fields*, not the octets + # behind them, so there is no payload for a flow to reassemble -- which + # is the same reason :mod:`pcapkit.toolkit.pyshark` carries no + # ``tcp_reassembly`` at all. Refuse rather than analyse empty payloads + # into an empty answer that looks like a real one. + if trace_analyse and self._exnam == 'pyshark': + warn(f"'Extractor(engine={self._exnam})' does not expose packet payloads; " + "using 'trace_analyse=False' instead", AttributeWarning, + stacklevel=stacklevel()) + trace_analyse = False + if self._tcp: logger.debug('TCP flow tracing enabled') @@ -919,7 +935,8 @@ def __init__(self, self.__traceflow__['tcp'] = trace_cls_tcp # update mapping upon import trace_obj_tcp = cast('TCP_TraceFlow', trace_cls_tcp(fout=trace_fout, format=trace_format, byteorder=trace_byteorder, nanosecond=trace_nanosecond, - bidirectional=trace_bidirectional)) + bidirectional=trace_bidirectional, + analyse=trace_analyse)) self._trace = TraceFlowManager( tcp=trace_obj_tcp, diff --git a/pcapkit/foundation/reassembly/data/data.py b/pcapkit/foundation/reassembly/data/data.py index e26a4a4dbd..3d17f28737 100644 --- a/pcapkit/foundation/reassembly/data/data.py +++ b/pcapkit/foundation/reassembly/data/data.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- """shared data models for reassembly""" -import enum from typing import TYPE_CHECKING from pcapkit.corekit.infoclass import Info, info_final +from pcapkit.utilities.compat import StrEnum __all__ = ['ReassemblyData', 'Completion', 'Deferred', 'DeferredPacket'] @@ -19,7 +19,7 @@ from pcapkit.protocols.protocol import ProtocolBase as Protocol -class Completion(enum.Enum): +class Completion(StrEnum): """How completely a datagram was reassembled, and why it stopped. This is the value of @@ -39,6 +39,23 @@ class Completion(enum.Enum): complete datagram -- so a caller comparing against a boolean has to compare against a member instead. + It derives from :class:`~pcapkit.utilities.compat.StrEnum`, as + :class:`~pcapkit.protocols.application.httpv1.Type` and + :class:`~pcapkit.protocols.misc.pcapng.TLSKeyLabel` do, which buys two things + a plain :class:`enum.Enum` does not: the value survives + :func:`json.dumps` -- a plain enumeration raises :exc:`TypeError` there, and + :meth:`Datagram.to_dict ` hands this + field straight out -- and ``datagram.completed == 'timeout'`` works, so the + new state can be tested for without importing this class. + + Warning: + Being a :class:`str` whose :attr:`PARTIAL` and :attr:`TIMEOUT` members are + **falsy** makes this a non-empty string that tests false, so ``bool(x)`` + and ``bool(str(x))`` disagree. That is deliberate -- the truthiness above + is the property callers of a former :obj:`bool` field rely on -- but code + that takes this for an ordinary string and tests it for truth will read it + backwards. + """ #: Reassembled in whole: every octet of the datagram was received. @@ -64,12 +81,16 @@ def __bool__(self) -> 'bool': Only :attr:`COMPLETE` is truthy; both :attr:`PARTIAL` and :attr:`TIMEOUT` describe an incomplete datagram. + Note: + This override is what a :class:`str` base does *not* give -- every + non-empty string is otherwise truthy, which would make an incomplete + datagram read as a complete one. :meth:`__str__` needs no such + override: :class:`~pcapkit.utilities.compat.StrEnum` already renders a + member as its value. + """ return self is Completion.COMPLETE - def __str__(self) -> 'str': - return self.value - class Deferred: """A postponed analysis of a reassembled payload. @@ -145,6 +166,13 @@ class DeferredPacket: """ + # NOTE: the ``super()`` calls below are suppressed for both checkers. They are + # undefined *on this mixin*, which is what a mixin is -- the base arrives at + # the point of use, where every subclass is declared + # ``class X(DeferredPacket, Info)`` and :class:`~pcapkit.corekit.infoclass.Info` + # supplies all three. Neither mypy nor pylint can see that from here, and + # pylint calls it an *error* rather than a warning. + def __analyse__(self) -> 'Optional[Protocol]': """Resolve a deferred analysis, at most once. @@ -170,13 +198,13 @@ def __getattr__(self, name: 'str') -> 'Any': def __getitem__(self, name: 'str') -> 'Any': if name == 'packet': return self.__analyse__() - return super().__getitem__(name) + return super().__getitem__(name) # type: ignore[misc] # pylint: disable=no-member def __contains__(self, name: 'object') -> 'bool': # NOTE: ``Mapping.__contains__`` answers by fetching the value, which # would run the deferred analysis merely to decide that the field exists. # ``packet`` is a declared field, so it is always there. - return name == 'packet' or super().__contains__(name) + return name == 'packet' or super().__contains__(name) # type: ignore[misc] # pylint: disable=no-member def __str__(self) -> 'str': self.__analyse__() @@ -196,7 +224,7 @@ def to_dict(self) -> 'dict[str, Any]': """ self.__analyse__() - return super().to_dict() + return super().to_dict() # type: ignore[misc] # pylint: disable=no-member @info_final diff --git a/pcapkit/foundation/reassembly/ip.py b/pcapkit/foundation/reassembly/ip.py index 72ea6f061b..908ddf3e3e 100644 --- a/pcapkit/foundation/reassembly/ip.py +++ b/pcapkit/foundation/reassembly/ip.py @@ -85,11 +85,11 @@ def reassembly(self, info: 'Packet[_AT]') -> 'None': self._flag_n = False self.__cached__.clear() - BUFID = info.bufid # Buffer Identifier - FO = info.fo # Fragment Offset - IHL = info.ihl # Internet Header Length - MF = info.mf # More Fragments flag - TL = info.tl # Total Length + BUFID = info.bufid # Buffer Identifier + FO = info.fo # Fragment Offset + IHL = info.ihl # Internet Header Length + MF = info.mf # More Fragments flag + TL = info.tl # Total Length TS = info.timestamp # Capture timestamp, i.e. the only clock we have # This fragment's arrival is the evidence that capture time has reached diff --git a/pcapkit/foundation/reassembly/tcp.py b/pcapkit/foundation/reassembly/tcp.py index 519e674865..35e818747b 100644 --- a/pcapkit/foundation/reassembly/tcp.py +++ b/pcapkit/foundation/reassembly/tcp.py @@ -110,12 +110,12 @@ def reassembly(self, info: 'Packet') -> 'None': self._flag_n = False self.__cached__.clear() - BUFID = info.bufid # Buffer Identifier - DSN = info.dsn # Data Sequence Number - ACK = info.ack # Acknowledgement Number - FIN = info.fin # Finish Flag (Termination) - RST = info.rst # Reset Connection Flag (Termination) - SYN = info.syn # Synchronise Flag (Establishment) + BUFID = info.bufid # Buffer Identifier + DSN = info.dsn # Data Sequence Number + ACK = info.ack # Acknowledgement Number + FIN = info.fin # Finish Flag (Termination) + RST = info.rst # Reset Connection Flag (Termination) + SYN = info.syn # Synchronise Flag (Establishment) TS = info.timestamp # Capture timestamp, i.e. the only clock we have # This segment's arrival is the evidence that capture time has reached diff --git a/pcapkit/foundation/traceflow/data/data.py b/pcapkit/foundation/traceflow/data/data.py index a250dca8ee..90c16e2617 100644 --- a/pcapkit/foundation/traceflow/data/data.py +++ b/pcapkit/foundation/traceflow/data/data.py @@ -5,14 +5,140 @@ from pcapkit.corekit.infoclass import Info, info_final -__all__ = ['TraceFlowData'] +__all__ = ['TraceFlowData', 'Deferred', 'DeferredPacket'] if TYPE_CHECKING: - from typing import Optional + from typing import Any, Optional + from pcapkit.foundation.reassembly.data.tcp import Datagram as TCP_Datagram + from pcapkit.foundation.reassembly.tcp import TCP as TCP_Reassembly from pcapkit.foundation.traceflow.data.tcp import Index as TCP_Index +class Deferred: + """A postponed reassembly of a traced flow's application layer. + + A traced flow's ``packet`` is the application-layer payload of the + conversation, one datagram per direction. Producing it means reassembling the + stream, which is neither free nor wanted by most callers of a *tracer* -- so + the flow keeps the reassembler it was fed and this holds it until somebody + reads + :attr:`Index.packet `. + + Note: + Deliberately not + :class:`pcapkit.foundation.reassembly.data.data.Deferred`, and not shared + with it. That one postpones a single ``analyze()`` call over bytes already + in hand; this postpones a *submit* over a reassembler's buffers. The two + subpackages are siblings and neither should depend on the other, so the + twenty lines are written twice rather than one importing the other -- the + same reason the two ``data/data.py`` modules mirror each other instead of + merging. + + Args: + reassembly: The flow's own + :class:`~pcapkit.foundation.reassembly.tcp.TCP` reassembler, fed the + segments of this conversation as they were traced. + + """ + + __slots__ = ('reassembly',) + + def __init__(self, reassembly: 'TCP_Reassembly') -> 'None': + self.reassembly = reassembly + + def __call__(self) -> 'tuple[TCP_Datagram, ...]': + """Run the postponed reassembly. + + Returns: + One reassembled datagram per direction of the conversation. Each + carries its *own* postponed analysis in + :attr:`Datagram.packet `, + so parsing the payload as an application-layer protocol is still not + paid for until that is read in turn. + + """ + return self.reassembly.datagram + + +class DeferredPacket: + """Resolves a :class:`Deferred` ``packet`` field on first read. + + The reading half of the arrangement above, and the counterpart of + :class:`pcapkit.foundation.reassembly.data.data.DeferredPacket`. + + A subclass has to list ``packet`` in its ``__additional__``. That is what makes + the field lazy at all: :class:`~pcapkit.corekit.infoclass.Info` stores a field + named there under a mangled key and maps it back on the way out, so ``packet`` + never lands in :attr:`~object.__dict__` itself -- which routes reading it + through :meth:`__getattr__`, where the deferred reassembly can run, while + ``dict(index)``, :meth:`to_dict` and iteration still report the field under its + own name. + + """ + + # NOTE: the ``super()`` calls below are suppressed for both checkers. They are + # undefined *on this mixin*, which is what a mixin is -- the base arrives at + # the point of use, where every subclass is declared + # ``class X(DeferredPacket, Info)`` and :class:`~pcapkit.corekit.infoclass.Info` + # supplies all three. Neither mypy nor pylint can see that from here, and + # pylint calls it an *error* rather than a warning. + + def __analyse__(self) -> 'Optional[tuple[TCP_Datagram, ...]]': + """Resolve a deferred reassembly, at most once. + + Returns: + The flow's reassembled datagrams, or :data:`None` when the tracer was + not asked to analyse the application layer. + + """ + key = self.__map__.get('packet', 'packet') + value = self.__dict__[key] + if isinstance(value, Deferred): + value = value() + self.__dict__[key] = value + return value + + def __getattr__(self, name: 'str') -> 'Any': + # NOTE: reached only for names absent from ``__dict__``, which ``packet`` + # always is -- see ``__additional__`` above. Everything else has to raise, + # or a typo would silently answer with a reassembled flow. + if name != 'packet': + raise AttributeError(f'{type(self).__name__!r} object has no attribute {name!r}') + return self.__analyse__() + + def __getitem__(self, name: 'str') -> 'Any': + if name == 'packet': + return self.__analyse__() + return super().__getitem__(name) # type: ignore[misc] # pylint: disable=no-member + + def __contains__(self, name: 'object') -> 'bool': + # NOTE: ``Mapping.__contains__`` answers by fetching the value, which would + # run the deferred reassembly merely to decide that the field exists. + # ``packet`` is a declared field, so it is always there. + return name == 'packet' or super().__contains__(name) # type: ignore[misc] # pylint: disable=no-member + + def __str__(self) -> 'str': + self.__analyse__() + return super().__str__() + + def __repr__(self) -> 'str': + self.__analyse__() + return super().__repr__() + + def to_dict(self) -> 'dict[str, Any]': + """Convert :class:`Index` into :obj:`dict`. + + Returns: + The flow's fields, with ``packet`` reassembled if it had not been read + yet -- a :obj:`dict` holding a :class:`Deferred` would leak an + implementation detail into what is meant to be plain data. + + """ + self.__analyse__() + return super().to_dict() # type: ignore[misc] # pylint: disable=no-member + + @info_final class TraceFlowData(Info): """Data storage for flow tracing.""" diff --git a/pcapkit/foundation/traceflow/data/tcp.py b/pcapkit/foundation/traceflow/data/tcp.py index d81de38bba..daeeaa7b8c 100644 --- a/pcapkit/foundation/traceflow/data/tcp.py +++ b/pcapkit/foundation/traceflow/data/tcp.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Generic, TypeVar from pcapkit.corekit.infoclass import Info, info_final +from pcapkit.foundation.traceflow.data.data import Deferred, DeferredPacket from pcapkit.utilities.compat import Tuple __all__ = ['BufferID', 'Packet', 'Buffer', 'Index'] @@ -16,6 +17,8 @@ from typing_extensions import TypeAlias from pcapkit.const.reg.linktype import LinkType as Enum_LinkType + from pcapkit.foundation.reassembly.data.tcp import Datagram as TCP_Datagram + from pcapkit.foundation.reassembly.tcp import TCP as TCP_Reassembly from pcapkit.protocols.data.misc.pcap.frame import Frame as Data_Frame _AT = TypeVar('_AT', 'IPv4Address', 'IPv6Address') @@ -71,9 +74,23 @@ class Packet(Info, Generic[_AT]): dstport: 'int' #: Frame timestamp. timestamp: 'float' + #: TCP sequence number. Carried so that a tracer asked to analyse the + #: application layer can hand the segment to + #: :class:`~pcapkit.foundation.reassembly.tcp.TCP` rather than reassemble the + #: stream itself -- a tracer that simply concatenated payloads in capture order + #: would be silently wrong on the first retransmission or reordering. + seq: 'int' + #: TCP acknowledgement number, which is what the reassembler keys a payload + #: buffer on. + ack: 'int' + #: Raw :obj:`bytes` type TCP header. + header: 'bytes' + #: Raw :obj:`bytearray` type TCP payload, i.e. the application-layer octets + #: this segment carries. + payload: 'bytearray' if TYPE_CHECKING: - def __init__(self, protocol: 'Enum_LinkType', index: 'int', frame: 'Data_Frame | dict[str, Any]', syn: 'bool', fin: 'bool', rst: 'bool', src: '_AT', dst: '_AT', srcport: 'int', dstport: 'int', timestamp: 'float') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long + def __init__(self, protocol: 'Enum_LinkType', index: 'int', frame: 'Data_Frame | dict[str, Any]', syn: 'bool', fin: 'bool', rst: 'bool', src: '_AT', dst: '_AT', srcport: 'int', dstport: 'int', timestamp: 'float', seq: 'int', ack: 'int', header: 'bytes', payload: 'bytearray') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long @info_final @@ -114,6 +131,13 @@ class Buffer(Info, Generic[_AT]): #: connection at once (:rfc:`9293#section-3.5.2`), where a polite close needs #: a FIN from each side, so it is tracked as a flag rather than per endpoint. reset: 'bool' + #: The flow's own :class:`~pcapkit.foundation.reassembly.tcp.TCP` reassembler, + #: fed each segment as it is traced, or :data:`None` when the tracer was not + #: asked to analyse the application layer. One per flow rather than one per + #: tracer, so that + #: :attr:`Index.packet ` can + #: flush *this* conversation without disturbing any other. + reassembly: 'Optional[TCP_Reassembly]' if TYPE_CHECKING: # NOTE: one line, however long. ``# pylint: disable`` is *line*-scoped and @@ -122,11 +146,11 @@ class Buffer(Info, Generic[_AT]): # disable's reach -- which is why the shorter form this replaces leaked # three ``unused-argument`` messages of its own. Every other data model in # :mod:`pcapkit` writes these stubs on one line for the same reason. - def __init__(self, fpout: 'Dumper', index: 'list[int]', label: 'str', origin: 'tuple[_AT, int]', forward: 'list[int]', reverse: 'list[int]', fin: 'set[tuple[_AT, int]]', reset: 'bool') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long + def __init__(self, fpout: 'Dumper', index: 'list[int]', label: 'str', origin: 'tuple[_AT, int]', forward: 'list[int]', reverse: 'list[int]', fin: 'set[tuple[_AT, int]]', reset: 'bool', reassembly: 'Optional[TCP_Reassembly]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long @info_final -class Index(Info): +class Index(DeferredPacket, Info): """Data structure for **TCP flow tracing**. See Also: @@ -136,6 +160,10 @@ class Index(Info): """ + #: Listing ``packet`` here is what makes :attr:`packet` lazy -- see + #: :class:`~pcapkit.foundation.traceflow.data.data.DeferredPacket`. + __additional__ = ['packet'] + #: Output filename if exists. fpout: 'Optional[str]' #: Tuple of frame index, **both directions**, in capture order. @@ -151,7 +179,17 @@ class Index(Info): #: Empty when tracing unidirectionally, in which case #: :attr:`forward` ``==`` :attr:`index`. reverse: 'tuple[int, ...]' + #: The conversation's **application layer**: one reassembled datagram per + #: direction, or :data:`None` when the tracer was not asked for it + #: (``analyse=False``, the default). + #: + #: Reassembled on the first read, not when the flow is finalised, and each + #: datagram's own + #: :attr:`~pcapkit.foundation.reassembly.data.tcp.Datagram.packet` is parsed + #: later still -- two layers of the same postponement, so a caller that only + #: wanted frame numbers pays for neither. + packet: 'Optional[tuple[TCP_Datagram, ...]]' if TYPE_CHECKING: # NOTE: on one line, for the reason given on :class:`Buffer` above. - def __init__(self, fpout: 'Optional[str]', index: 'tuple[int, ...]', label: 'str', forward: 'tuple[int, ...]', reverse: 'tuple[int, ...]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long + def __init__(self, fpout: 'Optional[str]', index: 'tuple[int, ...]', label: 'str', forward: 'tuple[int, ...]', reverse: 'tuple[int, ...]', packet: 'Optional[tuple[TCP_Datagram, ...] | Deferred]') -> 'None': ... # pylint: disable=unused-argument,super-init-not-called,multiple-statements,line-too-long diff --git a/pcapkit/foundation/traceflow/tcp.py b/pcapkit/foundation/traceflow/tcp.py index ca53768258..33e5606cb7 100644 --- a/pcapkit/foundation/traceflow/tcp.py +++ b/pcapkit/foundation/traceflow/tcp.py @@ -11,6 +11,7 @@ """ from typing import TYPE_CHECKING, Generic, overload +from pcapkit.foundation.traceflow.data.data import Deferred from pcapkit.foundation.traceflow.data.tcp import _AT, Buffer, BufferID, Index, Packet from pcapkit.foundation.traceflow.traceflow import TraceFlowBase as TraceFlow from pcapkit.protocols.transport.tcp import TCP as TCP_Protocol @@ -19,15 +20,19 @@ __all__ = ['TCP'] if TYPE_CHECKING: + from typing import Any, Optional + from dictdumper.dumper import Dumper from typing_extensions import Literal + from pcapkit.foundation.reassembly.tcp import TCP as TCP_Reassembly + #: logging.Logger: Module-level logger, a child of the package-wide #: :data:`pcapkit.utilities.logging.logger`. logger = get_logger(__name__) -class TCP(TraceFlow[BufferID, 'Buffer[_AT]', Index, Packet[_AT]], Generic[_AT]): +class TCP(TraceFlow[BufferID, Buffer[_AT], Index, Packet[_AT]], Generic[_AT]): """Trace TCP flows. Args: @@ -36,6 +41,9 @@ class TCP(TraceFlow[BufferID, 'Buffer[_AT]', Index, Packet[_AT]], Generic[_AT]): byteorder: output file byte order nanosecond: output nanosecond-resolution file flag bidirectional: trace both halves of a conversation as one flow + analyse: reassemble each flow's application layer, so that + :attr:`Index.packet ` + can be read *args: Arbitrary positional arguments. **kwargs: Arbitrary keyword arguments. @@ -70,8 +78,28 @@ class TCP(TraceFlow[BufferID, 'Buffer[_AT]', Index, Packet[_AT]], Generic[_AT]): what flow tracing did before conversations became one flow, RST included -- which is to say it ignores RST, as it always did. + Note: + With ``analyse=True`` a flow also carries its **application layer**: + :attr:`Index.packet ` + holds one reassembled datagram per direction, each with its payload parsed + on demand. The tracer does not reassemble the stream itself -- it feeds + :class:`~pcapkit.foundation.reassembly.tcp.TCP`, which already implements + :rfc:`815` and copes with the reordering and retransmission that a tracer + concatenating payloads in capture order would silently corrupt. + + It is **off by default** because buffering every traced payload is a cost + tracing does not otherwise pay, and tracing's per-packet cost is something + this package has deliberately driven down. Nothing is reassembled, parsed + or retained unless it is asked for. + """ + if TYPE_CHECKING: + #: The reassembly segment model, imported on the first flow that needs it + #: and cached so that :meth:`_make_segment` does not import per packet. Set + #: only when ``analyse`` is on, which is the only time it is read. + _reasm_packet: 'type[Any]' + ########################################################################## # Defaults. ########################################################################## @@ -209,6 +237,7 @@ def trace(self, packet: 'Packet[_AT]', *, output: 'bool' = False) -> 'Dumper | s reverse=[], fin=set(), reset=False, + reassembly=self._make_reassembly(), ) # trace frame record @@ -224,6 +253,8 @@ def trace(self, packet: 'Packet[_AT]', *, output: 'bool' = False) -> 'Dumper | s buffer.fin.add(END) if RST: buffer.__update__(reset=True) + if buffer.reassembly is not None: + buffer.reassembly(self._make_segment(packet)) fpout = buffer.fpout label = buffer.label @@ -250,6 +281,84 @@ def trace(self, packet: 'Packet[_AT]', *, output: 'bool' = False) -> 'Dumper | s # return label or output object return fpout if output else label + def _make_reassembly(self) -> 'Optional[TCP_Reassembly]': + """Build the reassembler a new flow will feed, if analysis was asked for. + + Returns: + A :class:`~pcapkit.foundation.reassembly.tcp.TCP` reassembler of this + flow's own, or :data:`None` when ``analyse`` is off. + + One reassembler **per flow** rather than one per tracer, so that + :attr:`Index.packet ` + can flush this conversation's buffers without touching another's -- a + shared instance would have to be asked for its datagrams by endpoint, and + flushing it early would finalise flows that are still open. + + It is constructed with ``strict=False`` so that each direction comes back + as one contiguous payload, which is what an application-layer parse needs, + and it inherits + :attr:`TCP.__timeout__ ` + -- no timeout -- so nothing here is evicted on a clock. + + """ + if not self._analyse: + return None + + # NOTE: imported here rather than at module scope. ``traceflow`` and + # ``reassembly`` are sibling subpackages and this is the only edge between + # them; at module scope it is evaluated while ``pcapkit/__init__`` is still + # running, which is the import cycle that already runs through + # ``foundation.extraction``. Kept off the per-*packet* path by caching the + # segment model here: this runs once per flow, ``_make_segment`` once per + # packet. + from pcapkit.foundation.reassembly.data.tcp import Packet as Reasm_Packet + from pcapkit.foundation.reassembly.tcp import TCP as TCP_Reassembly + + self._reasm_packet = Reasm_Packet + return TCP_Reassembly(strict=False) + + def _make_segment(self, packet: 'Packet[_AT]') -> 'Any': + """Describe a traced packet the way the reassembler expects a segment. + + Arguments: + packet: a flow packet (:term:`trace.tcp.packet`) + + Returns: + A :class:`reassembly packet ` + for the same segment. + + The tracer does not reassemble anything itself -- it hands the segment to + :class:`~pcapkit.foundation.reassembly.tcp.TCP`, which already implements + the :rfc:`815` hole-descriptor algorithm and handles the out-of-order and + retransmitted segments a tracer concatenating payloads in capture order + would silently corrupt. This is the whole of the translation. + + Note: + The buffer ID is **(source, destination)** and so unidirectional, which + is what makes each direction of the conversation reassemble separately + even though the *flow* is keyed on the pair. The sequence number is + passed through untouched: a SYN occupies one of its own, and + :meth:`TCP.reassembly ` + is where that is accounted for. + + """ + raw_len = len(packet.payload) + return self._reasm_packet( + bufid=(packet.src, packet.srcport, packet.dst, packet.dstport), + dsn=packet.seq, # data sequence number + ack=packet.ack, # acknowledgement number + num=packet.index, # original packet range number + syn=packet.syn, # synchronise flag + fin=packet.fin, # finish flag + rst=packet.rst, # reset connection flag + len=raw_len, # payload length, header excludes + first=packet.seq, # first sequence number of payload + last=packet.seq + raw_len - 1, # last sequence number of payload + header=packet.header, # raw bytes type header + payload=packet.payload, # raw bytearray type payload + timestamp=packet.timestamp, # capture timestamp + ) + @staticmethod def _ended(buffer: 'Buffer[_AT]') -> 'bool': """Whether a flow's connection has been seen to end. @@ -296,6 +405,9 @@ def _finalise(self, bufid: 'BufferID') -> 'Index': label=label, forward=tuple(buf.forward), reverse=tuple(buf.reverse), + # the reassembler goes in unflushed: reading ``packet`` is what asks it + # for its datagrams, and their own ``packet`` is parsed later still + packet=None if buf.reassembly is None else Deferred(buf.reassembly), ) for callback in self.__callback_fn__: callback(index) @@ -343,7 +455,8 @@ def submit(self) -> 'tuple[Index, ...]': index=tuple(buf.index), label=buf.label, forward=tuple(buf.forward), - reverse=tuple(buf.reverse),)) + reverse=tuple(buf.reverse), + packet=None if buf.reassembly is None else Deferred(buf.reassembly),)) ret.extend(self._stream) ret_submit = tuple(ret) diff --git a/pcapkit/foundation/traceflow/traceflow.py b/pcapkit/foundation/traceflow/traceflow.py index b38d0352ca..8c4abfa60b 100644 --- a/pcapkit/foundation/traceflow/traceflow.py +++ b/pcapkit/foundation/traceflow/traceflow.py @@ -91,6 +91,7 @@ class TraceFlowBase(Generic[_DT, _BT, _IT, _PT], metaclass=TraceFlowMeta): byteorder: output file byte order nanosecond: output nanosecond-resolution file flag bidirectional: trace both halves of a conversation as one flow + analyse: reassemble each flow's application layer Note: This class is for internal use only. For customisation, please use @@ -325,7 +326,8 @@ def __new__(cls, *args: 'Any', **kwargs: 'Any') -> 'Self': # pylint: disable=un def __init__(self, fout: 'Optional[str]', format: 'Optional[str]', # pylint: disable=redefined-builtin byteorder: 'Literal["little", "big"]' = sys.byteorder, - nanosecond: bool = False, bidirectional: 'bool' = True) -> 'None': + nanosecond: bool = False, bidirectional: 'bool' = True, + analyse: 'bool' = False) -> 'None': """Initialise instance. Arguments: @@ -338,6 +340,9 @@ def __init__(self, fout: 'Optional[str]', format: 'Optional[str]', # pylint: di endpoints rather than on (source, destination), so a connection is traced as the one thing it is; pass :data:`False` for the older per-direction behaviour. + analyse: whether to reassemble each flow's application layer, so that + its ``packet`` can be read. Off by default: it buffers every + traced payload, a cost tracing does not otherwise pay. """ if fout is None: @@ -361,6 +366,11 @@ def __init__(self, fout: 'Optional[str]', format: 'Optional[str]', # pylint: di #: of a conversation share one buffer entry, one label and one output #: file; otherwise each direction is a flow of its own. self._bidir = bidirectional + #: bool: Application-layer analysis flag. If set to :data:`True`, each + #: flow reassembles the payload it carries so that its ``packet`` can be + #: read; otherwise no payload is buffered and ``packet`` is + #: :data:`None`. + self._analyse = analyse # dump I/O object fio, ext = self.make_fout(fout, format) @@ -370,8 +380,8 @@ def __init__(self, fout: 'Optional[str]', format: 'Optional[str]', # pylint: di self._fdpext = ext logger.debug('%s flow tracing initialised (root=%s, format=%s, byteorder=%s, ' - 'nanosecond=%s, bidirectional=%s)', self.name, fout, format, - byteorder, nanosecond, bidirectional) + 'nanosecond=%s, bidirectional=%s, analyse=%s)', self.name, fout, + format, byteorder, nanosecond, bidirectional, analyse) def __call__(self, packet: '_PT') -> 'None': """Dump frame to output files. @@ -413,6 +423,7 @@ class MyProtocol(TraceFlow, protocol='my_protocol'): byteorder: output file byte order nanosecond: output nanosecond-resolution file flag bidirectional: trace both halves of a conversation as one flow + analyse: reassemble each flow's application layer """ diff --git a/pcapkit/interface/core.py b/pcapkit/interface/core.py index 4dacd50dcd..24c1109c2c 100644 --- a/pcapkit/interface/core.py +++ b/pcapkit/interface/core.py @@ -79,7 +79,7 @@ def extract(fin: 'Optional[str | IO[bytes]]' = None, fout: 'Optional[str]' = Non reasm_timeout: 'Optional[float]' = None, # reassembly settings # pylint: disable=line-too-long trace: 'bool' = False, trace_fout: 'Optional[str]' = None, trace_format: 'Optional[Formats]' = None, # trace settings # pylint: disable=line-too-long trace_byteorder: 'Literal["big", "little"]' = sys.byteorder, trace_nanosecond: 'bool' = False, # trace settings # pylint: disable=line-too-long - trace_bidirectional: 'bool' = True, # trace settings # pylint: disable=line-too-long + trace_bidirectional: 'bool' = True, trace_analyse: 'bool' = False, # trace settings # pylint: disable=line-too-long ip: 'bool' = False, ipv4: 'bool' = False, ipv6: 'bool' = False, tcp: 'bool' = False, # reassembly/trace settings # pylint: disable=line-too-long buffer_size: 'int' = io.DEFAULT_BUFFER_SIZE, buffer_save: 'bool' = False, buffer_path: 'Optional[str]' = None, # buffer settings # pylint: disable=line-too-long no_eof: 'bool' = False, # EOF settings # pylint: disable=line-too-long @@ -120,6 +120,8 @@ def extract(fin: 'Optional[str | IO[bytes]]' = None, fout: 'Optional[str]' = Non trace_nanosecond: output nanosecond-resolution file flag trace_bidirectional: whether both halves of a conversation are traced as one flow, which is the default + trace_analyse: whether each traced flow reassembles its application + layer, so that its ``packet`` can be read; off by default ip: if record data for IPv4 & IPv6 reassembly (must be used with ``reassembly=True``) ipv4: if perform IPv4 reassembly (must be used with ``reassembly=True``) @@ -161,7 +163,7 @@ def extract(fin: 'Optional[str | IO[bytes]]' = None, fout: 'Optional[str]' = Non reasm_timeout=reasm_timeout, trace=trace, trace_fout=trace_fout, trace_format=trace_format, trace_byteorder=trace_byteorder, trace_nanosecond=trace_nanosecond, - trace_bidirectional=trace_bidirectional, + trace_bidirectional=trace_bidirectional, trace_analyse=trace_analyse, buffer_size=buffer_size, buffer_path=buffer_path, buffer_save=buffer_save, no_eof=no_eof, context=context) diff --git a/pcapkit/interface/misc.py b/pcapkit/interface/misc.py index 80a070b59b..4829df6409 100644 --- a/pcapkit/interface/misc.py +++ b/pcapkit/interface/misc.py @@ -9,7 +9,6 @@ generally provided per user's requests. """ -import functools import sys from typing import TYPE_CHECKING, cast @@ -25,7 +24,7 @@ from pcapkit.utilities.warnings import EngineWarning, FormatWarning, warn if TYPE_CHECKING: - from typing import Callable, Optional + from typing import Any, Callable, Optional from typing_extensions import Literal @@ -77,7 +76,8 @@ def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, extension: 'bool' = True, engine: 'Optional[Engines]' = None, fout: 'Optional[str]' = None, format: 'Optional[Formats]' = None, # TraceFlow options # pylint: disable=redefined-builtin byteorder: 'ByteOrder' = sys.byteorder, nanosecond: 'bool' = False, - trace_bidirectional: 'bool' = True) -> 'tuple[Stream, ...]': + trace_bidirectional: 'bool' = True, + trace_analyse: 'bool' = False) -> 'tuple[Stream, ...]': """Follow TCP streams. Arguments: @@ -95,6 +95,12 @@ def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, frames and the reassembled payload of *both* directions, which is what "following a TCP stream" means elsewhere. :data:`False` restores one stream per direction. + trace_analyse: whether each traced flow reassembles its application + layer, so that ``Index.packet`` can be read off + :attr:`Extractor.trace `. + Off by default, and independent of the ``conversations`` this + function returns -- those come from the reassembly below, which runs + either way. Returns: List of extracted TCP streams. @@ -138,7 +144,8 @@ def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, layer=None, protocol=None, ip=False, ipv4=False, ipv6=False, tcp=True, reassembly=False, trace=True, trace_fout=fout, trace_format=format, trace_byteorder=byteorder, trace_nanosecond=nanosecond, - trace_bidirectional=trace_bidirectional) # type: ignore[var-annotated] + trace_bidirectional=trace_bidirectional, + trace_analyse=trace_analyse) # type: ignore[var-annotated] # NOTE: ``Extractor.engine`` returns the running engine *instance* (see # :meth:`Extractor.engine `), @@ -157,6 +164,10 @@ def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, exeng = extraction.engine tcp_reassembly = None # type: Optional[ReassemblyAdapter] pass_count = True + #: Reads a frame's capture timestamp back off the frame, for the one adapter + #: whose signature takes it. :data:`None` for the adapters that find it + #: themselves. + timestamp_of = None # type: Optional[Callable[[Any], float]] if isinstance(exeng, PCAP_Engine): from pcapkit.toolkit import pcap as tk_pcap # isort: skip # pylint: disable=import-outside-toplevel tcp_reassembly, pass_count = cast('ReassemblyAdapter', tk_pcap.tcp_reassembly), False @@ -165,21 +176,14 @@ def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, tcp_reassembly, pass_count = cast('ReassemblyAdapter', tk_pcapng.tcp_reassembly), False elif isinstance(exeng, DPKT_Engine): from pcapkit.toolkit import dpkt as tk_dpkt # isort: skip # pylint: disable=import-outside-toplevel - # NOTE: DPKT is the one adapter that cannot read a frame's capture - # timestamp off the frame -- DPKT hands ``(timestamp, bytes)`` back from - # its reader and keeps the two apart, and - # :class:`~pcapkit.foundation.engines.dpkt.DPKT` stores only the packet -- - # so the reassembly adapter takes it as an argument. Nothing here has one - # to give: the frames were stored during an extraction that has already - # finished. Binding zero is safe rather than merely convenient, because - # the reassembler below is constructed here and TCP reassembly has no - # timeout by default (see - # :attr:`TCP.__timeout__ `), - # so no deadline is computed from it. Enabling one for this call would - # first mean having the DPKT engine record each frame's timestamp beside - # the frame. - tcp_reassembly = cast('ReassemblyAdapter', - functools.partial(tk_dpkt.tcp_reassembly, timestamp=0.0)) + # NOTE: DPKT's reader hands ``(timestamp, bytes)`` back and only the octets + # become a packet, so its adapters take the capture timestamp as an + # argument rather than finding it on the frame. That timestamp is real and + # available -- :class:`~pcapkit.foundation.engines.dpkt.DPKT` attaches it to + # every frame it reads, precisely so that a reader arriving after the + # extraction loop (like this one) can get it back. + tcp_reassembly = cast('ReassemblyAdapter', tk_dpkt.tcp_reassembly) + timestamp_of = tk_dpkt.packet2timestamp elif isinstance(exeng, Scapy_Engine): from pcapkit.toolkit import scapy as tk_scapy # isort: skip # pylint: disable=import-outside-toplevel tcp_reassembly = cast('ReassemblyAdapter', tk_scapy.tcp_reassembly) @@ -205,7 +209,9 @@ def follow_tcp_stream(fin: 'Optional[str]' = None, verbose: 'bool' = False, frame = frames[index-1] packets.append(frame) - if pass_count: + if timestamp_of is not None: + data = tcp_reassembly(frame, timestamp_of(frame), count=index) + elif pass_count: data = tcp_reassembly(frame, count=index) else: data = tcp_reassembly(frame) diff --git a/pcapkit/toolkit/dpkt.py b/pcapkit/toolkit/dpkt.py index b5761909e9..8faa3358f1 100644 --- a/pcapkit/toolkit/dpkt.py +++ b/pcapkit/toolkit/dpkt.py @@ -19,6 +19,7 @@ from pcapkit.foundation.reassembly.data.ip import Packet as IP_Packet from pcapkit.foundation.reassembly.data.tcp import Packet as TCP_Packet from pcapkit.foundation.traceflow.data.tcp import Packet as TF_TCP_Packet +from pcapkit.utilities.exceptions import UnsupportedCall if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address @@ -32,10 +33,66 @@ from pcapkit.const.reg.linktype import LinkType as Enum_LinkType __all__ = [ - 'ipv6_hdr_len', 'packet2chain', 'packet2dict', + 'ipv6_hdr_len', 'attach_timestamp', 'packet2timestamp', 'packet2chain', 'packet2dict', 'ipv4_reassembly', 'ipv6_reassembly', 'tcp_reassembly', 'tcp_traceflow' ] +#: Attribute a frame's capture timestamp is stashed under. +#: +#: `DPKT`_ keeps the two halves of a record apart -- its reader yields +#: ``(timestamp, bytes)`` and only the bytes become a packet -- so a frame on its +#: own does not know when it was captured. Anything that reads a frame *after* the +#: extraction loop has moved on therefore has no way back to the timestamp unless +#: the engine puts it somewhere, and this is where +#: :class:`~pcapkit.foundation.engines.dpkt.DPKT` puts it. +#: +#: A `DPKT`_ packet carries a :attr:`~object.__dict__`, so this is an ordinary +#: attribute rather than anything exotic; the name is spelled out here so that +#: nothing has to know it by hand. +#: +#: .. _DPKT: https://dpkt.readthedocs.io +TIMESTAMP_ATTR = '__pcapkit_timestamp__' + + +def attach_timestamp(packet: 'Packet', timestamp: 'float') -> 'None': + """Stash a frame's capture timestamp on the frame. + + Args: + packet: DPKT packet. + timestamp: Capture timestamp of the packet, as `DPKT`_'s reader yielded it + beside the record's octets. + + .. _DPKT: https://dpkt.readthedocs.io + + """ + setattr(packet, TIMESTAMP_ATTR, timestamp) + + +def packet2timestamp(packet: 'Packet') -> 'float': + """Read back the capture timestamp of a DPKT packet. + + Args: + packet: DPKT packet, as stored by + :class:`~pcapkit.foundation.engines.dpkt.DPKT`. + + Returns: + Capture timestamp of the packet, in seconds since the epoch. + + Raises: + UnsupportedCall: If the packet carries no timestamp, i.e. it did not come + through :class:`~pcapkit.foundation.engines.dpkt.DPKT`. Raised rather + than defaulted, because a plausible-looking zero would silently + misdate whatever was going to use it. + + """ + timestamp = getattr(packet, TIMESTAMP_ATTR, None) + if timestamp is None: + raise UnsupportedCall( + f'{type(packet).__name__} carries no capture timestamp; only a frame read by ' + "'Extractor(engine=dpkt)' has one attached" + ) + return cast('float', timestamp) + def ipv6_hdr_len(ipv6: 'IP6') -> 'int': """Calculate length of headers before IPv6 Fragment header. @@ -116,8 +173,11 @@ def ipv4_reassembly(packet: 'Packet', timestamp: 'float', *, Args: packet: DPKT packet. timestamp: Capture timestamp of the packet, which drives the reassembly - timeout. DPKT hands it back beside the packet rather than on it, so - it has to be passed in -- as :func:`tcp_traceflow` already does. + timeout. `DPKT`_'s reader yields it beside the record's octets rather + than on the packet, so it is passed in -- as :func:`tcp_traceflow` + already does. A caller holding only a frame can read it back with + :func:`packet2timestamp`, which is where + :class:`~pcapkit.foundation.engines.dpkt.DPKT` leaves it. count: Packet index. If not provided, default to ``-1``. Returns: @@ -171,7 +231,7 @@ def ipv6_reassembly(packet: 'Packet', timestamp: 'float', *, Args: packet: DPKT packet. timestamp: Capture timestamp of the packet, which drives the reassembly - timeout. + timeout; :func:`packet2timestamp` reads it back off a stored frame. count: Packet index. If not provided, default to ``-1``. Returns: @@ -238,7 +298,7 @@ def tcp_reassembly(packet: 'Packet', timestamp: 'float', *, Args: packet: DPKT packet. timestamp: Capture timestamp of the packet, which drives the reassembly - timeout. + timeout; :func:`packet2timestamp` reads it back off a stored frame. count: Packet index. If not provided, default to ``-1``. Returns: @@ -338,6 +398,10 @@ def tcp_traceflow(packet: 'Packet', timestamp: 'float', *, srcport=tcp.sport, # TCP source port dstport=tcp.dport, # TCP destination port timestamp=timestamp, # timestamp + seq=tcp.seq, # TCP sequence number + ack=tcp.ack, # TCP acknowledgement number + header=tcp.pack()[:tcp.off * 4], # raw bytes type header + payload=bytearray(bytes(tcp.data)), # raw bytearray type payload ) return data return None diff --git a/pcapkit/toolkit/pcap.py b/pcapkit/toolkit/pcap.py index 9c4e151235..828aced385 100644 --- a/pcapkit/toolkit/pcap.py +++ b/pcapkit/toolkit/pcap.py @@ -226,6 +226,10 @@ def tcp_traceflow(frame: 'Frame', *, data_link: 'LinkType') -> 'TF_TCP_Packet | srcport=tcp_info.srcport.port, # TCP source port dstport=tcp_info.dstport.port, # TCP destination port timestamp=float(frame.info.time_epoch), # frame timestamp + seq=tcp_info.seq, # TCP sequence number + ack=tcp_info.ack, # TCP acknowledgement number + header=tcp.packet.header, # raw bytes type header + payload=bytearray(tcp.packet.payload), # raw bytearray type payload ) return data return None diff --git a/pcapkit/toolkit/pcapng.py b/pcapkit/toolkit/pcapng.py index 9ffe05c742..21a9fafb14 100644 --- a/pcapkit/toolkit/pcapng.py +++ b/pcapkit/toolkit/pcapng.py @@ -234,6 +234,10 @@ def tcp_traceflow(frame: 'PCAPNG', *, nanosecond: 'bool' = False) -> 'TF_TCP_Pac srcport=tcp_info.srcport.port, # TCP source port dstport=tcp_info.dstport.port, # TCP destination port timestamp=float(frame_info.timestamp_epoch), # frame timestamp + seq=tcp_info.seq, # TCP sequence number + ack=tcp_info.ack, # TCP acknowledgement number + header=tcp.packet.header, # raw bytes type header + payload=bytearray(tcp.packet.payload), # raw bytearray type payload ) return data return None diff --git a/pcapkit/toolkit/pypcapfile.py b/pcapkit/toolkit/pypcapfile.py index 08b277ec94..96facd5559 100644 --- a/pcapkit/toolkit/pypcapfile.py +++ b/pcapkit/toolkit/pypcapfile.py @@ -442,6 +442,7 @@ def tcp_traceflow(packet: 'Packet', *, data_link: 'Enum_LinkType', from pcapfile.protocols.transport.tcp import TCP # isort:skip tcp = TCP(segment) + hdr_len = max(tcp.data_offset, TCP_MIN_HEADER_LEN) return TF_TCP_Packet( # type: ignore[type-var] protocol=data_link, # data link type from savefile header index=count, # frame number @@ -454,4 +455,8 @@ def tcp_traceflow(packet: 'Packet', *, data_link: 'Enum_LinkType', srcport=tcp.src_port, # TCP source port dstport=tcp.dst_port, # TCP destination port timestamp=packet2timestamp(packet), # timestamp + seq=tcp.seqnum, # TCP sequence number + ack=tcp.acknum, # TCP acknowledgement number + header=segment[:hdr_len], # raw bytes type header + payload=bytearray(segment[hdr_len:]), # raw bytearray type payload ) diff --git a/pcapkit/toolkit/pyshark.py b/pcapkit/toolkit/pyshark.py index 5ced8079f7..27b5fa1940 100644 --- a/pcapkit/toolkit/pyshark.py +++ b/pcapkit/toolkit/pyshark.py @@ -95,6 +95,15 @@ def tcp_traceflow(packet: 'Packet') -> 'TF_TCP_Packet | None': srcport=int(tcp.srcport), # TCP source port dstport=int(tcp.dstport), # TCP destination port timestamp=packet.frame_info.time_epoch, # timestamp + seq=int(tcp.seq), # TCP sequence number + ack=int(tcp.ack), # TCP acknowledgement number + # NOTE: PyShark reports dissected *fields*, not the octets behind + # them, so there is no header or payload to hand over -- which is + # the same reason this module carries no ``tcp_reassembly`` at all. + # ``Extractor`` refuses ``trace_analyse=True`` on this engine, so + # nothing reads these two. + header=b'', # unavailable + payload=bytearray(), # unavailable ) return data return None diff --git a/pcapkit/toolkit/scapy.py b/pcapkit/toolkit/scapy.py index 38bf03d473..02f517902a 100644 --- a/pcapkit/toolkit/scapy.py +++ b/pcapkit/toolkit/scapy.py @@ -333,6 +333,10 @@ def tcp_traceflow(packet: 'Packet', *, count: 'int' = -1) -> 'TF_TCP_Packet | No # own timestamp on ``Packet.time``, which is what every other # engine's adapter reports. timestamp=float(packet.time), # capture timestamp + seq=tcp.seq, # TCP sequence number + ack=tcp.ack, # TCP acknowledgement number + header=bytes(tcp)[:tcp.dataofs * 4], # raw bytes type header + payload=bytearray(bytes(tcp.payload)), # raw bytearray type payload ) return data return None diff --git a/tests/foundation/reassembly/data/test_models.py b/tests/foundation/reassembly/data/test_models.py index 1ebca66913..b8b67e8bf0 100644 --- a/tests/foundation/reassembly/data/test_models.py +++ b/tests/foundation/reassembly/data/test_models.py @@ -49,6 +49,39 @@ def test_ip_data_models_and_package_aliases(self) -> None: self.assertEqual(storage.ipv6, ()) self.assertEqual(storage.tcp, ()) + def test_completion_is_a_string_that_still_reads_as_the_old_bool(self) -> None: + """The contract ``Datagram.completed`` has to keep, now it is a StrEnum. + + Deriving from :class:`~pcapkit.utilities.compat.StrEnum` -- as + :class:`~pcapkit.protocols.application.httpv1.Type` does -- buys + serialisability and comparison against a plain string. What it must not + cost is the truthiness callers of the former :obj:`bool` field rely on, + which needs ``__bool__`` overridden because every non-empty string is + otherwise truthy. + + """ + import json + + from pcapkit.foundation.reassembly.data.data import Completion + + # only COMPLETE is truthy, so ``if datagram.completed:`` reads as it did + self.assertTrue(Completion.COMPLETE) + self.assertFalse(Completion.PARTIAL) + self.assertFalse(Completion.TIMEOUT) + + # ... while equality against a bool stays broken, as documented + self.assertNotEqual(Completion.COMPLETE, True) + self.assertNotEqual(Completion.PARTIAL, False) + + # what the str base adds + self.assertIsInstance(Completion.TIMEOUT, str) + self.assertEqual(Completion.TIMEOUT, 'timeout') + self.assertEqual(str(Completion.PARTIAL), 'partial') + self.assertEqual(json.dumps(Completion.PARTIAL), '"partial"') + + # and identity still works, which is what the assertions elsewhere use + self.assertIs(Completion('complete'), Completion.COMPLETE) + def test_tcp_data_models_and_package_aliases(self) -> None: from pcapkit.foundation.reassembly.data import (Completion, TCP_Buffer, TCP_Datagram, TCP_DatagramID, TCP_Fragment, diff --git a/tests/foundation/traceflow/data/test_models.py b/tests/foundation/traceflow/data/test_models.py index 9d29a5892b..0552db397b 100644 --- a/tests/foundation/traceflow/data/test_models.py +++ b/tests/foundation/traceflow/data/test_models.py @@ -24,29 +24,38 @@ def test_tcp_traceflow_data_models_and_package_aliases(self) -> None: src = ip_address('2001:db8::1') dst = ip_address('2001:db8::2') frame = {'frame': 1} - packet = Packet(LinkType.ETHERNET, 1, frame, True, False, False, src, dst, 12345, 443, 1.5) + packet = Packet(LinkType.ETHERNET, 1, frame, True, False, False, src, dst, 12345, 443, + 1.5, 7, 8, b'tcp-header', bytearray(b'payload')) self.assertIsInstance(packet, TCP_Packet) self.assertEqual(packet.src, src) self.assertEqual(packet.frame, frame) self.assertFalse(packet.rst) + # the segment fields a tracer asked to analyse hands to the reassembler + self.assertEqual((packet.seq, packet.ack), (7, 8)) + self.assertEqual(packet.payload, bytearray(b'payload')) dumper = object() origin = (src, 12345) buffer = Buffer(dumper, [1, 2], '2001_db8_1-12345_2001_db8_2-443', - origin, [1], [2], {origin}, False) + origin, [1], [2], {origin}, False, None) self.assertIsInstance(buffer, TCP_Buffer) self.assertEqual(buffer.fpout, dumper) self.assertEqual(buffer.origin, origin) self.assertEqual((buffer.forward, buffer.reverse), ([1], [2])) self.assertEqual(buffer.fin, {origin}) self.assertFalse(buffer.reset) + self.assertIsNone(buffer.reassembly) - index = Index('/tmp/flow.json', (1, 2), buffer.label, (1,), (2,)) + index = Index('/tmp/flow.json', (1, 2), buffer.label, (1,), (2,), None) self.assertIsInstance(index, TCP_Index) self.assertEqual(index.index, (1, 2)) # each direction stays recoverable, so a caller can still ask which way a # given frame went without taking the label apart self.assertEqual((index.forward, index.reverse), ((1,), (2,))) + # ``packet`` reads through __getattr__ even when there is nothing deferred + self.assertIsNone(index.packet) + self.assertIn('packet', index) + self.assertIsNone(index.to_dict()['packet']) storage = TraceFlowData((index,)) self.assertEqual(storage.tcp, (index,)) diff --git a/tests/foundation/traceflow/test_tcp.py b/tests/foundation/traceflow/test_tcp.py index 112937dea1..dd27cb5c2a 100644 --- a/tests/foundation/traceflow/test_tcp.py +++ b/tests/foundation/traceflow/test_tcp.py @@ -20,14 +20,16 @@ def setUp(self) -> None: def _packet(self, *, index: int, src: str = '192.0.2.1', dst: str = '198.51.100.2', srcport: int = 12345, dstport: int = 443, syn: bool = False, fin: bool = False, rst: bool = False, timestamp: float = 1.25, - frame: object | None = None): + frame: object | None = None, seq: int = 0, ack: int = 0, + payload: bytes = b''): from pcapkit.const.reg.linktype import LinkType from pcapkit.foundation.traceflow.data.tcp import Packet if frame is None: frame = {'frame': index} return Packet(LinkType.ETHERNET, index, frame, syn, fin, rst, ip_address(src), - ip_address(dst), srcport, dstport, timestamp) + ip_address(dst), srcport, dstport, timestamp, + seq, ack, b'tcp-header', bytearray(payload)) def test_tcp_trace_ipv4_fin_submit_cache_callback_and_dump(self) -> None: """One direction, traced with ``bidirectional=False``. @@ -286,6 +288,90 @@ def test_a_half_open_connection_is_still_reported_at_end_of_capture(self) -> Non self.assertEqual(len(callbacks), before + 1) self.assertEqual(len(trace.index), 1) + def test_the_application_layer_is_not_reassembled_unless_asked_for(self) -> None: + """``analyse`` is off by default, and off means nothing is buffered. + + Reassembling a flow's payload is a cost tracing does not otherwise pay, so + a caller who only wanted frame numbers must not be charged for it. + + """ + from pcapkit.foundation.traceflow.tcp import TCP + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unknown-unit-format') + trace.trace(self._packet(index=1, syn=True, seq=0)) + trace.trace(self._reply(index=2, timestamp=1.5, seq=0, payload=b'hello')) + + bufid, = trace._buffer + self.assertIsNone(trace._buffer[bufid].reassembly) + + flow, = trace.index + self.assertIsNone(flow.packet) + + def test_analyse_gives_a_flow_its_application_layer_per_direction(self) -> None: + """One reassembled datagram per direction, parsed on demand. + + The tracer does not reassemble the stream itself -- it feeds + :class:`~pcapkit.foundation.reassembly.tcp.TCP`, which is why the segments + below are delivered *out of order* and still come back in sequence order. + A tracer concatenating payloads as they arrived would return ``b'worldhello'``. + + """ + from pcapkit.foundation.traceflow.tcp import TCP + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unknown-unit-format', analyse=True) + + # client: SYN, then the second half of its request before the first + trace.trace(self._packet(index=1, syn=True, seq=100)) + trace.trace(self._packet(index=2, seq=106, payload=b'world', timestamp=1.5)) + trace.trace(self._packet(index=3, seq=101, payload=b'hello', timestamp=1.6)) + # server: one reply + trace.trace(self._reply(index=4, seq=500, payload=b'reply', timestamp=1.7)) + + flow, = trace.index + datagrams = flow.packet + self.assertIsNotNone(datagrams) + self.assertEqual(len(datagrams), 2, 'expected one datagram per direction') + + payloads = {dgram.id.src[1]: bytes(dgram.payload) for dgram in datagrams} + # sequence order, not arrival order -- this is the reassembler's work + self.assertEqual(payloads[12345], b'helloworld') + self.assertEqual(payloads[443], b'reply') + + def test_the_application_layer_is_reassembled_only_on_first_read(self) -> None: + """``Index.packet`` holds a :class:`Deferred` until somebody reads it. + + The same postponement reassembly uses for + :attr:`Datagram.packet `, + one layer out: the flow defers its *reassembly*, and each datagram then + defers its own *parse*. + + """ + from pcapkit.foundation.traceflow.data.data import Deferred + from pcapkit.foundation.traceflow.tcp import TCP + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unknown-unit-format', analyse=True) + trace.trace(self._packet(index=1, syn=True, seq=100)) + trace.trace(self._packet(index=2, seq=101, payload=b'hello', timestamp=1.5)) + + flow, = trace.index + key = flow.__map__.get('packet', 'packet') + self.assertIsInstance(flow.__dict__[key], Deferred, + 'the flow was reassembled before anyone asked') + + first = flow.packet + # resolved in place, so a second read is the same object rather than a + # second reassembly + self.assertNotIsInstance(flow.__dict__[key], Deferred) + self.assertIs(flow.packet, first) + + # and the mapping views report it under its own name, resolved + self.assertIn('packet', flow) + self.assertIs(flow.to_dict()['packet'], first) + self.assertIs(flow['packet'], first) + def test_the_buffer_id_is_canonical_and_stays_a_tuple(self) -> None: """Both directions reduce to the same key, whichever is seen first. diff --git a/tests/toolkit/test_dpkt_unit.py b/tests/toolkit/test_dpkt_unit.py index 28b55ad7e1..d03fe5d36b 100644 --- a/tests/toolkit/test_dpkt_unit.py +++ b/tests/toolkit/test_dpkt_unit.py @@ -826,5 +826,73 @@ def test_tcp_reassembly_matches_the_default_engine(self) -> None: self.assertEqual(actual[1], expected[1]) +@unittest.skipUnless(HAS_RUNTIME and HAS_DPKT, 'runtime dependencies not installed') +class DPKTTimestampTests(unittest.TestCase): + """The engine must not drop the timestamp DPKT hands it. + + DPKT's reader yields ``(timestamp, bytes)`` and only the octets become a + packet, so a frame does not know when it was captured unless the engine says + so. It did not, and + :func:`~pcapkit.interface.misc.follow_tcp_stream` -- which reads frames back + after the extraction loop has finished -- had nothing to pass its reassembler + but a bound zero. + + """ + + def setUp(self) -> None: + purge_modules(['pcapkit']) + + def test_the_engine_attaches_each_frames_capture_timestamp(self) -> None: + import pcapkit + from pcapkit.toolkit.dpkt import packet2timestamp + + extractor = pcapkit.extract(fin=sample_path('in.pcap'), nofile=True, store=True, + engine='dpkt') + try: + stamps = [packet2timestamp(frame) for frame in extractor.frame] + finally: + close_extractor(extractor) + + self.assertEqual(len(stamps), 6) + # the capture's own clock, not a placeholder and not the host's + self.assertTrue(all(stamp > 1_500_000_000 for stamp in stamps), stamps) + self.assertEqual(sorted(stamps), stamps, 'timestamps went backwards') + + def test_a_frame_from_elsewhere_has_no_timestamp_and_says_so(self) -> None: + """Loudly, rather than defaulting -- a zero would silently misdate. + + Only a frame that came through the engine carries one, so a packet built + by hand has to be refused rather than dated to the epoch. + + """ + import dpkt + + from pcapkit.toolkit.dpkt import packet2timestamp + from pcapkit.utilities.exceptions import UnsupportedCall + + bare = dpkt.ethernet.Ethernet(b'\x00' * 12 + b'\x08\x00' + b'E' + b'\x00' * 19) + with self.assertRaises(UnsupportedCall): + packet2timestamp(bare) + + def test_following_a_stream_agrees_with_the_default_engine(self) -> None: + """And the real timestamp changes nothing, which is worth pinning. + + TCP reassembly has no timeout by default, so the value never reaches a + decision -- the fix removes a workaround rather than altering a result. + + """ + import tempfile + + from pcapkit.interface.misc import follow_tcp_stream + + def follow(engine: str): + with tempfile.TemporaryDirectory() as tempdir: + streams = follow_tcp_stream(fin=sample_path('in.pcap'), engine=engine, + fout=tempdir, format='json') + return [(len(stream.packets), stream.conversations) for stream in streams] + + self.assertEqual(follow('dpkt'), follow('default')) + + if __name__ == '__main__': unittest.main() diff --git a/tests/toolkit/test_pyshark_unit.py b/tests/toolkit/test_pyshark_unit.py index 1dccdc76c8..5338768aa3 100644 --- a/tests/toolkit/test_pyshark_unit.py +++ b/tests/toolkit/test_pyshark_unit.py @@ -27,9 +27,11 @@ def __init__(self, *, ipv6: bool = False, tcp: bool = True, FakeLayer('ethernet', src='aa:aa:aa:aa:aa:aa'), FakeLayer('ip', src='192.0.2.1', dst='198.51.100.1'), # ``flags_reset`` is PyShark's spelling of Wireshark's - # ``tcp.flags.reset``, the field the RST flag comes from + # ``tcp.flags.reset``, the field the RST flag comes from. ``seq`` and + # ``ack`` are reported as strings too, like every PyShark field. FakeLayer('tcp', srcport='1234', dstport='80', - flags_syn='1', flags_fin='0', flags_reset='0'), + flags_syn='1', flags_fin='0', flags_reset='0', + seq='101', ack='202'), ] self._contains = set() if ip: From 4daf2bf1a30c3a19ca2ba58d740e0ce9291af1eb Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 17 Sep 2026 18:28:15 -0400 Subject: [PATCH 7/8] traceflow: state what analyse=True actually returns, and pin it Three review findings, none of which changes behaviour -- the behaviour was right and the documentation overpromised. The class docstring said `Index.packet` "holds one reassembled datagram per direction". It holds one per direction *per acknowledgement number*: the reassembler buckets as `_buffer[BUFID].ack[ACK]` and emits one datagram per bucket, and `_make_segment` passes `ack` through untouched. Measured on three request/response round trips over one connection: six datagrams, three each way. That is the useful shape rather than an accident -- the acknowledgement number advances exactly when the peer has spoken, so each datagram's payload is one application message that `Datagram.packet` can parse alone, where merging a direction would hand the parser several concatenated messages and have it read only the first. The docstring now says so and shows the six. The existing test could not see this: every segment it sends carries the default `ack=0`, which is the one shape where "one per direction" is the whole truth. Its assertion message said the general rule; it now says the reason, and a new test drives three real round trips and pins all six datagrams by (source port, acknowledgement number). `submit()` did not say that an open flow's `packet` is a mid-capture snapshot. The `Deferred` resolves on first read and is fixed there, so an `Index` a caller kept cannot see later segments and carries no marker distinguishing it from a final result. Measured: read at two frames gives `b'first'`, stays `b'first'` after the third arrives, and is `b'firstsecond'` after `finish()`. Documented, and pinned by a test. This method's own cache is not the cause -- `trace()` and `finish()` both clear it -- and the docstring says that too, so the note is not mistaken for a caching bug later. Docs: `Deferred` and `DeferredPacket` move from `traceflow/tcp.rst` to `traceflow/index.rst`, next to `TraceFlowData`, which is where reassembly declares its equivalents. They live in `traceflow/data/data.py`, so declaring them under `tcp.rst`'s `data.tcp` module directive was wrong twice over. A fourth finding is pre-existing and filed as #443 rather than folded in here: conflicting retransmissions are resolved last-write-wins and still reported COMPLETE. `git diff origin/main` over `reassembly/tcp.py` touches neither overlap branch. Full suite on 3.14.7: 932 passed, 17 skipped (930 before, plus the two tests added here). --- .../pcapkit/foundation/traceflow/index.rst | 8 ++ .../pcapkit/foundation/traceflow/tcp.rst | 8 -- pcapkit/foundation/traceflow/tcp.py | 50 +++++++++- tests/foundation/traceflow/test_tcp.py | 99 ++++++++++++++++++- 4 files changed, 154 insertions(+), 11 deletions(-) diff --git a/docs/source/pcapkit/foundation/traceflow/index.rst b/docs/source/pcapkit/foundation/traceflow/index.rst index 4b8330dc4a..dc3d6dcb8e 100644 --- a/docs/source/pcapkit/foundation/traceflow/index.rst +++ b/docs/source/pcapkit/foundation/traceflow/index.rst @@ -61,3 +61,11 @@ Auxiliary Data .. autoclass:: pcapkit.foundation.traceflow.data.data.TraceFlowData :members: :show-inheritance: + +.. autoclass:: pcapkit.foundation.traceflow.data.data.Deferred + :members: + :show-inheritance: + +.. autoclass:: pcapkit.foundation.traceflow.data.data.DeferredPacket + :members: + :show-inheritance: diff --git a/docs/source/pcapkit/foundation/traceflow/tcp.rst b/docs/source/pcapkit/foundation/traceflow/tcp.rst index 03408938ef..cff16823b4 100644 --- a/docs/source/pcapkit/foundation/traceflow/tcp.rst +++ b/docs/source/pcapkit/foundation/traceflow/tcp.rst @@ -165,14 +165,6 @@ Data Structures :members: :show-inheritance: -.. autoclass:: pcapkit.foundation.traceflow.data.data.Deferred - :members: - :show-inheritance: - -.. autoclass:: pcapkit.foundation.traceflow.data.data.DeferredPacket - :members: - :show-inheritance: - Type Variables -------------- diff --git a/pcapkit/foundation/traceflow/tcp.py b/pcapkit/foundation/traceflow/tcp.py index 33e5606cb7..13e1d5c9d5 100644 --- a/pcapkit/foundation/traceflow/tcp.py +++ b/pcapkit/foundation/traceflow/tcp.py @@ -81,12 +81,39 @@ class TCP(TraceFlow[BufferID, Buffer[_AT], Index, Packet[_AT]], Generic[_AT]): Note: With ``analyse=True`` a flow also carries its **application layer**: :attr:`Index.packet ` - holds one reassembled datagram per direction, each with its payload parsed - on demand. The tracer does not reassemble the stream itself -- it feeds + holds the flow's reassembled datagrams, each with its payload parsed on + demand. The tracer does not reassemble the stream itself -- it feeds :class:`~pcapkit.foundation.reassembly.tcp.TCP`, which already implements :rfc:`815` and copes with the reordering and retransmission that a tracer concatenating payloads in capture order would silently corrupt. + How many datagrams that is follows from the reassembler's own notion of a + datagram, which is **per direction and per acknowledgement number**: it + buckets as ``self._buffer[BUFID].ack[ACK]`` and emits one datagram per + bucket. Since :meth:`_make_segment` passes each segment's ``ack`` through + untouched, a direction yields one datagram per distinct acknowledgement + number it carried -- not one datagram per direction. Three request/response + round trips on one connection therefore give **six** datagrams, three each + way, one per exchange: + + .. code-block:: text + + src=12345 ack=501 payload=b'req1' src=443 ack=105 payload=b'resp1' + src=12345 ack=506 payload=b'req2' src=443 ack=109 payload=b'resp2' + src=12345 ack=511 payload=b'req3' src=443 ack=113 payload=b'resp3' + + That is deliberate rather than incidental: the acknowledgement number + advances exactly when the peer has spoken, so bucketing on it splits a + conversation at its message boundaries, and each datagram's payload is one + application message that :attr:`Datagram.packet + ` can parse on its + own. Merging a direction into a single stream would instead hand the + application parser several concatenated messages and have it read only the + first. A direction that carried one acknowledgement number throughout -- + a single request and its reply, which is what the unit tests exercise -- + does collapse to one datagram each way, which is where "one per direction" + holds. + It is **off by default** because buffering every traced payload is a cost tracing does not otherwise pay, and tracing's per-packet cost is something this package has deliberately driven down. Nothing is reassembled, parsed @@ -445,6 +472,25 @@ def submit(self) -> 'tuple[Index, ...]': flow. Such a flow is therefore reported here but has not fired its callback; :meth:`finish` is what does that, at the end of the capture. + With ``analyse=True`` that makes an open flow's + :attr:`Index.packet ` + a **snapshot**, and a mid-capture one. The + :class:`~pcapkit.foundation.traceflow.data.data.Deferred` resolves when + it is first read and is then fixed in place, so it holds the flow as it + stood at *that read* -- not as it stood when this method returned, and + not as it will stand once the conversation ends. Read it while the flow + is open and the datagrams cover only the segments seen so far; segments + arriving afterwards are reassembled into the buffer but cannot reach an + already-resolved snapshot, and nothing on the returned :class:`Index` + distinguishes one from the final result that :meth:`finish` produces. + + This call's own cache is not the cause and does not soften it: both + :meth:`trace` and :meth:`finish` clear ``__cached__['submit']``, so a + later call rebuilds the tuple with fresh + :class:`~pcapkit.foundation.traceflow.data.data.Deferred` objects. It is + the *previously returned* :class:`Index`, if a caller kept one, that + cannot catch up. For a result that is final, read after :meth:`finish`. + """ if (cached := self.__cached__.get('submit')) is not None: return cached diff --git a/tests/foundation/traceflow/test_tcp.py b/tests/foundation/traceflow/test_tcp.py index dd27cb5c2a..fa38840f5c 100644 --- a/tests/foundation/traceflow/test_tcp.py +++ b/tests/foundation/traceflow/test_tcp.py @@ -316,6 +316,11 @@ def test_analyse_gives_a_flow_its_application_layer_per_direction(self) -> None: below are delivered *out of order* and still come back in sequence order. A tracer concatenating payloads as they arrived would return ``b'worldhello'``. + Every segment here carries the default ``ack=0``, which is the *only* + reason the count is one per direction -- see + :meth:`test_each_exchange_of_a_multi_round_trip_flow_is_its_own_datagram` + for what a real, advancing acknowledgement number does. + """ from pcapkit.foundation.traceflow.tcp import TCP @@ -332,13 +337,105 @@ def test_analyse_gives_a_flow_its_application_layer_per_direction(self) -> None: flow, = trace.index datagrams = flow.packet self.assertIsNotNone(datagrams) - self.assertEqual(len(datagrams), 2, 'expected one datagram per direction') + self.assertEqual(len(datagrams), 2, + 'one acknowledgement number each way, so one datagram each way') payloads = {dgram.id.src[1]: bytes(dgram.payload) for dgram in datagrams} # sequence order, not arrival order -- this is the reassembler's work self.assertEqual(payloads[12345], b'helloworld') self.assertEqual(payloads[443], b'reply') + def test_each_exchange_of_a_multi_round_trip_flow_is_its_own_datagram(self) -> None: + """A direction yields one datagram per acknowledgement number, not one total. + + The reassembler buckets as ``self._buffer[BUFID].ack[ACK]`` and emits one + datagram per bucket, and :meth:`~pcapkit.foundation.traceflow.tcp.TCP._make_segment` + passes ``ack`` through untouched. So the count follows the *conversation*: + the acknowledgement number advances exactly when the peer has spoken, which + splits each direction at its message boundaries and leaves every datagram's + payload one application message that can be parsed on its own. + + Pinned because the sibling test above holds ``ack`` at its default of zero + throughout, and so cannot see this: it is the one shape in which "one + datagram per direction" is the whole truth. + + """ + from pcapkit.foundation.traceflow.tcp import TCP + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unknown-unit-format', analyse=True) + + # three request/response round trips on one connection, with the + # acknowledgement number advancing as it does on any real exchange + trace.trace(self._packet(index=1, syn=True, seq=100, ack=0)) + trace.trace(self._reply(index=2, syn=True, seq=500, ack=101, timestamp=1.3)) + trace.trace(self._packet(index=3, seq=101, ack=501, timestamp=1.4)) + trace.trace(self._packet(index=4, seq=101, ack=501, payload=b'req1', timestamp=1.5)) + trace.trace(self._reply(index=5, seq=501, ack=105, payload=b'resp1', timestamp=1.6)) + trace.trace(self._packet(index=6, seq=105, ack=506, payload=b'req2', timestamp=1.7)) + trace.trace(self._reply(index=7, seq=506, ack=109, payload=b'resp2', timestamp=1.8)) + trace.trace(self._packet(index=8, seq=109, ack=511, payload=b'req3', timestamp=1.9)) + trace.trace(self._reply(index=9, seq=511, ack=113, payload=b'resp3', timestamp=2.0)) + + flow, = trace.index + datagrams = flow.packet + self.assertIsNotNone(datagrams) + self.assertEqual(len(datagrams), 6, + 'three exchanges each way, one datagram per exchange') + + # keyed by (source port, acknowledgement number), which is what the + # reassembler buckets on + got = {(dgram.id.src[1], dgram.id.ack): bytes(dgram.payload) + for dgram in datagrams} + self.assertEqual(got, { + (12345, 501): b'req1', + (12345, 506): b'req2', + (12345, 511): b'req3', + (443, 105): b'resp1', + (443, 109): b'resp2', + (443, 113): b'resp3', + }) + + # the payload-free handshake and bare acknowledgement raise no datagram + # of their own -- they fill no hole, so no bucket of theirs is emitted + self.assertNotIn((12345, 0), got) + self.assertNotIn((443, 101), got) + + def test_an_open_flows_application_layer_is_a_snapshot_of_when_it_was_read(self) -> None: + """Reading ``packet`` on an open flow freezes it there. + + :meth:`~pcapkit.foundation.traceflow.tcp.TCP.submit` deliberately reports + open flows without finalising them, and the + :class:`~pcapkit.foundation.traceflow.data.data.Deferred` resolves on first + read and is then fixed in place. So an :class:`Index` a caller kept from a + mid-capture read cannot see segments that arrived afterwards, and nothing on + it says so -- which is why the docstring tells callers to read after + :meth:`~pcapkit.foundation.traceflow.tcp.TCP.finish` for a final result. + + """ + from pcapkit.foundation.traceflow.tcp import TCP + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unknown-unit-format', analyse=True) + trace.trace(self._packet(index=1, syn=True, seq=100)) + trace.trace(self._packet(index=2, seq=101, payload=b'first', timestamp=1.5)) + + held, = trace.index # a caller keeping a mid-capture Index + resolved = held.packet # ... and resolving its Deferred here + self.assertEqual([bytes(dgram.payload) for dgram in resolved], [b'first']) + + # the rest of the conversation reaches the buffer, but not the snapshot + trace.trace(self._packet(index=3, seq=106, payload=b'second', timestamp=1.6)) + self.assertIs(held.packet, resolved, + 'the resolved snapshot was rebuilt behind the caller') + self.assertEqual([bytes(dgram.payload) for dgram in held.packet], [b'first']) + + # a fresh read does see it, because trace() cleared the submit cache + trace.finish() + final, = trace.index + self.assertEqual([bytes(dgram.payload) for dgram in final.packet], + [b'firstsecond']) + def test_the_application_layer_is_reassembled_only_on_first_read(self) -> None: """``Index.packet`` holds a :class:`Deferred` until somebody reads it. From c8e72b87209660610be4a76a96315334d91774f1 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Thu, 17 Sep 2026 19:07:47 -0400 Subject: [PATCH 8/8] traceflow: the ACK bucket boundary is the peer's turn, not a message boundary Review finding, and correct. The docstring added in 908913106 said bucketing on the acknowledgement number "splits a conversation at its message boundaries", so each datagram's payload is one parseable application message. That holds only where an exchange is one request and one reply. Under pipelining it does not, because several requests in flight before any reply all carry the same acknowledgement number and so share a bucket. Measured on this tree: two requests back to back on one ACK come back as a single datagram carrying b'req1req2', where the wording implied two. So the docstring now says the boundary is the peer's turn -- one datagram per exchange -- and states plainly that this narrows the concatenation to within an exchange rather than eliminating it, and that a parser handed a pipelined datagram still sees only the first message. The reason for documenting the behaviour rather than changing it survives that correction: merging a whole direction concatenates every message it ever sent, which is strictly worse than concatenating within one exchange. The claim was too absolute, not wrong about which option is better. Pinned by test_pipelined_sends_share_a_datagram_because_the_ack_never_moved, since a limitation that lives only in prose is one the next reader has to rediscover. tests/foundation/traceflow: 18 passed. --- pcapkit/foundation/traceflow/tcp.py | 30 +++++++++++++++----- tests/foundation/traceflow/test_tcp.py | 39 ++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/pcapkit/foundation/traceflow/tcp.py b/pcapkit/foundation/traceflow/tcp.py index 13e1d5c9d5..f1fa80a05d 100644 --- a/pcapkit/foundation/traceflow/tcp.py +++ b/pcapkit/foundation/traceflow/tcp.py @@ -104,13 +104,29 @@ class TCP(TraceFlow[BufferID, Buffer[_AT], Index, Packet[_AT]], Generic[_AT]): That is deliberate rather than incidental: the acknowledgement number advances exactly when the peer has spoken, so bucketing on it splits a - conversation at its message boundaries, and each datagram's payload is one - application message that :attr:`Datagram.packet - ` can parse on its - own. Merging a direction into a single stream would instead hand the - application parser several concatenated messages and have it read only the - first. A direction that carried one acknowledgement number throughout -- - a single request and its reply, which is what the unit tests exercise -- + direction wherever the other end got a word in -- one datagram per + **exchange**. Merging a whole direction into one stream would instead hand + the application parser every message it ever sent, concatenated, and have + it read only the first. + + Be precise about what that does and does not buy, though, because the + boundary is the peer's turn and **not** the application's message + boundary. Where an exchange is one request and one reply, the two coincide + and each datagram's payload is a single message that + :attr:`Datagram.packet + ` can parse alone. + Where a sender **pipelines** -- several requests in flight before any reply + -- they all carry the same acknowledgement number, so they share a bucket + and are concatenated after all. Measured: two requests sent back to back + on one acknowledgement number come back as a single datagram carrying + ``b'req1req2'``, which + :meth:`test_pipelined_sends_share_a_datagram_because_the_ack_never_moved` + pins. So this narrows the concatenation to within one exchange rather than + eliminating it, and an application parser handed a pipelined datagram + still sees only the first message. + + A direction that carried one acknowledgement number throughout -- a single + request and its reply, which is what most of the unit tests exercise -- does collapse to one datagram each way, which is where "one per direction" holds. diff --git a/tests/foundation/traceflow/test_tcp.py b/tests/foundation/traceflow/test_tcp.py index fa38840f5c..696520e9ea 100644 --- a/tests/foundation/traceflow/test_tcp.py +++ b/tests/foundation/traceflow/test_tcp.py @@ -401,6 +401,45 @@ def test_each_exchange_of_a_multi_round_trip_flow_is_its_own_datagram(self) -> N self.assertNotIn((12345, 0), got) self.assertNotIn((443, 101), got) + def test_pipelined_sends_share_a_datagram_because_the_ack_never_moved(self) -> None: + """The bucket boundary is the peer's turn, not the message boundary. + + The sibling test above gets one datagram per request because each request + waited for its reply, so every one carried a fresh acknowledgement number. + A sender that *pipelines* -- several requests in flight before any reply -- + emits them all on the same acknowledgement number, so they land in one + bucket and are concatenated. + + Pinned because the class docstring would otherwise be read as promising one + application message per datagram, which holds only where an exchange is one + request and one reply. This is the case where it does not. + + """ + from pcapkit.foundation.traceflow.tcp import TCP + + with tempfile.TemporaryDirectory() as tempdir: + trace = TCP(tempdir, 'unknown-unit-format', analyse=True) + + trace.trace(self._packet(index=1, syn=True, seq=100, ack=0)) + trace.trace(self._reply(index=2, syn=True, seq=500, ack=101, timestamp=1.1)) + # two requests back to back, both acknowledging the same server byte + trace.trace(self._packet(index=3, seq=101, ack=501, payload=b'req1', timestamp=1.2)) + trace.trace(self._packet(index=4, seq=105, ack=501, payload=b'req2', timestamp=1.3)) + trace.trace(self._reply(index=5, seq=501, ack=109, payload=b'resp1', timestamp=1.4)) + + flow, = trace.index + datagrams = flow.packet + self.assertIsNotNone(datagrams) + + got = {(dgram.id.src[1], dgram.id.ack): bytes(dgram.payload) + for dgram in datagrams} + # the two requests are *one* datagram, not two -- concatenated, and an + # application parser handed this sees only the first message + self.assertEqual(got, { + (12345, 501): b'req1req2', + (443, 109): b'resp1', + }) + def test_an_open_flows_application_layer_is_a_snapshot_of_when_it_was_read(self) -> None: """Reading ``packet`` on an open flow freezes it there.