fix(ipv6): parse unrecognised extension headers generically (#891)

docs/source/pcapkit/protocols/internet/index.rst

@@ -16,6 +16,7 @@ internet layer, with detailed implementation and methods.
    ipv4
    ipv6
    ipv6_frag
+   ipv6_generic_ext
    ipv6_opts
    ipv6_route
    hopopt

docs/source/pcapkit/protocols/internet/ipv6_generic_ext.rst

@@ -0,0 +1,72 @@
+IPv6_GenericExt - Generic IPv6 Extension Header
+================================================
+
+.. module:: pcapkit.protocols.internet.ipv6_generic_ext
+
+:mod:`pcapkit.protocols.internet.ipv6_generic_ext` contains
+:class:`~pcapkit.protocols.internet.ipv6_generic_ext.IPv6_GenericExt`
+only, which implements a **generic** extractor for IPv6 extension
+headers [*]_, standing in for one whenever the header's own dedicated
+parser is unavailable or has failed. :rfc:`6564#section-4` guarantees,
+with an RFC 2119 **MUST**, that any IPv6 extension header defined
+since April 2012 carries the same first two octets:
+
+======= ========= ===================== =====================================
+Octets      Bits        Name                    Description
+======= ========= ===================== =====================================
+  0           0   ``next``                    Next Header
+  1           8   ``len``                     Hdr Ext Len (8-octet units,
+                                                excluding the first 8 octets)
+  2          16   ``payload``                 Header-specific content
+======= ========= ===================== =====================================
+
+so those two octets are parseable without knowing anything else about
+the header. See the module docstring below for the closed exception
+table (``IPv6-Frag`` and ``AH`` each use their own length rule; ``ESP``
+has a dedicated parser whose own info reports no next header rather than
+lacking one, and ``BIT-EMU``, ``253`` and ``254`` have no dedicated
+parser at all -- none of the four ever reaches this class), the two ways
+this class is dispatched to, and why an overrun stops the walk instead of
+clipping it.
+
+.. autoclass:: pcapkit.protocols.internet.ipv6_generic_ext.IPv6_GenericExt
+   :no-members:
+   :show-inheritance:
+
+   .. autoproperty:: name
+   .. autoproperty:: alias
+   .. autoproperty:: length
+   .. autoproperty:: protocol
+   .. autoproperty:: next
+   .. autoproperty:: payload
+   .. autoproperty:: protochain
+
+   .. automethod:: read
+   .. automethod:: make
+
+   .. automethod:: _make_data
+
+   .. automethod:: __post_init__
+   .. automethod:: __index__
+
+Header Schemas
+--------------
+
+.. module:: pcapkit.protocols.schema.internet.ipv6_generic_ext
+
+.. autoclass:: pcapkit.protocols.schema.internet.ipv6_generic_ext.IPv6_GenericExt
+   :members:
+   :show-inheritance:
+
+Data Models
+-----------
+
+.. module:: pcapkit.protocols.data.internet.ipv6_generic_ext
+
+.. autoclass:: pcapkit.protocols.data.internet.ipv6_generic_ext.IPv6_GenericExt
+   :members:
+   :show-inheritance:
+
+.. rubric:: Footnotes
+
+.. [*] :rfc:`6564`

examples/generators/dispatch.py

@@ -392,6 +392,13 @@ def _internet_payload(code: 'int') -> 'bytes':
     if code == TransType.OSPFIGP:
         from pcapkit.protocols.link.ospf import OSPF
         return bytes(OSPF())
+    if code == TransType.Shim6:
+        # #904: no dedicated dissector exists for Shim6 -- IPv6_GenericExt
+        # parses only the two octets RFC 6564 §4 guarantees (next header,
+        # Hdr Ext Len), so this is hand-built rather than constructed
+        # through a class: next=TCP(6), Hdr Ext Len=0 -> an 8-octet header,
+        # six of them padding.
+        return bytes([int(TransType.TCP), 0]) + b'\x00' * 6 + _tcp(9999)
     raise LookupError(f'internet: no payload builder for {code!r}')  # pragma: no cover
 
 
@@ -621,6 +628,12 @@ def _internet_enum() -> 'Any':
     'internet/HIP': ('pcapkit.protocols.internet.hip', 'HIP'),
     'internet/SCTP': ('pcapkit.protocols.transport.sctp', 'SCTP'),
     'internet/OSPFIGP': ('pcapkit.protocols.link.ospf', 'OSPF'),
+    # #904: Shim6 (140) previously had no entry at all, and the default
+    # factory made it resolve to Raw. It is now registered directly at
+    # IPv6_GenericExt, which parses the RFC 6564 §4 generic layout it has
+    # never had a dedicated dissector for -- a deliberate addition, not a
+    # regression.
+    'internet/Shim6': ('pcapkit.protocols.internet.ipv6_generic_ext', 'IPv6_GenericExt'),
 
     # -- TCP.__proto__ (port) --------------------------------------------------
     'tcp/20': ('pcapkit.protocols.application.ftp', 'FTP_DATA'),

pcapkit/__init__.py

@@ -116,7 +116,7 @@
     'L2TPv2', 'OSPF', 'RARP', 'S_Tag', 'VLAN',
 
     'AH', 'ESP', 'IP', 'IPsec', 'IPv4', 'IPv6', 'IPX',      # Internet Layer
-    'HIP', 'HOPOPT', 'IPv6_Frag', 'IPv6_Opts', 'IPv6_Route', 'MH',
+    'HIP', 'HOPOPT', 'IPv6_Frag', 'IPv6_GenericExt', 'IPv6_Opts', 'IPv6_Route', 'MH',
                                                             # IPv6 Extension Header
 
     'TCP', 'UDP', 'SCTP',                                   # Transport Layer

pcapkit/protocols/__init__.py

@@ -57,7 +57,7 @@
     'AH', 'ESP', 'IP', 'IPsec', 'IPv4', 'IPv6', 'IPX',
 
     # IPv6 Extension Header
-    'HIP', 'HOPOPT', 'IPv6_Frag', 'IPv6_Opts',
+    'HIP', 'HOPOPT', 'IPv6_Frag', 'IPv6_GenericExt', 'IPv6_Opts',
     'IPv6_Route', 'MH',
 
     # Transport Layer

pcapkit/protocols/data/__init__.py

@@ -117,6 +117,9 @@
     # IPv6 Fragment Header
     'IPv6_Frag',
 
+    # Generic IPv6 Extension Header
+    'IPv6_GenericExt',
+
     # IPv6 Destination Options Header
     'IPv6_Opts',
     'IPv6_Opts_RPLFlags', 'IPv6_Opts_MPLFlags', 'IPv6_Opts_DFFFlags',

pcapkit/protocols/data/internet/__init__.py

@@ -131,6 +131,9 @@
 # IPv6 Fragment Header
 from pcapkit.protocols.data.internet.ipv6_frag import IPv6_Frag
 
+# Generic IPv6 Extension Header
+from pcapkit.protocols.data.internet.ipv6_generic_ext import IPv6_GenericExt
+
 # IPv6 Destination Options
 from pcapkit.protocols.data.internet.ipv6_opts import CALIPSOOption as IPv6_Opts_CALIPSOOption
 from pcapkit.protocols.data.internet.ipv6_opts import DFFFlags as IPv6_Opts_DFFFlags
@@ -266,6 +269,9 @@
     # IPv6 Fragment Header
     'IPv6_Frag',
 
+    # Generic IPv6 Extension Header
+    'IPv6_GenericExt',
+
     # IPv6 Destination Options Header
     'IPv6_Opts',
     'IPv6_Opts_RPLFlags', 'IPv6_Opts_MPLFlags', 'IPv6_Opts_DFFFlags',

pcapkit/protocols/data/internet/ipv6_generic_ext.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+"""data model for generically-parsed IPv6 extension headers"""
+
+from typing import TYPE_CHECKING
+
+from pcapkit.corekit.infoclass import info_final
+from pcapkit.protocols.data.protocol import Protocol
+
+if TYPE_CHECKING:
+    from typing import Optional
+
+    from pcapkit.const.ipv6.extension_header import ExtensionHeader
+    from pcapkit.const.reg.transtype import TransType
+
+__all__ = ['IPv6_GenericExt']
+
+
+@info_final
+class IPv6_GenericExt(Protocol):
+    """Data model for a generically-parsed IPv6 extension header.
+
+    See :class:`pcapkit.protocols.internet.ipv6_generic_ext.IPv6_GenericExt`
+    for how each field below is derived.
+
+    """
+
+    #: The extension header this instance stands in for -- the numeric code
+    #: the caller dispatched on, resolved to its
+    #: :class:`~pcapkit.const.ipv6.extension_header.ExtensionHeader` member.
+    #: :data:`None` when ``alias`` named no such member (:meth:`read
+    #: <pcapkit.protocols.internet.ipv6_generic_ext.IPv6_GenericExt.read>`
+    #: sets it so on a lookup miss, and the class property at
+    #: :attr:`~pcapkit.protocols.internet.ipv6_generic_ext.IPv6_GenericExt.protocol`
+    #: is typed to match).
+    protocol: 'Optional[ExtensionHeader]'
+    #: Next header, parsed off the wire. :data:`None` when the declared
+    #: length would have overrun what remained and the walk stopped instead
+    #: of trusting it.
+    next: 'Optional[TransType]'
+    #: Length of this extension header, in octets, actually consumed.
+    length: 'int'
+    #: Original parsing error, if this instance was reached as a
+    #: :func:`~pcapkit.utilities.decorators.beholder` fallback rather than by
+    #: direct dispatch.
+    error: 'Optional[Exception]'

pcapkit/protocols/internet/__init__.py

@@ -25,6 +25,7 @@
 from pcapkit.protocols.internet.hip import HIP
 from pcapkit.protocols.internet.hopopt import HOPOPT
 from pcapkit.protocols.internet.ipv6_frag import IPv6_Frag
+from pcapkit.protocols.internet.ipv6_generic_ext import IPv6_GenericExt
 from pcapkit.protocols.internet.ipv6_opts import IPv6_Opts
 from pcapkit.protocols.internet.ipv6_route import IPv6_Route
 from pcapkit.protocols.internet.mh import MH
@@ -39,6 +40,6 @@
 __all__ = [
     'ETHERTYPE',                                        # Protocol Numbers
     'AH', 'ESP', 'IP', 'IPsec', 'IPv4', 'IPv6', 'IPX',  # Internet Layer
-    'HIP', 'HOPOPT', 'IPv6_Frag',
+    'HIP', 'HOPOPT', 'IPv6_Frag', 'IPv6_GenericExt',
     'IPv6_Opts', 'IPv6_Route', 'MH',                    # IPv6 Extension Header
 ]

pcapkit/protocols/internet/ipv6.py

@@ -56,6 +56,67 @@ class IPv6(IP[Data_IPv6, Schema_IPv6],
            schema=Schema_IPv6, data=Data_IPv6):
     """This class implements Internet Protocol version 6."""
 
+    ##########################################################################
+    # Defaults.
+    ##########################################################################
+
+    #: Extension header codes that have a *dedicated* parser class in this
+    #: package whose own layout follows :rfc:`6564#section-4`'s generic
+    #: ``next`` + ``Hdr Ext Len`` format (see the module docstring of
+    #: :mod:`pcapkit.protocols.internet.ipv6_generic_ext` for the exception
+    #: table in full). When that dedicated parser raises,
+    #: :meth:`_import_next_layer` substitutes
+    #: :class:`~pcapkit.protocols.internet.ipv6_generic_ext.IPv6_GenericExt`
+    #: instead of letting the generic
+    #: :func:`~pcapkit.utilities.decorators.beholder` fall back to plain
+    #: :class:`~pcapkit.protocols.misc.raw.Raw`, which has no ``next`` field
+    #: and used to crash the whole packet at :meth:`_decode_next_layer`'s
+    #: ``proto = info.next`` (GitHub issue #891).
+    #:
+    #: :attr:`~pcapkit.const.ipv6.extension_header.ExtensionHeader.Shim6`
+    #: is deliberately absent, even though its wire format also conforms:
+    #: this package has never had a dedicated parser for it to begin with
+    #: (``pcapkit/protocols/internet/NotImplemented/shim6.py`` is a 0-byte
+    #: placeholder, excluded from the wheel by ``MANIFEST.in``), so there is
+    #: no "own parser" here for it to raise from -- ``Shim6`` reaches
+    #: :class:`IPv6_GenericExt` by *direct* registration instead (see the
+    #: bottom of :mod:`pcapkit.protocols.internet.ipv6_generic_ext`), which
+    #: already produces exactly this class without needing this set to name
+    #: it. ``ESP``, ``BIT-EMU``, ``253`` and ``254`` are absent, but not for
+    #: the same reason as each other, and not because a generic fallback
+    #: would help them:
+    #:
+    #: * ``ESP`` *does* have a dedicated, registered parser
+    #:   (:class:`~pcapkit.protocols.internet.esp.ESP`) -- it is excluded
+    #:   because :rfc:`4303` places the real Next Header byte inside the
+    #:   encrypted trailer, so its own info always *carries* a ``next``
+    #:   attribute (unlike ``BIT-EMU``/``253``/``254`` below), just one that is
+    #:   :data:`None` whenever the payload could not be decrypted -- which,
+    #:   with no key material available to a generic parse, is always. The
+    #:   :meth:`_decode_next_layer` walk below still ends there, one iteration
+    #:   later, because :data:`None` fails
+    #:   :class:`~pcapkit.const.ipv6.extension_header.ExtensionHeader`'s
+    #:   constructor at the top of the loop -- the *existing* end-of-chain
+    #:   path, unrelated to the structural check this set exists for.
+    #: * ``BIT-EMU``, ``253`` and ``254`` have no dedicated parser at all, so
+    #:   they resolve to plain :class:`~pcapkit.protocols.misc.raw.Raw`, whose
+    #:   info has no ``next`` *attribute* -- this is what the structural check
+    #:   catches.
+    #:
+    #: :meth:`_decode_next_layer`'s walk stops cleanly at whichever of these
+    #: (or any other IANA code this package has not implemented) it meets,
+    #: and keeps this layer's own header intact instead of losing the whole
+    #: packet as it used to.
+    __generic_ext_codes__ = frozenset({
+        Enum_ExtensionHeader.HOPOPT,
+        Enum_ExtensionHeader.IPv6_Route,
+        Enum_ExtensionHeader.IPv6_Opts,
+        Enum_ExtensionHeader.Mobility_Header,
+        Enum_ExtensionHeader.HIP,
+        Enum_ExtensionHeader.IPv6_Frag,
+        Enum_ExtensionHeader.AH,
+    })
+
     ##########################################################################
     # Properties.
     ##########################################################################
@@ -335,7 +396,6 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None,
             # record protocol name
             # self._protos = ProtoChain(name, chain, alias)
             _protos.append(next_)
-            proto = info.next
 
             # update header & payload length
             hdr_len += next_.length  # type: ignore[assignment]
@@ -350,6 +410,45 @@ def _decode_next_layer(self, ipv6: 'Data_IPv6', proto: 'Optional[int]' = None,
             # is what gets handed to ``super()._decode_next_layer`` below
             payload = payload[next_.length:]
 
+            # GitHub issue #891: a layer with no ``next`` field cannot
+            # safely continue the walk. This is a *structural* check --
+            # does the parsed info even carry a ``next`` attribute? -- not
+            # a fixed set of codes, and deliberately so: HOPOPT, IPv6-Route,
+            # IPv6-Opts, MH, HIP, IPv6-Frag and AH all have dedicated
+            # parsers whose data carries ``next``, and Shim6 and any
+            # recognised header whose own parser raised are both handled by
+            # :class:`~pcapkit.protocols.internet.ipv6_generic_ext.IPv6_GenericExt`,
+            # which also carries ``next`` (possibly :data:`None`, on an
+            # overrun -- see its module docstring). Every IANA extension
+            # header code this package has not implemented a dedicated
+            # parser for -- today that is ``BIT-EMU``, ``253`` and ``254``,
+            # and tomorrow it is whatever IANA assigns next -- has no
+            # generic fallback either (see :attr:`__generic_ext_codes__`'s
+            # docstring for why), so :meth:`_import_next_layer` returns a
+            # plain :class:`~pcapkit.protocols.misc.raw.Raw`, whose info
+            # carries no ``next`` at all. Reading ``info.next`` on that
+            # unconditionally is what used to raise ``AttributeError``
+            # here and let a further-out :func:`~pcapkit.utilities.decorators.beholder`
+            # catch it and degrade the *whole* packet -- the actual #891
+            # defect, for every code nobody has implemented. Stopping here
+            # instead keeps this layer's own fields (still recorded above,
+            # in ``self._exthdr`` and in the packet dict) and reports no
+            # further next header, exactly like the overrun case.
+            #
+            # This has to run -- and, on a hit, has to set ``proto`` --
+            # *before* the fragment-header special case below: IPv6-Frag
+            # always carries a real ``next`` (the ``hasattr`` check above
+            # never actually fires for it), and that ``next`` is the real
+            # transport layer's code, which the fragment branch's own
+            # ``break`` must leave in ``proto`` for the final
+            # ``super()._decode_next_layer`` call below the loop to dispatch
+            # to correctly.
+            if not hasattr(info, 'next'):
+                proto = None
+                break
+
+            proto = info.next
+
             # keep original data after fragment header
             if ex_proto == Enum_ExtensionHeader.IPv6_Frag:
                 ipv6.__update__({
@@ -387,6 +486,35 @@ def _import_next_layer(self, proto: 'int', length: 'Optional[int]' = None, *,  #
         Returns:
             Instance of next layer.
 
+        Notes:
+            If the dedicated parser for a code in :attr:`__generic_ext_codes__`
+            raises, this substitutes
+            :class:`~pcapkit.protocols.internet.ipv6_generic_ext.IPv6_GenericExt`
+            for it rather than letting the exception reach the
+            :func:`~pcapkit.utilities.decorators.beholder` decorating this
+            method, which would otherwise substitute plain
+            :class:`~pcapkit.protocols.misc.raw.Raw` -- and ``Raw`` has no
+            ``next`` field, which is what used to crash the whole packet at
+            :meth:`_decode_next_layer`'s ``proto = info.next`` (GitHub issue
+            #891). Every other exception -- including one raised by
+            ``IPv6_GenericExt`` itself, or by ``ESP``'s own dedicated
+            parser -- still reaches ``beholder`` unchanged, so *this
+            method's own* behaviour for anything outside that closed set is
+            exactly what it was before this method learned the
+            substitution: a plain ``Raw`` for that one layer. What changed
+            for ``BIT-EMU``, ``253`` and ``254`` -- which have no dedicated
+            parser at all, so they were *already* reaching plain ``Raw``
+            with no exception involved -- is one level up:
+            :meth:`_decode_next_layer` now stops its walk structurally on
+            any layer whose info carries no ``next`` attribute, ``Raw``
+            included, instead of reading ``info.next`` unconditionally and
+            crashing the whole packet. ``ESP`` is unaffected either way: its
+            info always carries a ``next`` (:data:`None`, since :rfc:`4303`
+            encrypts the real value), so neither this substitution nor that
+            structural check ever engages for it, and the walk ends after it
+            exactly as it always has -- see :attr:`__generic_ext_codes__`'s
+            docstring for the full distinction.
+
         """
         if TYPE_CHECKING:
             protocol: 'Type[ProtocolBase]'
@@ -407,7 +535,19 @@ def _import_next_layer(self, proto: 'int', length: 'Optional[int]' = None, *,  #
         else:
             protocol = self._lookup_next_layer(self.__proto__, proto)
 
-        next_ = protocol(file_, length, version=version, extension=extension,  # type: ignore[abstract]
-                         alias=proto, packet=packet, layer=self._exlayer, protocol=self._exproto,
-                         __context__=self._exctx)
+        try:
+            next_ = protocol(file_, length, version=version, extension=extension,  # type: ignore[abstract]
+                             alias=proto, packet=packet, layer=self._exlayer, protocol=self._exproto,
+                             __context__=self._exctx)
+        except Exception as exc:
+            from pcapkit.protocols.internet.ipv6_generic_ext import \
+                IPv6_GenericExt  # isort: skip # pylint: disable=import-outside-toplevel
+
+            if not (extension and protocol is not IPv6_GenericExt
+                    and proto in self.__generic_ext_codes__):
+                raise
+
+            next_ = IPv6_GenericExt(file_, length, version=version, extension=extension,
+                                    alias=proto, error=exc, packet=packet, layer=self._exlayer,
+                                    protocol=self._exproto, __context__=self._exctx)
         return next_