diff --git a/docs/source/conf.py b/docs/source/conf.py index c06b691413..5037c04b1b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -8,14 +8,22 @@ import importlib import logging import os +import pkgutil import sys import typing from typing import TYPE_CHECKING +# NB: a private module of ``sphinx-autodoc-typehints``, used deliberately -- see +# ``bind_type_checking_names`` below for what it buys and why reimplementing it +# here would be worse. If a future release moves it the import fails loudly at +# build time, which is the outcome to want: the alternative is a build that +# silently goes back to emitting unresolvable annotations. +from sphinx_autodoc_typehints._resolver import resolve_type_guarded_imports + import pcapkit if TYPE_CHECKING: - from typing import Any, Dict, List + from typing import Any, Dict, List, Optional from sphinx.application import Sphinx os.environ['PCAPKIT_SPHINX'] = '1' @@ -137,6 +145,16 @@ always_document_param_types = False typehints_document_rtype = True +# NB: a ``:rtype: None`` field says nothing the rendered signature has not already +# said, and injecting it is actively harmful in one measured case: for a docstring +# whose field list is followed by a directive, +# ``sphinx_autodoc_typehints`` computes an insertion point *inside* that directive +# and splits it from its content. ``PCAP_CT.run`` was the case -- its trailing +# ``Note:`` lost its body and the build reported ``Content block expected for the +# "note" directive; none found``. Skipping the ``None`` returns removes the noise +# and the corruption together; non-``None`` returns are still documented. +typehints_document_rtype_none = False + toc_object_entries = False # Add any paths that contain templates here, relative to this directory. @@ -207,6 +225,111 @@ def maybe_skip_member(app: 'Sphinx', what: str, name: str, # pylint: disable=un return skip +def bind_type_checking_names(app: 'Sphinx') -> None: + """Execute every ``pcapkit`` module's ``if TYPE_CHECKING:`` block. + + Annotations throughout the package are quoted and their types imported only + under :data:`~typing.TYPE_CHECKING`, which is right at runtime and awkward + here. Autodoc reads a class attribute's type with + :func:`typing.get_type_hints`, and a single unresolvable name makes that raise + :exc:`NameError` for the *whole class*; Sphinx then falls back to the raw + ``__annotations__`` strings. So ``pre: 'ToSPrecedence'`` was rendered as the + bare word ``ToSPrecedence``, which the Python domain has to resolve by + suffix-matching -- and both :class:`pcapkit.const.ipv4.tos_pre.ToSPrecedence` + and its generator :class:`pcapkit.vendor.ipv4.tos_pre.ToSPrecedence` end in + it. That is where the bulk of the ``more than one target found`` warnings came + from, and roughly half of the links Sphinx picked went to the vendor crawler + rather than to the enumeration the attribute actually holds. + + Binding the guarded names into each module up front lets + :func:`typing.get_type_hints` succeed, so the annotation is stringified from + the resolved object and carries its full dotted path. Nothing is suppressed: + the references are qualified rather than silenced. + + ``sphinx-autodoc-typehints`` already does this per module, lazily, for the + objects it processes -- but it skips classes outright, since it keys off + ``__globals__`` which a class does not have, so class attributes never + benefited. Its implementation is reused rather than rewritten because it + executes the block one statement at a time, so an unimportable optional + dependency (``pcap``, ``pcapfile``) cannot strand the names declared after it. + + One consequence is worth knowing before it is met as a mystery: making the + annotations resolvable also makes a malformed one **fatal**. A quoted + annotation with stray whitespace inside the quotes -- ``rank: ' int'``, the + space that belongs after the colon typed one character late -- is meaningless + to a type checker and was previously harmless here, because these annotations + failed earlier with :exc:`NameError`, which + :func:`sphinx.util.typing.get_type_hints` catches. Now they resolve, and Python + 3.14's :mod:`annotationlib` rejects the leading space with a + :exc:`SyntaxError` that :func:`~sphinx.util.typing.get_type_hints` does *not* + catch, so it aborts the whole build. Two such annotations existed and both are + fixed at source; a third would stop the build rather than be worked around + here, which is the right way round for a typo. + + Args: + app: Sphinx application. + + """ + for info in pkgutil.walk_packages(pcapkit.__path__, 'pcapkit.', + onerror=lambda name: None): + try: + module = importlib.import_module(info.name) + except Exception as exc: # pylint: disable=broad-except + # A module that cannot be imported has no documentable annotations + # either, so this is not worth failing the build over -- but it is + # worth saying out loud rather than passing silently. + logger.info('skipped unimportable module %s: %s', info.name, exc) + continue + resolve_type_guarded_imports(app.config.autodoc_mock_imports, module) + + +def claim_attribute_signature(app: 'Sphinx', what: str, name: str, # pylint: disable=unused-argument + obj: 'Any', options: 'Dict[str, Any]', # pylint: disable=unused-argument + signature: 'Optional[str]', # pylint: disable=unused-argument + return_annotation: 'Optional[str]') -> 'Optional[tuple]': # pylint: disable=unused-argument + """Stop a class-valued attribute being given the class's own signature. + + Several class attributes here hold a *class* as their value -- + :attr:`Protocol.__schema__ ` and + the ``__protocol_type__`` of each reassembly and flow-tracing class. Autodoc + documents them as attributes, so it collects no signature for them, but + ``sphinx_autodoc_typehints`` keys its own handler off ``callable(obj)`` alone, + sees a class, and hands one back. Sphinx 9.1 then stores it with + ``signatures[0] = ...`` on a list it never populated, which raises + ``IndexError: list assignment index out of range`` -- and autodoc turns that + into ``error while formatting signature for ...`` and drops the member from the + page entirely. + + Claiming the event first is what prevents that. The returned value has to be + non-:data:`None` to win ``emit_firstresult`` yet must not look like a + ``(args, retann)`` pair, so it is the empty tuple: falsy, and of the wrong + length for the ``len(result) == 2`` guard Sphinx applies before unpacking. An + attribute has no signature of its own to lose, so nothing is suppressed here + beyond a value that was never meaningful. + + ``callable(obj)`` keeps this off ordinary attributes and off properties, whose + ``obj`` is the :class:`property` itself and therefore not callable -- those + already resolve correctly and are left alone. + + Args: + app: Sphinx application. + what: Type of the object being documented. + name: Fully qualified name of the object. + obj: The object itself. + options: Directive options. + signature: Signature autodoc has so far, if any. + return_annotation: Return annotation autodoc has so far, if any. + + Returns: + The empty tuple for a class-valued attribute, to claim the event without + recording a signature; :data:`None` otherwise, to let other handlers run. + + """ + if what in {'attribute', 'property'} and callable(obj): + return () + return None + + def remove_module_docstring(app: 'Sphinx', what: str, name: str, # pylint: disable=unused-argument obj: 'Any', options: 'Dict[str, Any]', lines: 'List[str]') -> None: # pylint: disable=unused-argument if what == "module" and "pcapkit" in name: @@ -243,7 +366,12 @@ def source_read(app: 'Sphinx', docname: str, source_text: str) -> 'None': # pyl def setup(app: 'Sphinx') -> None: #app.connect('autodoc-process-docstring', process_docstring, 0) #app.connect("autodoc-process-docstring", remove_module_docstring) + app.connect('builder-inited', bind_type_checking_names) app.connect('autodoc-skip-member', maybe_skip_member) + # NB: below ``sphinx_autodoc_typehints``, which connects at the default 500 and + # would otherwise win the tie on registration order -- conf.py's ``setup`` runs + # after the extensions in ``extensions`` have been set up. + app.connect('autodoc-process-signature', claim_attribute_signature, priority=400) #app.connect('source-read', source_read) #app.connect('autodoc-process-docstring', process_fields) diff --git a/docs/source/pcapkit/foundation/traceflow/traceflow.rst b/docs/source/pcapkit/foundation/traceflow/traceflow.rst index 5e4cb55c73..568281a4d5 100644 --- a/docs/source/pcapkit/foundation/traceflow/traceflow.rst +++ b/docs/source/pcapkit/foundation/traceflow/traceflow.rst @@ -27,7 +27,7 @@ which is an abstract base class for all flow tracing classes. value can be set by :attr:`__protocol_name__` class attribute. .. property:: protocol - :type: Type[Protocol] + :type: typing.Type[pcapkit.protocols.protocol.ProtocolBase] Protocol of current class. @@ -76,22 +76,22 @@ Internal Definitions Type Variables -------------- -.. data:: pcapkit.foundation.traceflow.traceflow._DT +.. data:: _DT :type: typing.Any Buffer ID data structure. -.. data:: pcapkit.foundation.traceflow.traceflow._BT +.. data:: _BT :type: pcapkit.corekit.infoclass.Info Buffer data structure. -.. data:: pcapkit.foundation.traceflow.traceflow._IT +.. data:: _IT :type: pcapkit.corekit.infoclass.Info Index data structure. -.. data:: pcapkit.foundation.traceflow.traceflow._PT +.. data:: _PT :type: pcapkit.corekit.infoclass.Info Packet data structure. diff --git a/docs/source/pcapkit/protocols/internet/ipv4.rst b/docs/source/pcapkit/protocols/internet/ipv4.rst index ddd426da93..fbac300ab3 100644 --- a/docs/source/pcapkit/protocols/internet/ipv4.rst +++ b/docs/source/pcapkit/protocols/internet/ipv4.rst @@ -219,7 +219,7 @@ Data Models :show-inheritance: .. attribute:: del - :type: ToSDelay + :type: pcapkit.const.ipv4.tos_del.ToSDelay Delay. @@ -241,7 +241,7 @@ Data Models :show-inheritance: .. attribute:: class - :type: OptionClass + :type: pcapkit.const.ipv4.option_class.OptionClass Option class. diff --git a/pcapkit/foundation/extraction.py b/pcapkit/foundation/extraction.py index 89ed941303..2c3e89e5fb 100644 --- a/pcapkit/foundation/extraction.py +++ b/pcapkit/foundation/extraction.py @@ -52,6 +52,7 @@ from scapy.packet import Packet as ScapyPacket from typing_extensions import Literal + from pcapkit.corekit.context import ProtocolContext from pcapkit.foundation.reassembly.ipv4 import IPv4 as IPv4_Reassembly from pcapkit.foundation.reassembly.ipv6 import IPv6 as IPv6_Reassembly from pcapkit.foundation.reassembly.tcp import TCP as TCP_Reassembly diff --git a/pcapkit/foundation/reassembly/reassembly.py b/pcapkit/foundation/reassembly/reassembly.py index 92bad08a53..7be2ef42f8 100644 --- a/pcapkit/foundation/reassembly/reassembly.py +++ b/pcapkit/foundation/reassembly/reassembly.py @@ -19,6 +19,20 @@ from pcapkit.utilities.exceptions import UnsupportedCall from pcapkit.utilities.logging import get_logger +# NB: declared above the ``TYPE_CHECKING`` block, not below it, so that +# ``CallbackFn`` can name ``_DT`` outright. As a quoted forward reference it was +# resolvable only from this module's namespace, and every module that spells +# ``CallbackFn`` in an annotation -- ``pcapkit.foundation.registry.foundation`` +# does -- has to evaluate the alias in its own. +# packet +_PT = TypeVar('_PT', bound='Info') +# datagram +_DT = TypeVar('_DT', bound='Info') +# buffer ID +_IT = TypeVar('_IT', bound='tuple') +# buffer +_BT = TypeVar('_BT', bound='Info') + if TYPE_CHECKING: from typing import Any, Callable, Optional, Type @@ -27,7 +41,7 @@ from pcapkit.corekit.infoclass import Info from pcapkit.protocols.protocol import ProtocolBase as Protocol - CallbackFn = Callable[[list['_DT']], None] + CallbackFn = Callable[[list[_DT]], None] __all__ = ['Reassembly'] @@ -35,15 +49,6 @@ #: :data:`pcapkit.utilities.logging.logger`. logger = get_logger(__name__) -# packet -_PT = TypeVar('_PT', bound='Info') -# datagram -_DT = TypeVar('_DT', bound='Info') -# buffer ID -_IT = TypeVar('_IT', bound='tuple') -# buffer -_BT = TypeVar('_BT', bound='Info') - class ReassemblyMeta(abc.ABCMeta): """Meta class to add dynamic support to :class:`Reassembly`. diff --git a/pcapkit/foundation/traceflow/traceflow.py b/pcapkit/foundation/traceflow/traceflow.py index b54d97e903..fb4a868094 100644 --- a/pcapkit/foundation/traceflow/traceflow.py +++ b/pcapkit/foundation/traceflow/traceflow.py @@ -32,6 +32,16 @@ #: :data:`pcapkit.utilities.logging.logger`. logger = get_logger(__name__) +# NB: declared above the ``TYPE_CHECKING`` block, not below it, so that +# ``CallbackFn`` can name ``_IT`` outright. As a quoted forward reference it was +# resolvable only from this module's namespace, and every module that spells +# ``CallbackFn`` in an annotation -- ``pcapkit.foundation.registry.foundation`` +# does -- has to evaluate the alias in its own. +_DT = TypeVar('_DT') +_BT = TypeVar('_BT', bound='Info') +_IT = TypeVar('_IT', bound='Info') +_PT = TypeVar('_PT', bound='Info') + if TYPE_CHECKING: from typing import Any, Callable, DefaultDict, Optional, Type @@ -40,12 +50,7 @@ from pcapkit.corekit.infoclass import Info from pcapkit.protocols.protocol import ProtocolBase as Protocol - CallbackFn = Callable[['_IT'], None] - -_DT = TypeVar('_DT') -_BT = TypeVar('_BT', bound='Info') -_IT = TypeVar('_IT', bound='Info') -_PT = TypeVar('_PT', bound='Info') + CallbackFn = Callable[[_IT], None] class TraceFlowMeta(abc.ABCMeta): diff --git a/pcapkit/protocols/__init__.py b/pcapkit/protocols/__init__.py index 3e61edeb55..b457493e45 100644 --- a/pcapkit/protocols/__init__.py +++ b/pcapkit/protocols/__init__.py @@ -68,7 +68,7 @@ 'HTTP', 'HTTPv1', 'HTTPv2', ] -#: dict[str, Type[Protocol]]: Protocol registry. +#: dict[str, ~typing.Type[Protocol]]: Protocol registry. __proto__ = {} # type: dict[str, Type[ProtocolBase]] for name in __all__: __proto__[name.upper()] = globals()[name] diff --git a/pcapkit/protocols/application/httpv2.py b/pcapkit/protocols/application/httpv2.py index 94628a1df8..f00023f407 100644 --- a/pcapkit/protocols/application/httpv2.py +++ b/pcapkit/protocols/application/httpv2.py @@ -83,8 +83,12 @@ Flags = Schema_FrameType.Flags FrameParser = Callable[[Schema_FrameType, NamedArg(Schema_HTTP, 'header')], Data_HTTP] + # NB: ``Flags`` is bound just above, so it goes in unquoted. Quoting it left a + # bare ``ForwardRef('Flags')`` inside the alias, which every *other* module + # that spells ``FrameConstructor`` in an annotation then had to resolve in its + # own namespace -- where the name does not exist. FrameConstructor = Callable[[DefaultArg(Optional[Data_HTTP]), - KwArg(Any)], Tuple[Schema_FrameType, 'Flags']] + KwArg(Any)], Tuple[Schema_FrameType, Flags]] __all__ = ['HTTP'] diff --git a/pcapkit/protocols/data/internet/hopopt.py b/pcapkit/protocols/data/internet/hopopt.py index fbf155f5b5..6a479c65a3 100644 --- a/pcapkit/protocols/data/internet/hopopt.py +++ b/pcapkit/protocols/data/internet/hopopt.py @@ -240,7 +240,7 @@ class RPLOption(Option): #: RPL instance ID. id: 'int' #: Sender rank. - rank:' int' + rank: 'int' if TYPE_CHECKING: def __init__(self, type: 'Enum_Option', action: 'int', change: 'bool', length: 'int', flags: 'RPLFlags', id: 'int', rank: 'int') -> 'None': ... # pylint: disable=super-init-not-called,unused-argument,redefined-builtin,multiple-statements,line-too-long diff --git a/pcapkit/protocols/data/internet/ipv6_opts.py b/pcapkit/protocols/data/internet/ipv6_opts.py index 91258064e7..afedab2a48 100644 --- a/pcapkit/protocols/data/internet/ipv6_opts.py +++ b/pcapkit/protocols/data/internet/ipv6_opts.py @@ -251,7 +251,7 @@ class RPLOption(Option): #: RPL instance ID. id: 'int' #: Sender rank. - rank: ' int' + rank: 'int' if TYPE_CHECKING: def __init__(self, type: 'Enum_Option', action: 'int', change: 'bool', length: 'int', flags: 'RPLFlags', id: 'int', diff --git a/pcapkit/protocols/internet/internet.py b/pcapkit/protocols/internet/internet.py index d698b7dc04..2630c66ac7 100644 --- a/pcapkit/protocols/internet/internet.py +++ b/pcapkit/protocols/internet/internet.py @@ -83,7 +83,7 @@ class Internet(Protocol[_PT, _ST], Generic[_PT, _ST]): # pylint: disable=abstra #: Layer of protocol. __layer__ = 'Internet' # type: Literal['Internet'] - #: DefaultDict[int, ModuleDescriptor[Protocol] | Type[Protocol]]: Protocol index mapping for decoding next layer, + #: DefaultDict[int, ModuleDescriptor[Protocol] | ~typing.Type[Protocol]]: Protocol index mapping for decoding next layer, #: c.f. :meth:`self._decode_next_layer ` #: & :meth:`self._import_next_layer `. __proto__ = collections.defaultdict( diff --git a/pcapkit/protocols/internet/ipv6.py b/pcapkit/protocols/internet/ipv6.py index 1d0c94556d..1569495142 100644 --- a/pcapkit/protocols/internet/ipv6.py +++ b/pcapkit/protocols/internet/ipv6.py @@ -48,6 +48,7 @@ from typing_extensions import Literal from pcapkit.protocols.protocol import ProtocolBase as Protocol + from pcapkit.protocols.schema.schema import Schema __all__ = ['IPv6'] diff --git a/pcapkit/protocols/internet/ipv6_route.py b/pcapkit/protocols/internet/ipv6_route.py index f8bae07d84..179196c5fa 100644 --- a/pcapkit/protocols/internet/ipv6_route.py +++ b/pcapkit/protocols/internet/ipv6_route.py @@ -48,12 +48,14 @@ if TYPE_CHECKING: from enum import IntEnum as StdlibEnum from ipaddress import IPv6Address - from typing import IO, Any, Callable, DefaultDict, Optional, Type + from typing import IO, Any, Callable, DefaultDict, NoReturn, Optional, Type from aenum import IntEnum as AenumEnum from mypy_extensions import DefaultArg, KwArg, NamedArg from typing_extensions import Literal + from pcapkit.corekit.protochain import ProtoChain + from pcapkit.protocols.protocol import ProtocolBase as Protocol from pcapkit.protocols.schema.internet.ipv6_route import RoutingType as Schema_RoutingType TypeParser = Callable[[Schema_RoutingType, NamedArg(Schema_IPv6_Route, 'header')], Data_IPv6_Route] diff --git a/pcapkit/protocols/link/link.py b/pcapkit/protocols/link/link.py index e348c7d3ef..e5201c3e6c 100644 --- a/pcapkit/protocols/link/link.py +++ b/pcapkit/protocols/link/link.py @@ -68,7 +68,7 @@ class Link(Protocol[_PT, _ST], Generic[_PT, _ST]): # pylint: disable=abstract-m #: Layer of protocol. __layer__ = 'Link' # type: Literal['Link'] - #: DefaultDict[int, ModuleDescriptor[Protocol] | Type[Protocol]]: Protocol index mapping for decoding next layer, + #: DefaultDict[int, ModuleDescriptor[Protocol] | ~typing.Type[Protocol]]: Protocol index mapping for decoding next layer, #: c.f. :meth:`self._decode_next_layer ` #: & :meth:`self._import_next_layer `. __proto__ = collections.defaultdict( diff --git a/pcapkit/protocols/misc/pcap/frame.py b/pcapkit/protocols/misc/pcap/frame.py index a527776196..bdf2f8c7a5 100644 --- a/pcapkit/protocols/misc/pcap/frame.py +++ b/pcapkit/protocols/misc/pcap/frame.py @@ -82,7 +82,7 @@ class Frame(Protocol[Data_Frame, Schema_Frame], # Defaults. ########################################################################## - #: DefaultDict[Enum_LinkType, ModuleDescriptor[Protocol] | Type[Protocol]]: Protocol index mapping for + #: DefaultDict[Enum_LinkType, ModuleDescriptor[Protocol] | ~typing.Type[Protocol]]: Protocol index mapping for #: decoding next layer, c.f. :meth:`self._decode_next_layer ` #: & :meth:`self._import_next_layer `. #: The values should be a tuple representing the module name and class name, or diff --git a/pcapkit/protocols/misc/pcapng.py b/pcapkit/protocols/misc/pcapng.py index 61ab0764d3..74f4814b34 100644 --- a/pcapkit/protocols/misc/pcapng.py +++ b/pcapkit/protocols/misc/pcapng.py @@ -548,7 +548,7 @@ class PCAPNG(Protocol[Data_PCAPNG, Schema_PCAPNG], # Defaults. ########################################################################## - #: DefaultDict[Enum_LinkType, ModuleDescriptor[Protocol] | Type[Protocol]]: Protocol index mapping for + #: DefaultDict[Enum_LinkType, ModuleDescriptor[Protocol] | ~typing.Type[Protocol]]: Protocol index mapping for #: decoding next layer, c.f. :meth:`self._decode_next_layer ` #: & :meth:`self._import_next_layer `. #: The values should be a tuple representing the module name and class name, diff --git a/pcapkit/protocols/transport/sctp.py b/pcapkit/protocols/transport/sctp.py index 5a8b8103ac..5b1cba3848 100644 --- a/pcapkit/protocols/transport/sctp.py +++ b/pcapkit/protocols/transport/sctp.py @@ -332,7 +332,7 @@ class SCTP(Transport[Data_SCTP, Schema_SCTP], #: Payload protocol identifier of the first DATA chunk found in the packet. _ppid = None # type: Optional[Enum_PayloadProtocolIdentifier] - #: DefaultDict[int, ModuleDescriptor[Protocol] | Type[Protocol]]: Protocol + #: DefaultDict[int, ModuleDescriptor[Protocol] | ~typing.Type[Protocol]]: Protocol #: index mapping for decoding next layer, c.f. #: :meth:`self._decode_next_layer ` #: & :meth:`self._import_next_layer `. diff --git a/pcapkit/protocols/transport/tcp.py b/pcapkit/protocols/transport/tcp.py index e8131eb61e..71d9de2387 100644 --- a/pcapkit/protocols/transport/tcp.py +++ b/pcapkit/protocols/transport/tcp.py @@ -302,7 +302,7 @@ class TCP(Transport[Data_TCP, Schema_TCP], # Defaults. ########################################################################## - #: DefaultDict[int, ModuleDescriptor[Protocol] | Type[Protocol]]: Protocol + #: DefaultDict[int, ModuleDescriptor[Protocol] | ~typing.Type[Protocol]]: Protocol #: index mapping for decoding next layer, c.f. #: :meth:`self._decode_next_layer ` #: & :meth:`self._import_next_layer `. diff --git a/pcapkit/protocols/transport/udp.py b/pcapkit/protocols/transport/udp.py index 78873bd5cc..e654131f95 100644 --- a/pcapkit/protocols/transport/udp.py +++ b/pcapkit/protocols/transport/udp.py @@ -65,7 +65,7 @@ class UDP(Transport[Data_UDP, Schema_UDP], # Defaults. ########################################################################## - #: DefaultDict[int, ModuleDescriptor[Protocol] | Type[Protocol]]: Protocol + #: DefaultDict[int, ModuleDescriptor[Protocol] | ~typing.Type[Protocol]]: Protocol #: index mapping for decoding next layer, c.f. #: :meth:`self._decode_next_layer ` #: & :meth:`self._import_next_layer `.