diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 97f5d8d33..4a495ac18 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -152,8 +152,35 @@ jobs: echo "Upstream Wireshark captures were used, matching their pinned SHA-256 digests." fi + # The ``test`` job above already ran every unit-tier test for real, on this + # same commit. Re-running them here (a bare ``pytest -q`` used to) bought + # no signal and cost ~21.75 minutes per Python version -- ~130 minutes per + # PR push across the matrix (see #715). So this selects only the + # fixture-dependent tier plus the handful of unit-tier modules whose + # capture read is skipped, not exercised, in the `test` job -- see + # tests/_tiers.py's module docstring and fixture_tier_paths() for why a + # module can be unit-tier by path and still belong here. The selection is + # not spelled out as literal flags: it is asked of tests/_tiers.py + # directly (FIXTURE_TIER_SELECTOR), so this step and that module cannot + # drift the way two independent copies of the same list could. + # + # A failure to compute the selection intentionally does not fall back to + # a bare `pytest -q` -- that would silently re-introduce the whole-suite + # duplication this change exists to remove, with nothing in the log to + # say so. - name: Run full test suite - run: python -m pytest -q + shell: bash + run: | + if ! selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths()))"); then + echo "::error title=Could not compute the fixture-tier selection::tests._tiers.fixture_tier_paths() failed -- see the traceback above. Refusing to fall back to a bare 'pytest -q', which would silently re-run the entire suite instead of failing loudly." + exit 1 + fi + if [ -z "$selection" ]; then + echo "::error title=Fixture-tier selection is empty::tests._tiers.fixture_tier_paths() returned nothing, which cannot be right -- tests/integration alone should always be part of it. Refusing to run pytest with no arguments." + exit 1 + fi + echo "Fixture-dependent selection: $selection" + python -m pytest -q $selection # ``CHANGELOG.md`` is generated from the newest entry under # ``docs/source/changelog/`` by ``util/changelog_md.py``, so it falls out of step @@ -235,5 +262,16 @@ jobs: echo "Upstream Wireshark captures were used, matching their pinned SHA-256 digests." fi + # Deliberately still the whole suite, unlike the `integration` job above: + # this is the shipping gate's own independent re-verification (see the + # job comment above), not the same accidental duplication #715 measured + # for `integration`. `gate` does not run per PR push -- only via + # `create-release.yml`'s `gate-only: true` -- so it is not part of the + # ~130 minutes/push #715 measured, and partitioning it would trade away + # the "trust nothing, re-run everything" property this job exists for. + # Whether `gate` should run at all on a commit the matrix already + # covered is #715's separate item 2 ("stop the gate re-running on main + # pushes"), a trigger-frequency question, not a selection one -- out of + # scope here. - name: Run full test suite run: python -m pytest -q diff --git a/tests/_tiers.py b/tests/_tiers.py index 592a45d63..5f7711987 100644 --- a/tests/_tiers.py +++ b/tests/_tiers.py @@ -41,12 +41,28 @@ :func:`~tests._support.sample_path` call go ahead? * :func:`explain` -- the message that says what is wrong and what to do about it. +* :func:`skip_idiom_modules`, :func:`skip_idiom_test_ids`, and + :func:`fixture_tier_paths` -- which unit-tier modules (and, more precisely, + which of their test methods) skip a generated capture rather than reading it + for real, and the complete pytest selection the fixture-dependent tier needs + in order to still exercise them. :file:`tests/conftest.py` drives the first four at collection time, and :func:`tests._support.sample_path` consults :func:`check_unit_tier_read` on every call. Both paths no-op when git cannot answer, so an unpacked source tarball still runs its tests. +:func:`fixture_tier_paths` is what the ``integration`` job of +:file:`.github/workflows/unit-tests.yml` runs directly, rather than embedding +its own copy of the selection: the job calls this module at run time (``python +-c "from tests._tiers import fixture_tier_paths; ..."``, see +:data:`FIXTURE_TIER_SELECTOR`) instead of spelling out ``tests/integration`` +and the two globs itself. That closes the same gap :data:`UNIT_TIER_SELECTION` +leaves open in the other direction: a workflow that asks this module for the +answer cannot drift from it, whereas one that repeats the answer as literal +flags always can. See :func:`fixture_tier_paths`'s docstring for why the +selection is more than just the complement of the unit tier. + The two halves cover each other. The collection-time audit is static, so it sees a violation in a test that never runs -- one skipped for a missing optional engine, say -- but only when the capture name is a string literal. The call-time @@ -113,10 +129,11 @@ __all__ = [ 'GeneratedFixtureInUnitTierError', 'TierGuardWarning', 'SampleCall', 'ROOT', 'TESTS_ROOT', 'SAMPLE_ROOT', 'REGENERATE_SAMPLES_CMD', 'UNIT_TIER_SELECTION', - 'FIXTURE_TIER_SUFFIXES', 'FIXTURE_TIER_DIRS', + 'FIXTURE_TIER_SUFFIXES', 'FIXTURE_TIER_DIRS', 'FIXTURE_TIER_SELECTOR', 'is_unit_tier', 'committed_captures', 'committed_capture_names', 'guard_unavailable_reason', 'handled_lines', 'sample_path_calls', 'audit_module', 'check_unit_tier_read', 'explain', + 'skip_idiom_modules', 'skip_idiom_test_ids', 'fixture_tier_paths', ] #: Repository root, i.e. the parent of the directory holding this file. The @@ -143,6 +160,18 @@ #: Directories under :file:`tests/` that are fixture-dependent in their entirety, #: matching the ``--ignore`` arguments above. FIXTURE_TIER_DIRS = frozenset({'integration'}) +#: The fixture-dependent tier's selector, verbatim from the ``integration`` job of +#: :file:`.github/workflows/unit-tests.yml`. Unlike :data:`UNIT_TIER_SELECTION` this +#: cannot be written as a fixed list of flags: which paths belong in the +#: selection depends on git (which captures are committed) and on which unit-tier +#: modules use the skip idiom, neither of which is stable source text pytest +#: flags could name. So the workflow does not repeat the answer -- it asks this +#: module for it at run time, with this exact invocation, quoted here so a reader +#: can run precisely what CI runs. See :func:`fixture_tier_paths`. +FIXTURE_TIER_SELECTOR = ( + 'python -c "from tests._tiers import fixture_tier_paths; ' + 'print(\' \'.join(fixture_tier_paths()))"' +) #: Suffixes that make a tracked file worth suggesting as a replacement capture. #: Committedness itself is whatever git says -- this filter only keeps the #: suggestion in :func:`explain` from offering :file:`out.txt` as a capture. @@ -637,3 +666,220 @@ def check_unit_tier_read(name: 'str', module_path: 'Optional[str]', return None return explain(name, module_path, lineno) + + +@functools.lru_cache(maxsize=1) +def skip_idiom_modules() -> 'tuple[pathlib.Path, ...]': + """Unit-tier modules that skip rather than read, when a capture is absent. + + A unit-tier module is allowed to read a generated capture provided it + handles the capture's absence at the call site -- the sanctioned + ``try``/``except FileNotFoundError`` idiom :data:`SKIP_IDIOM_EXAMPLE` + demonstrates. On the ``test`` job, which never runs + :data:`REGENERATE_SAMPLES_CMD`, such a call always finds the capture + missing and the test skips; :func:`~tests._support.sample_path` raises a + bare :exc:`FileNotFoundError` rather than + :exc:`GeneratedFixtureInUnitTierError` for exactly this reason. So the + call's *real* body -- the assertions that run once the capture is actually + read -- is only ever exercised on a run that has built the fixtures + first, i.e. the fixture-dependent tier. + + That makes this function's result the missing half of the fixture + tier's selection: :data:`FIXTURE_TIER_DIRS` and + :data:`FIXTURE_TIER_SUFFIXES` name the modules that are fixture-dependent + *in their entirety*, but these modules are unit-tier by every other + measure and would never be selected by that rule alone. Leaving them out + of the fixture tier's selection would mean their skip-idiom reads are + exercised nowhere at all once the unit and fixture tiers stop overlapping. + See :func:`fixture_tier_paths`, which folds this in. + + Only literal capture names are seen here, the same limitation + :func:`audit_module` carries and for the same reason: a name computed at + call time, e.g. ``sample_path(sample)`` in a parametrised loop, is + invisible to a static pass. No module in the suite does this inside a + handled call today -- see ``test_every_handled_call_names_a_literal_capture`` + in :file:`tests/test_tier_guard.py`, which exists to catch the day one does, + since such a call would silently drop out of this function's result and + its fixture-backed coverage would quietly stop running anywhere. + + Returns: + Absolute paths, sorted, to every unit-tier module with at least one + handled read of a capture git does not track. Empty when git cannot + answer :func:`committed_captures` -- with no committed-capture list to + compare against, nothing can be told apart from a committed read, and + claiming otherwise would be a guess. + + """ + tracked = committed_captures() + if tracked is None: + return () + + modules = [] + for path in sorted(TESTS_ROOT.rglob('*.py')): + if not is_unit_tier(path): + continue + for call in sample_path_calls(str(path)): + if call.handled and call.name is not None and not _is_committed(call.name, tracked): + modules.append(path) + break + return tuple(modules) + + +def _enclosing_scope(tree: 'ast.Module', lineno: 'int') -> 'Optional[tuple[Optional[str], str]]': + """The ``(class name or None, function name)`` most tightly wrapping ``lineno``. + + A manual recursive descent rather than :func:`ast.walk`: the answer needs + the *nesting path* down to a line -- which function, and which class (if + any) that function is a method of -- and a breadth-first walk does not + carry that context as it goes. "Most tightly" is decided by span, so a + helper function defined inside a test method and containing the line in + question would win over the test method itself; nothing in this suite's + skip idiom does that today, but the rule is stated so a future one is + handled the same way :func:`sample_path_calls` already treats nesting. + + Returns: + :data:`None` if no function definition contains ``lineno`` at all -- + a module-level call, which does not happen in this suite today. + + """ + best = None # type: Optional[tuple[Optional[str], str, int]] + + def visit(node: 'ast.AST', class_name: 'Optional[str]') -> None: + nonlocal best + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + start = child.lineno + end = getattr(child, 'end_lineno', None) or start + if start <= lineno <= end: + span = end - start + if best is None or span < best[2]: + best = (class_name, child.name, span) + visit(child, class_name) + elif isinstance(child, ast.ClassDef): + visit(child, child.name) + else: + visit(child, class_name) + + visit(tree, None) + if best is None: + return None + return best[0], best[1] + + +@functools.lru_cache(maxsize=1) +def skip_idiom_test_ids() -> 'tuple[str, ...]': + """Precise pytest node IDs for every call :func:`skip_idiom_modules` counts. + + :func:`skip_idiom_modules` names the *files* that need to run again in the + fixture-dependent tier; this names the individual test methods inside + them. The difference matters because those files are not small -- + :file:`tests/protocols/misc/test_pcapng_unit.py`'s + ``PCAPNGUnitTests`` alone runs to thousands of lines and dozens of + unrelated methods -- and pulling in the whole module to reach the one that + actually needs the fixtures would reintroduce, for just these modules, the + same duplication :func:`fixture_tier_paths` exists to remove everywhere + else. Each :class:`unittest.TestCase` method here gets its own ``setUp``, + with no ``setUpClass`` state shared across siblings in the classes this + has been checked against, so selecting one by node ID runs it exactly as + it would run as part of the whole file. + + Falls back to the *module* path when a handled call cannot be resolved to + one enclosing function -- :func:`_enclosing_scope` returns :data:`None` for + a module-level call, which is not a shape this suite's skip idiom uses + today, but a fallback that still runs the file is safer than one that + drops the call's coverage entirely. + + Returns: + Node IDs (``path/to/module.py::TestClass::test_method``) and/or plain + module paths, sorted, relative to :data:`ROOT` and POSIX-separated. + Empty under the same condition as :func:`skip_idiom_modules`. + + """ + tracked = committed_captures() + if tracked is None: + return () + + ids = [] + for path in sorted(TESTS_ROOT.rglob('*.py')): + if not is_unit_tier(path): + continue + + relevant = [ + call for call in sample_path_calls(str(path)) + if call.handled and call.name is not None and not _is_committed(call.name, tracked) + ] + if not relevant: + continue + + relative = path.relative_to(ROOT).as_posix() + tree = _parse(str(path)) + if tree is None: + # sample_path_calls() above already parsed this module successfully, + # so this should not happen -- but if it somehow does, run the whole + # file rather than claim no coverage is needed. + ids.append(relative) + continue + + for call in relevant: + scope = _enclosing_scope(tree, call.lineno) + if scope is None: + ids.append(relative) + continue + class_name, func_name = scope + ids.append(f'{relative}::{class_name}::{func_name}' if class_name else f'{relative}::{func_name}') + + return tuple(sorted(set(ids))) + + +@functools.lru_cache(maxsize=1) +def fixture_tier_paths() -> 'tuple[str, ...]': + """The complete pytest selection the fixture-dependent tier needs to run. + + Three things go into it: + + * one directory argument per :data:`FIXTURE_TIER_DIRS` entry, e.g. + ``'tests/integration'``, rather than every file under it individually; + * every module under :file:`tests/` whose name matches + :data:`FIXTURE_TIER_SUFFIXES` and is not already reachable through one + of those directory arguments, found on disk rather than assumed, so a + new ``*_runtime.py`` module is picked up the moment it is added; and + * :func:`skip_idiom_test_ids`, without which those calls' fixture-backed + reads would never run for real anywhere -- see its docstring and + :func:`skip_idiom_modules`'s. + + The third component is node IDs, not module paths, deliberately: it is + added by :func:`skip_idiom_test_ids` rather than by + :func:`skip_idiom_modules` directly, so that pulling in one skip-idiom test + does not also pull in every unrelated method the same module happens to + hold. Without that, the two files this suite has today would add roughly a + hundred tests back into this selection to reach the two or three that + actually need it. + + This is deliberately *not* "the complement of :func:`is_unit_tier`" + expressed as a single predicate the way :func:`is_unit_tier` itself is: the + third component is not part of the fixture tier by path, only by the + coverage gap leaving it out would open. Combining them here, once, is what + lets the ``integration`` job of :file:`.github/workflows/unit-tests.yml` + ask a single function for its selection instead of re-deriving it -- see + :data:`FIXTURE_TIER_SELECTOR`. + + Returns: + Paths and/or node IDs relative to :data:`ROOT`, POSIX-separated, + sorted, and deduplicated -- ready to hand to ``pytest`` as positional + arguments. Never empty while :data:`FIXTURE_TIER_DIRS` is non-empty: + the directory arguments do not depend on git, so they are always + present even when :func:`skip_idiom_test_ids` cannot answer. + + """ + paths = {f'tests/{name}' for name in FIXTURE_TIER_DIRS} + + for path in sorted(TESTS_ROOT.rglob('*.py')): + relative_to_tests = path.relative_to(TESTS_ROOT) + if not FIXTURE_TIER_DIRS.isdisjoint(relative_to_tests.parts[:-1]): + continue # already reachable through a directory argument above + if path.name.endswith(FIXTURE_TIER_SUFFIXES): + paths.add(path.relative_to(ROOT).as_posix()) + + paths.update(skip_idiom_test_ids()) + + return tuple(sorted(paths)) diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index 2509f4b5e..b6255380f 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -26,6 +26,7 @@ """ from __future__ import annotations +import ast import pathlib import re import subprocess @@ -59,6 +60,39 @@ def write_module(directory: 'pathlib.Path', name: 'str', source: 'str') -> 'path return path +def job_section(text: 'str', name: 'str') -> 'str': + """The YAML text of job ``name``, from its header to the next top-level job. + + A plain slice rather than a real YAML parse: :class:`WorkflowAgreementTests` + and :class:`FixtureTierSelectionTests` both need to know what one job's + steps say without tripping over another job that happens to mention the + same words, and a two-space-indented ``key:`` line is what marks a job + boundary in this file however its body is written. + + """ + match = re.search(rf'(?m)^ {re.escape(name)}:\n(.*?)(?=^ \w[\w-]*:\n|\Z)', text, re.DOTALL) + if match is None: + raise AssertionError(f'no job named {name!r} found in the workflow') + return match.group(1) + + +def step_run_block(section: 'str', step_name: 'str') -> 'str': + """The ``run:`` block of the step named ``step_name`` within a job section. + + Scoped to one step, not just one job, because a job can hold several + steps and only one of them is the one a test cares about -- see + :func:`job_section` for why a text slice rather than a YAML parse. + + """ + match = re.search( + rf'(?m)^ - name: {re.escape(step_name)}\n(.*?)(?=^ - name:|\Z)', + section, re.DOTALL, + ) + if match is None: + raise AssertionError(f'no step named {step_name!r} found in this job') + return match.group(1) + + class TierClassificationTests(unittest.TestCase): """:func:`~tests._tiers.is_unit_tier` against the CI ignore rules.""" @@ -109,6 +143,297 @@ def test_ignore_flags_match_the_fixture_tier_constants(self) -> None: self.assertEqual(globs, set(_tiers.FIXTURE_TIER_SUFFIXES)) self.assertEqual(directories, set(_tiers.FIXTURE_TIER_DIRS)) + def test_integration_job_selects_positively_by_asking_tiers_for_it(self) -> None: + """The ``integration`` job's selection runs the other direction, so it + is checked the other way. + + The ``test`` job above is checked by parsing its ``--ignore`` / + ``--ignore-glob`` flags out of the workflow text and comparing them to + :data:`~tests._tiers.FIXTURE_TIER_DIRS` / + :data:`~tests._tiers.FIXTURE_TIER_SUFFIXES` -- a *negative* selection, + so those two regexes are what could drift from this module. The + ``integration`` job instead *selects* the fixture-dependent tier + positively, and a positive selection spelled out as literal paths + would be invisible to those same regexes: nothing would stop it from + drifting from :func:`~tests._tiers.skip_idiom_modules` while this test + stayed green. + + So the ``integration`` job does not spell the selection out. Its "Run + full test suite" step has to call + :func:`~tests._tiers.fixture_tier_paths` directly instead of + reimplementing the answer -- which makes drift structurally + impossible rather than merely checked for, and is what this asserts. + + """ + if not WORKFLOW.is_file(): + self.skipTest(f'{WORKFLOW} is not present, e.g. in a source distribution') + + text = WORKFLOW.read_text(encoding='utf-8') + section = job_section(text, 'integration') + run_block = step_run_block(section, 'Run full test suite') + + self.assertIn( + 'fixture_tier_paths', run_block, + "the integration job's \"Run full test suite\" step no longer calls " + "tests._tiers.fixture_tier_paths() -- see this test's docstring for why a " + "hand-written selection here can drift silently" + ) + self.assertIn('pytest', run_block) + + +class EnclosingScopeTests(unittest.TestCase): + """:func:`~tests._tiers._enclosing_scope`, on synthetic sources. + + Needs no git and no real module -- it is a pure function of an + :mod:`ast` tree and a line number, so it is pinned the same way + :class:`AuditTests` pins :func:`~tests._tiers.audit_module`: against + source written for the purpose, not against whatever the suite happens to + contain today. + + """ + + def test_a_method_on_a_class_reports_both_names(self) -> None: + tree = ast.parse(textwrap.dedent(""" + class Tests: + def test_it(self): + line_two = 2 + """)) + # Line 3 is `def test_it(self):` itself; line 4 is its body. + self.assertEqual(_tiers._enclosing_scope(tree, 4), ('Tests', 'test_it')) + + def test_a_module_level_function_reports_no_class(self) -> None: + tree = ast.parse(textwrap.dedent(""" + def test_it(): + line_two = 2 + """)) + self.assertEqual(_tiers._enclosing_scope(tree, 3), (None, 'test_it')) + + def test_a_line_outside_every_function_reports_nothing(self) -> None: + tree = ast.parse(textwrap.dedent(""" + class Tests: + def test_it(self): + pass + """)) + self.assertIsNone(_tiers._enclosing_scope(tree, 1)) + + def test_the_innermost_function_wins_over_its_enclosing_method(self) -> None: + """A nested helper's line belongs to the helper, not the test method. + + Not a shape the suite's own skip idiom uses today, but + :func:`~tests._tiers.skip_idiom_test_ids` documents that "most tightly + wrapping" is the rule, and this is what pins it. + + """ + tree = ast.parse(textwrap.dedent(""" + class Tests: + def test_it(self): + def helper(): + line_four = 4 + helper() + """)) + self.assertEqual(_tiers._enclosing_scope(tree, 5), ('Tests', 'helper')) + + def test_two_sibling_classes_are_not_confused(self) -> None: + tree = ast.parse(textwrap.dedent(""" + class First: + def test_a(self): + pass + + class Second: + def test_b(self): + line_seven = 7 + """)) + self.assertEqual(_tiers._enclosing_scope(tree, 8), ('Second', 'test_b')) + + +class FixtureTierSelectionTests(unittest.TestCase): + """:func:`~tests._tiers.skip_idiom_modules` and :func:`~tests._tiers.fixture_tier_paths`. + + Together these are what the ``integration`` job runs instead of the whole + suite -- see :class:`WorkflowAgreementTests` for the half of the guarantee + that lives in the workflow file itself. + + """ + + def setUp(self) -> None: + reason = _tiers.guard_unavailable_reason() + if reason is not None: + self.skipTest(f'git cannot answer here: {reason}') + + def test_skip_idiom_modules_matches_an_independent_scan(self) -> None: + """Built from the same lower-level facts, but not by calling the function. + + Re-derived here from :func:`~tests._tiers.sample_path_calls` and + :func:`~tests._tiers.committed_captures` directly, the same way + :class:`CommittedCaptureTests`'s + ``test_every_tracked_name_exists_and_matches_git`` re-derives the + committed set independently of + :func:`~tests._tiers.committed_captures` -- calling + :func:`~tests._tiers.skip_idiom_modules` a second time would only show + that it agrees with itself. + + """ + tracked = _tiers.committed_captures() + assert tracked is not None + + expected = set() + for path in sorted(_tiers.TESTS_ROOT.rglob('*.py')): + if not _tiers.is_unit_tier(path): + continue + for call in _tiers.sample_path_calls(str(path)): + if call.handled and call.name is not None and not _tiers._is_committed(call.name, tracked): + expected.add(path) + break + + self.assertEqual(set(_tiers.skip_idiom_modules()), expected) + + def test_skip_idiom_modules_matches_a_grep_based_scan(self) -> None: + """A second, textually independent check, catching a bug the first one could not. + + The scan above is re-derived from :func:`~tests._tiers.sample_path_calls` + and :func:`~tests._tiers.committed_captures`, so a bug shared by those two + primitives and :func:`~tests._tiers.skip_idiom_modules` would pass it + undetected -- all three would agree with each other and still be wrong. + This check shares nothing with them: it is a plain substring scan of the + file text, no :mod:`ast` involved. + + Two files are excluded on purpose, not overlooked: :file:`tests/_tiers.py` + itself and :file:`tests/test_tier_guard.py`, this module. Both mention + ``sample_path(`` and ``except FileNotFoundError`` freely -- in + docstrings, in error messages, and in source strings ``write_module()`` + writes out for other tests to audit -- without containing a single real + call to either. A first version of this check included them and failed + immediately for exactly that reason, which is the point: a check that + cannot fail is not a check. + + Restricted to ``test_*.py`` for the same reason :mod:`pytest` itself + is (see ``python_files`` in :file:`pyproject.toml`): a module outside + that pattern is not a test module regardless of what + :func:`~tests._tiers.is_unit_tier` says about its path, and + :file:`tests/_tiers.py` is the case in point. + + """ + grep_matches = set() + for path in sorted(_tiers.TESTS_ROOT.rglob('test_*.py')): + if path == _tiers.TESTS_ROOT / 'test_tier_guard.py': + continue + if not _tiers.is_unit_tier(path): + continue + text = path.read_text(encoding='utf-8') + if 'sample_path(' in text and 'except FileNotFoundError' in text: + grep_matches.add(path) + + self.assertEqual(set(_tiers.skip_idiom_modules()), grep_matches) + + def test_every_handled_call_names_a_literal_capture(self) -> None: + """A computed name inside a handler would be invisible to the scan above. + + Pins today's fact that no unit-tier module needs the computed case, so + that the day one does, this fails loudly instead of that module's + fixture-backed coverage silently dropping out of + :func:`~tests._tiers.fixture_tier_paths` -- see + :func:`~tests._tiers.skip_idiom_modules`'s docstring for the mechanism. + + """ + for path in sorted(_tiers.TESTS_ROOT.rglob('*.py')): + if not _tiers.is_unit_tier(path): + continue + for call in _tiers.sample_path_calls(str(path)): + if call.handled and call.name is None: + self.fail( + f'{path.relative_to(_tiers.ROOT)}:{call.lineno} handles a computed ' + f'sample_path() call -- skip_idiom_modules() cannot tell whether it ' + f'reads a generated capture and may need including by hand' + ) + + def test_skip_idiom_test_ids_resolve_to_a_real_method_in_a_skip_idiom_module(self) -> None: + """Every node ID names a method that really exists, in a module the + module-level scan above also flagged. + + Checked by parsing the module independently with :mod:`ast` rather + than by importing it -- unit-tier modules are not meant to be imported + outside pytest collecting them, and a static check is enough to prove + the node ID is not simply wrong. + + """ + skip_idiom = {module.relative_to(_tiers.ROOT).as_posix() for module in _tiers.skip_idiom_modules()} + + for node_id in _tiers.skip_idiom_test_ids(): + with self.subTest(node_id=node_id): + parts = node_id.split('::') + self.assertGreaterEqual(len(parts), 2, f'{node_id!r} is not a module::function node ID') + relative, *scope = parts + self.assertIn(relative, skip_idiom, f'{relative} was not flagged by skip_idiom_modules()') + + tree = ast.parse((_tiers.ROOT / relative).read_text(encoding='utf-8')) + node = tree # type: ast.AST + for name in scope: + found = next( + (child for child in ast.walk(node) + if isinstance(child, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) + and child.name == name), + None, + ) + self.assertIsNotNone(found, f'{node_id}: no definition named {name!r} in {relative}') + node = found + + def test_fixture_tier_paths_includes_every_component(self) -> None: + """The directory, suffix, and skip-idiom components are all present.""" + paths = _tiers.fixture_tier_paths() + self.assertIn('tests/integration', paths) + + for path in sorted(_tiers.TESTS_ROOT.rglob('*.py')): + relative_to_tests = path.relative_to(_tiers.TESTS_ROOT) + under_fixture_dir = not _tiers.FIXTURE_TIER_DIRS.isdisjoint(relative_to_tests.parts[:-1]) + if under_fixture_dir: + continue + if path.name.endswith(_tiers.FIXTURE_TIER_SUFFIXES): + with self.subTest(module=str(relative_to_tests)): + self.assertIn(path.relative_to(_tiers.ROOT).as_posix(), paths) + + for node_id in _tiers.skip_idiom_test_ids(): + with self.subTest(node_id=node_id): + self.assertIn(node_id, paths) + + def test_fixture_tier_paths_never_selects_a_pure_unit_tier_module(self) -> None: + """The property the whole partition depends on: nothing runs twice. + + Every entry is either outside the unit tier by + :func:`~tests._tiers.is_unit_tier`, or is a + :func:`~tests._tiers.skip_idiom_test_ids` node ID scoped to one method + of a :func:`~tests._tiers.skip_idiom_modules` module -- never a + unit-tier module, or a bare unit-tier module path with no ``::`` + scope, or the ``test`` and ``integration`` jobs would be back to + running the same test twice. + + """ + skip_idiom = set(_tiers.skip_idiom_modules()) + node_ids = set(_tiers.skip_idiom_test_ids()) + paths = _tiers.fixture_tier_paths() + self.assertTrue(paths, 'fixture_tier_paths() returned nothing') + + for entry in paths: + with self.subTest(entry=entry): + if '::' in entry: + # A skip-idiom node ID: legal only when it is one of the + # exact IDs skip_idiom_test_ids() names -- not merely a + # module skip_idiom_modules() flagged, which would still + # pull the whole file in and reintroduce the duplication + # this split exists to avoid. + self.assertIn(entry, node_ids) + continue + + candidate = _tiers.ROOT / entry + if candidate.is_dir(): + continue # a directory argument, e.g. tests/integration + self.assertTrue(candidate.is_file(), f'{entry} does not exist') + if _tiers.is_unit_tier(candidate): + self.assertIn( + candidate, skip_idiom, + f'{entry} is a bare unit-tier module path, not a node ID scoped to one ' + f'of its methods -- it would run in both the test and integration jobs' + ) + class CommittedCaptureTests(unittest.TestCase): """Committedness is asked of git, not remembered."""