diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 32f4b7d2d6e08b..3d6f9cdf6965bb 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -22,7 +22,7 @@ import sys import textwrap import types -from collections.abc import Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping from contextlib import suppress from importlib import import_module from importlib.abc import MetaPathFinder @@ -35,15 +35,14 @@ from ._functools import method_cache, noop, pass_none, passthrough from ._itertools import always_iterable, bucket, unique_everseen from ._meta import PackageMetadata, SimplePath -from ._typing import md_none __all__ = [ 'Distribution', 'DistributionFinder', + 'MetadataNotFound', 'PackageMetadata', 'PackageNotFoundError', 'PackagePath', - 'MetadataNotFound', 'SimplePath', 'distribution', 'distributions', @@ -175,9 +174,11 @@ class EntryPoint: ... ValueError: ('Invalid object reference...invalid-name... - The same thing happens on construction. + Construction never raises; the value is validated only when + accessed or loaded. - >>> EntryPoint(name=None, group=None, value='invalid-name') + >>> ep = EntryPoint(name=None, group=None, value='invalid-name') + >>> ep.load() Traceback (most recent call last): ... ValueError: ('Invalid object reference...invalid-name... @@ -213,7 +214,6 @@ class EntryPoint: def __init__(self, name: str, value: str, group: str) -> None: vars(self).update(name=name, value=value, group=group) - self.module def load(self) -> Any: """Load the entry point from its definition. If only a module @@ -335,7 +335,7 @@ def __repr__(self): Repr with classname and tuple constructor to signal that we deviate from regular tuple behavior. """ - return '%s(%r)' % (self.__class__.__name__, tuple(self)) + return f'{self.__class__.__name__}({tuple(self)!r})' def select(self, **params) -> EntryPoints: """ @@ -396,6 +396,25 @@ def __repr__(self) -> str: return f'' +def _identify_dist( + accessor: Callable[[Distribution], str], +) -> Callable[[Distribution], str]: + """ + Decorate a required metadata accessor to identify the distribution + in the error raised when its field is absent. + """ + + @functools.wraps(accessor) + def wrapper(self: Distribution) -> str: + try: + return accessor(self) + except KeyError as exc: + (field,) = exc.args + raise KeyError(f'Metadata field {field} not present in {self}') from None + + return wrapper + + class Distribution(metaclass=abc.ABCMeta): """ An abstract Python distribution package. @@ -556,6 +575,7 @@ def _ensure_metadata_present(self, text: str | None) -> str: raise MetadataNotFound('No package metadata was found.') @property + @_identify_dist def name(self) -> str: """Return the 'Name' metadata for the distribution package.""" return self.metadata['Name'] @@ -566,6 +586,7 @@ def _normalized_name(self): return Prepared.normalize(self.name) @property + @_identify_dist def version(self) -> str: """Return the 'Version' metadata for the distribution package.""" return self.metadata['Version'] @@ -707,7 +728,7 @@ def make_condition(name): def quoted_marker(section): section = section or '' - extra, sep, markers = section.partition(':') + extra, _sep, markers = section.partition(':') if extra and markers: markers = f'({markers})' conditions = list(filter(None, [markers, make_condition(extra)])) @@ -832,7 +853,7 @@ class FastPath: """ @_clear_after_fork # type: ignore[misc] - @functools.lru_cache() + @functools.lru_cache def __new__(cls, root): return super().__new__(cls) @@ -885,6 +906,14 @@ def __init__(self, path: FastPath): From the children in the path, calculate early all of the children that appear to represent metadata (infos) or legacy metadata (eggs). + + Children are considered in reverse lexicographic order, so when + a path supplies more than one metadata directory for a + distribution, as an interrupted or unclean upgrade may leave + behind, the greater one is found first + (python/cpython#100666). Because the order is lexicographic on + the directory name and not on the parsed version, it only + approximates a preference for the newer version. """ base = os.path.basename(path.root).lower() @@ -892,7 +921,7 @@ def __init__(self, path: FastPath): self.infos = FreezableDefaultDict(list) self.eggs = FreezableDefaultDict(list) - for child in path.children(): + for child in sorted(path.children(), key=str.lower, reverse=True): low = child.lower() if low.endswith((".dist-info", ".egg-info")): # rpartition is faster than splitext and suitable for this purpose. @@ -958,15 +987,8 @@ def __init__(self, name: str | None): def normalize(name): """ PEP 503 normalization plus dashes as underscores. - - Specifically avoids ``re.sub`` as prescribed for performance - benefits (see python/cpython#143658). """ - value = name.lower().replace("-", "_").replace(".", "_") - # Condense repeats - while "__" in value: - value = value.replace("__", "_") - return value + return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_') @staticmethod def legacy_normalize(name): @@ -1017,6 +1039,16 @@ def __init__(self, path: SimplePath) -> None: """ self._path = path + def __str__(self) -> str: + """ + Render as the metadata path, identifying the distribution + even when its metadata cannot name it. + + >>> str(Distribution.at('foo-1.0.dist-info')) + 'foo-1.0.dist-info' + """ + return str(self._path) + def read_text(self, filename: str | os.PathLike[str]) -> str | None: with suppress( FileNotFoundError, @@ -1060,7 +1092,7 @@ def _name_from_stem(stem): filename, ext = os.path.splitext(stem) if ext not in ('.dist-info', '.egg-info'): return - name, sep, rest = filename.partition('-') + name, _sep, _rest = filename.partition('-') return name diff --git a/Lib/importlib/metadata/_adapters.py b/Lib/importlib/metadata/_adapters.py index dede395d79a38b..0bc230008502c9 100644 --- a/Lib/importlib/metadata/_adapters.py +++ b/Lib/importlib/metadata/_adapters.py @@ -54,7 +54,7 @@ class Message(email.message.Message): """ - multiple_use_keys = set( + multiple_use_keys = frozenset( map( FoldedCase, [ diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py index 0c20eff3da7522..219b2a86d1755c 100644 --- a/Lib/importlib/metadata/_meta.py +++ b/Lib/importlib/metadata/_meta.py @@ -54,11 +54,11 @@ class SimplePath(Protocol): """ def joinpath( - self, other: str | os.PathLike[str] + self, other: str | os.PathLike[str], / ) -> SimplePath: ... # pragma: no cover def __truediv__( - self, other: str | os.PathLike[str] + self, other: str | os.PathLike[str], / ) -> SimplePath: ... # pragma: no cover @property diff --git a/Lib/importlib/metadata/_text.py b/Lib/importlib/metadata/_text.py index c88cfbb2349c64..a7b87447a1a177 100644 --- a/Lib/importlib/metadata/_text.py +++ b/Lib/importlib/metadata/_text.py @@ -95,5 +95,5 @@ def index(self, sub): return self.lower().index(sub.lower()) def split(self, splitter=' ', maxsplit=0): - pattern = re.compile(re.escape(splitter), re.I) + pattern = re.compile(re.escape(splitter), re.IGNORECASE) return pattern.split(self, maxsplit) diff --git a/Lib/importlib/metadata/_typing.py b/Lib/importlib/metadata/_typing.py deleted file mode 100644 index 32b1d2b98ac987..00000000000000 --- a/Lib/importlib/metadata/_typing.py +++ /dev/null @@ -1,15 +0,0 @@ -import functools -import typing - -from ._meta import PackageMetadata - -md_none = functools.partial(typing.cast, PackageMetadata) -""" -Suppress type errors for optional metadata. - -Although Distribution.metadata can return None when metadata is corrupt -and thus None, allow callers to assume it's not None and crash if -that's the case. - -# python/importlib_metadata#493 -""" diff --git a/Lib/importlib/metadata/diagnose.py b/Lib/importlib/metadata/diagnose.py index e405471ac4d943..f8e77731970da4 100644 --- a/Lib/importlib/metadata/diagnose.py +++ b/Lib/importlib/metadata/diagnose.py @@ -1,6 +1,22 @@ import sys -from . import Distribution +from . import Distribution, MetadataNotFound + + +def describe(dist: Distribution) -> str: + """ + Describe the distribution by name, or by location when the + distribution is too invalid to name itself. + + >>> describe(Distribution.at('missing-1.0.dist-info')) + 'missing-1.0.dist-info (no metadata)' + """ + try: + return dist.name + except MetadataNotFound: + return f'{dist} (no metadata)' + except KeyError: + return f'{dist} (metadata without a name)' def inspect(path): @@ -9,7 +25,7 @@ def inspect(path): if not dists: return print("Found", len(dists), "packages:", end=' ') - print(', '.join(dist.name for dist in dists)) + print(', '.join(map(describe, dists))) def run(): diff --git a/Lib/test/test_importlib/metadata/_path.py b/Lib/test/test_importlib/metadata/_path.py index e63d889f96bf13..fded31b2ba7295 100644 --- a/Lib/test/test_importlib/metadata/_path.py +++ b/Lib/test/test_importlib/metadata/_path.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Protocol, Union, runtime_checkable if TYPE_CHECKING: - from typing_extensions import Self + from typing import Self class Symlink(str): diff --git a/Lib/test/test_importlib/metadata/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py index 3283697d418188..79b2e1e33ad7ff 100644 --- a/Lib/test/test_importlib/metadata/fixtures.py +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -71,7 +71,7 @@ def setUp(self): class DistInfoPkg(OnSysPath, SiteBuilder): - files: FilesSpec = { + files: FilesSpec = frozendict({ "distinfo_pkg-1.0.0.dist-info": { "METADATA": """ Name: distinfo-pkg @@ -95,7 +95,7 @@ class DistInfoPkg(OnSysPath, SiteBuilder): def main(): print("hello world") """, - } + }) def make_uppercase(self): """ @@ -114,7 +114,7 @@ class DistInfoPkgEditable(DistInfoPkg): """ some_hash = '524127ce937f7cb65665130c695abd18ca386f60bb29687efb976faa1596fdcc' - files: FilesSpec = { + files: FilesSpec = frozendict({ 'distinfo_pkg-1.0.0.dist-info': { 'direct_url.json': json.dumps({ "archive_info": { @@ -124,22 +124,22 @@ class DistInfoPkgEditable(DistInfoPkg): "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", }) }, - } + }) class DistInfoPkgWithDot(OnSysPath, SiteBuilder): - files: FilesSpec = { + files: FilesSpec = frozendict({ "pkg_dot-1.0.0.dist-info": { "METADATA": """ Name: pkg.dot Version: 1.0.0 """, }, - } + }) class DistInfoPkgWithDotLegacy(OnSysPath, SiteBuilder): - files: FilesSpec = { + files: FilesSpec = frozendict({ "pkg.dot-1.0.0.dist-info": { "METADATA": """ Name: pkg.dot @@ -152,7 +152,7 @@ class DistInfoPkgWithDotLegacy(OnSysPath, SiteBuilder): Version: 1.0.0 """, }, - } + }) class DistInfoPkgOffPath(SiteBuilder): @@ -160,7 +160,7 @@ class DistInfoPkgOffPath(SiteBuilder): class EggInfoPkg(OnSysPath, SiteBuilder): - files: FilesSpec = { + files: FilesSpec = frozendict({ "egginfo_pkg.egg-info": { "PKG-INFO": """ Name: egginfo-pkg @@ -192,11 +192,11 @@ class EggInfoPkg(OnSysPath, SiteBuilder): def main(): print("hello world") """, - } + }) class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteBuilder): - files: FilesSpec = { + files: FilesSpec = frozendict({ "egg_with_module_pkg.egg-info": { "PKG-INFO": "Name: egg_with_module-pkg", # SOURCES.txt is made from the source archive, and contains files @@ -223,11 +223,11 @@ class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteBuilder): def main(): print("hello world") """, - } + }) class EggInfoPkgPipInstalledExternalDataFiles(OnSysPath, SiteBuilder): - files: FilesSpec = { + files: FilesSpec = frozendict({ "egg_with_module_pkg.egg-info": { "PKG-INFO": "Name: egg_with_module-pkg", # SOURCES.txt is made from the source archive, and contains files @@ -257,11 +257,11 @@ class EggInfoPkgPipInstalledExternalDataFiles(OnSysPath, SiteBuilder): def main(): print("hello world") """, - } + }) class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteBuilder): - files: FilesSpec = { + files: FilesSpec = frozendict({ "egg_with_no_modules_pkg.egg-info": { "PKG-INFO": "Name: egg_with_no_modules-pkg", # SOURCES.txt is made from the source archive, and contains files @@ -283,11 +283,11 @@ class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteBuilder): # top_level.txt correctly reflects that no modules are installed "top_level.txt": b"\n", }, - } + }) class EggInfoPkgSourcesFallback(OnSysPath, SiteBuilder): - files: FilesSpec = { + files: FilesSpec = frozendict({ "sources_fallback_pkg.egg-info": { "PKG-INFO": "Name: sources_fallback-pkg", # SOURCES.txt is made from the source archive, and contains files @@ -305,11 +305,11 @@ class EggInfoPkgSourcesFallback(OnSysPath, SiteBuilder): def main(): print("hello world") """, - } + }) class EggInfoFile(OnSysPath, SiteBuilder): - files: FilesSpec = { + files: FilesSpec = frozendict({ "egginfo_file.egg-info": """ Metadata-Version: 1.0 Name: egginfo_file @@ -322,7 +322,7 @@ class EggInfoFile(OnSysPath, SiteBuilder): Description: UNKNOWN Platform: UNKNOWN """, - } + }) # dedent all text strings before writing diff --git a/Lib/test/test_importlib/metadata/test_api.py b/Lib/test/test_importlib/metadata/test_api.py index 5449f0484492fb..41ea57659467a0 100644 --- a/Lib/test/test_importlib/metadata/test_api.py +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -54,9 +54,8 @@ def test_name_normalization(self): def test_prefix_not_matched(self): prefixes = 'p', 'pkg', 'pkg.' for prefix in prefixes: - with self.subTest(prefix): - with self.assertRaises(PackageNotFoundError): - distribution(prefix) + with self.subTest(prefix), self.assertRaises(PackageNotFoundError): + distribution(prefix) def test_for_top_level(self): tests = [ @@ -77,9 +76,9 @@ def test_read_text(self): ] for pkg_name, expect_content in tests: with self.subTest(pkg_name): - top_level = [ + top_level = next( path for path in files(pkg_name) if path.name == 'top_level.txt' - ][0] + ) self.assertEqual(top_level.read_text(), expect_content) def test_entry_points(self): @@ -133,6 +132,36 @@ def test_entry_points_missing_name(self): def test_entry_points_missing_group(self): assert entry_points(group='missing') == () + def test_entry_points_invalid_value_in_other_group(self): + """ + An invalid value in one group should not prevent entry points + in other groups from being selected. + """ + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + alt_pkg = { + "invalid_entry_points-1.0.0.dist-info": { + "METADATA": """ + Name: invalid-entry-points + Version: 1.0.0 + """, + "entry_points.txt": """ + [selected] + valid = mod:main + + [unselected] + invalid = invalid-name + """, + }, + } + fixtures.build_files(alt_pkg, alt_site_dir) + + assert [ep.name for ep in entry_points(group='selected')] == ['valid'] + + # the invalid value is still rejected when the entry point is used + with self.assertRaises(ValueError): + entry_points(group='unselected')['invalid'].load() + def test_entry_points_allows_no_attributes(self): ep = entry_points().select(group='entries', name='main') with self.assertRaises(AttributeError): @@ -182,7 +211,7 @@ def _test_files(files): file.read_text() def test_file_hash_repr(self): - util = [p for p in files('distinfo-pkg') if p.name == 'mod.py'][0] + util = next(p for p in files('distinfo-pkg') if p.name == 'mod.py') self.assertRegex(repr(util.hash), '') def test_files_dist_info(self): diff --git a/Lib/test/test_importlib/metadata/test_main.py b/Lib/test/test_importlib/metadata/test_main.py index aae052160d9763..3a7afa31fe030d 100644 --- a/Lib/test/test_importlib/metadata/test_main.py +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -1,4 +1,6 @@ +import contextlib import importlib +import io import pickle import re import unittest @@ -15,6 +17,7 @@ MetadataNotFound, PackageNotFoundError, _unique, + diagnose, distributions, entry_points, metadata, @@ -59,7 +62,7 @@ def test_abc_enforced(self): dict(name=''), ) def test_invalid_inputs_to_from_name(self, name): - with self.assertRaises(Exception): + with self.assertRaises(ValueError): Distribution.from_name(name) @@ -136,7 +139,7 @@ def test_unique_distributions(self): class InvalidMetadataTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): @staticmethod - def make_pkg(name, files=dict(METADATA="VERSION: 1.0")): + def make_pkg(name, files=frozendict(METADATA="VERSION: 1.0")): """ Create metadata for a dist-info package with name and files. """ @@ -166,10 +169,115 @@ def test_missing_metadata(self): """ fixtures.build_files(self.make_pkg('foo-4.3', files={}), self.site_dir) with self.assertRaises(MetadataNotFound): - Distribution.from_name('foo').metadata + _ = Distribution.from_name('foo').metadata with self.assertRaises(MetadataNotFound): metadata('foo') + def test_diagnose_reports_invalid(self): + """ + The diagnostic should report invalid dists by location + rather than crashing on them. + + Ref python/importlib_metadata#548. + """ + fixtures.build_files(self.make_pkg('nameless-1.0'), self.site_dir) + fixtures.build_files(self.make_pkg('empty-1.0', files={}), self.site_dir) + out = io.StringIO() + with contextlib.redirect_stdout(out): + diagnose.inspect(str(self.site_dir)) + report = out.getvalue() + assert 'empty-1.0.dist-info (no metadata)' in report + assert 'nameless-1.0.dist-info (metadata without a name)' in report + + def test_missing_required_metadata(self): + """ + A missing required metadata field should raise a ``KeyError`` + naming the field and locating the distribution. + + Ref python/importlib_metadata#508. + """ + fixtures.build_files(self.make_pkg('nameless-4.4'), self.site_dir) + fixtures.build_files( + self.make_pkg('unversioned-4.4', files=dict(METADATA='Name: unversioned')), + self.site_dir, + ) + nameless = Distribution.at(self.site_dir / 'nameless-4.4.dist-info') + with self.assertRaisesRegex(KeyError, 'Name.*nameless-4.4.dist-info'): + nameless.name + unversioned = Distribution.at(self.site_dir / 'unversioned-4.4.dist-info') + with self.assertRaisesRegex(KeyError, 'Version.*unversioned-4.4.dist-info'): + unversioned.version + + +class MultipleVersionsTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + """ + An unclean upgrade can leave more than one metadata directory for a + distribution in one path. Prefer the greater one. + + Ref python/cpython#100666. + """ + + # Deliberately not in version order; the preference follows the + # lexicographic ordering of the directory names, which only + # approximates the ordering of the versions they name. + versions = '1.12.1', '1.13.1', '1.9.0', '2.0.0' + expected = '2.0.0', '1.9.0', '1.13.1', '1.12.1' + + @staticmethod + def make_pkg(name, version, files=None): + """ + Create metadata for a dist-info package at the indicated version. + """ + return { + f'{name}-{version}.dist-info': ( + {'METADATA': f'Name: {name}\nVersion: {version}\n'} + if files is None + else files + ), + } + + def build_versions(self, *versions): + for ver in versions: + fixtures.build_files(self.make_pkg('foo', ver), self.site_dir) + + def test_greater_version_preferred(self): + """ + Retrieving by name resolves to the greater version. + """ + self.build_versions(*self.versions) + assert version('foo') == self.expected[0] + + def test_discovery_order(self): + """ + Preferring one version doesn't hide the others from discovery, + which reports them greatest first. + """ + self.build_versions(*self.versions) + found = tuple(dist.version for dist in Distribution.discover(name='foo')) + assert found == self.expected + + def test_greater_version_preferred_in_eggs(self): + """ + Legacy egg-info metadata gets the same preference. + """ + fixtures.build_files( + { + f'foo-{ver}.egg-info': {'PKG-INFO': f'Name: foo\nVersion: {ver}\n'} + for ver in self.versions + }, + self.site_dir, + ) + assert version('foo') == self.expected[0] + + def test_validity_preferred_to_version(self): + """ + A valid dist is still preferred to an invalid one of a greater + version, the condition that prompted the report. + """ + self.build_versions('1.12.1') + fixtures.build_files(self.make_pkg('foo', '2.0.0', files={}), self.site_dir) + assert version('foo') == '1.12.1' + class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): @staticmethod @@ -261,9 +369,8 @@ def test_egg_info(self): def test_egg(self): egg = self.site_dir.joinpath('foo-3.6.egg') egg.mkdir() - with self.add_sys_path(egg): - with self.assertRaises(PackageNotFoundError): - version('foo') + with self.add_sys_path(egg), self.assertRaises(PackageNotFoundError): + version('foo') class MissingSysPath(fixtures.OnSysPath, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2026-10-22-22-51-40.gh-issue-160386.wYyboz.rst b/Misc/NEWS.d/next/Library/2026-10-22-22-51-40.gh-issue-160386.wYyboz.rst new file mode 100644 index 00000000000000..51da0478ee6455 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-22-51-40.gh-issue-160386.wYyboz.rst @@ -0,0 +1,11 @@ +Applied changes to ``importlib.metadata`` from `importlib_metadata 9.1 +`_: +distribution discovery now prefers the greater of several metadata +directories supplied by one path, so stale metadata left by an unclean +upgrade no longer shadows the version in use; a missing required metadata +field now raises a ``KeyError`` naming both the field and the distribution; +``EntryPoint`` value validation is deferred until the value is used, so one +malformed entry point no longer prevents others from being selected; +``importlib.metadata.diagnose`` no longer crashes on a metadata directory +with no metadata file or no declared name; and ``SimplePath.joinpath`` and +``SimplePath.__truediv__`` take ``other`` positional-only.