From ccae248a91bfc505b48e302db9829435d2b06d2a Mon Sep 17 00:00:00 2001 From: TenSt Date: Thu, 27 Aug 2026 17:06:54 +0200 Subject: [PATCH 1/3] feat: add HTTP catalog for Python packages and repository metrics Clients can list distinct packages and repository counts over the REST API instead of querying the database. The content list also supports collapsing rebuilds and returns base_version. Closes #1358. Assisted-By: Cursor --- .gitignore | 4 + CHANGES/1358.feature | 1 + CLAUDE.md | 4 + docs/index.md | 4 +- docs/user/guides/_SUMMARY.md | 1 + docs/user/guides/catalog.md | 116 +++++ pulp_python/app/catalog.py | 215 +++++++++ ...thonpackagecontent_name_normalized_trgm.py | 23 + pulp_python/app/models.py | 8 + pulp_python/app/pypi/views.py | 2 +- pulp_python/app/serializers.py | 81 ++++ pulp_python/app/tasks/publish.py | 7 +- pulp_python/app/utils.py | 1 - pulp_python/app/versions.py | 105 +++++ pulp_python/app/viewsets.py | 245 +++++++++- .../tests/functional/api/test_catalog.py | 446 ++++++++++++++++++ .../functional/api/test_crud_publications.py | 64 ++- pulp_python/tests/unit/test_catalog.py | 135 ++++++ 18 files changed, 1437 insertions(+), 25 deletions(-) create mode 100644 CHANGES/1358.feature create mode 100644 docs/user/guides/catalog.md create mode 100644 pulp_python/app/catalog.py create mode 100644 pulp_python/app/migrations/0025_pythonpackagecontent_name_normalized_trgm.py create mode 100644 pulp_python/app/versions.py create mode 100644 pulp_python/tests/functional/api/test_catalog.py create mode 100644 pulp_python/tests/unit/test_catalog.py diff --git a/.gitignore b/.gitignore index 29c94b654..444a89de7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python env/ +.venv/ build/ develop-eggs/ dist/ @@ -61,3 +62,6 @@ target/ # PyCharm .idea + +# VS Code / Cursor +.vscode/ diff --git a/CHANGES/1358.feature b/CHANGES/1358.feature new file mode 100644 index 000000000..d5407bec4 --- /dev/null +++ b/CHANGES/1358.feature @@ -0,0 +1 @@ +Added repository package catalog and metrics endpoints, plus ``collapse_builds`` and ``base_version`` on the Python package content API. The catalog includes ``last_updated``, ``ordering``, newest-first PEP 440 ``versions``/``latest_releases``, and ``name_normalized`` prefix/substring search (at least 3 characters). A trailing rebuild suffix is ``\.[a-zA-Z]+-[^.]+$`` (for example ``5.3.17.rhlw-00001-n0001`` groups with ``5.3.17``). ``latest_releases[].release`` is that suffix on the newest unit in the group, or empty when the stored version has none. Existing installs pick up access policy for the new actions on migrate unless the policy was customized. diff --git a/CLAUDE.md b/CLAUDE.md index 525ea0401..9d7809459 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,3 +58,7 @@ When patchback fails to cherry-pick a PR into an older branch, you need to manua ## Contributing When preparing to commit and create a PR you **must** follow our [PR checklist](https://pulpproject.org/pulpcore/docs/dev/guides/pull-request-walkthrough/) Important to note is the AI attribution requirement in our commit messages. Also, note that our changelog entries are markdown. + +## Catalog `strip_build_suffix` and CI unit tests + +CI runs unit tests with ``pytest -p no:pulpcore``. Collection must not import Django-backed modules (``pulp_python.app.utils``, ``catalog``, models, viewsets). Keep ``strip_build_suffix``, ``BUILD_SUFFIX_PATTERN``, ``version_sort_key``, ``normalize_package_index_ordering``, and ``normalize_name_normalized_search`` in ``pulp_python/app/versions.py``. The rebuild suffix is the last dot-segment matching POSIX ``\.[a-zA-Z]+-[^.]+$`` (letters, dash, rest of that segment; not hard-coded to ``rhlw``). Python ``re`` and SQL ``REGEXP_REPLACE`` share ``BUILD_SUFFIX_PATTERN``; ``catalog.py`` may import it. Catalog ``latest_releases`` keeps the newest ``pulp_created`` unit per logical version; ``release`` is ``rebuild_release`` of that stored ``version`` (empty when there is no suffix). Catalog ``name_normalized`` prefix/substring filters lowercase the input, use ``LIKE`` (not ``ILIKE``) against the trigram GIN index, and reject values shorter than 3 characters. Simple-index ``DISTINCT ON (name_normalized)`` must ``ORDER BY name_normalized, name`` so the displayed project name is deterministic when metadata names differ (``msg-parser`` vs ``msg_parser``). Without the secondary sort, ``ensure_simple`` can miss the ``msg-parser`` link even though both files were published. diff --git a/docs/index.md b/docs/index.md index 80c26e434..ff9bb16d1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,7 @@ The REST API documentation for `pulp_python` is available [here](site:pulp_pytho - [Create local mirrors of PyPI](site:pulp_python/docs/user/guides/sync/) that you have full control over - [Upload your own Python packages](site:pulp_python/docs/user/guides/upload/) +- [Browse the package catalog](site:pulp_python/docs/user/guides/catalog/) over the REST API - [Perform pip install](site:pulp_python/docs/user/guides/host/) from your Pulp Python repositories - Download packages on-demand to reduce disk usage - Every operation creates a restorable snapshot with Versioned Repositories @@ -34,5 +35,4 @@ Users may also find pulpcore’s conceptual docs useful. This documentation falls into two main categories: 1. `How-to Guides` shows the **major features** of the Python plugin, with links to reference docs. -2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each -minor feature, including all fields and options. +2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each minor feature, including all fields and options. diff --git a/docs/user/guides/_SUMMARY.md b/docs/user/guides/_SUMMARY.md index d9c01bdb2..36699ede5 100644 --- a/docs/user/guides/_SUMMARY.md +++ b/docs/user/guides/_SUMMARY.md @@ -1,6 +1,7 @@ * [Set up your own PyPI](pypi.md) * [Sync from Remote Repositories](sync.md) * [Upload and Manage Content](upload.md) +* [Browse the package catalog](catalog.md) * [Host Python Content](host.md) * [Vulnerability Report](vulnerability_report.md) * [Attestation Hosting](attestation.md) diff --git a/docs/user/guides/catalog.md b/docs/user/guides/catalog.md new file mode 100644 index 000000000..1f78d9d63 --- /dev/null +++ b/docs/user/guides/catalog.md @@ -0,0 +1,116 @@ +# Browse the package catalog + +Pulp CLI commands for these endpoints are generated from the OpenAPI spec in a separate package; until that is updated, use HTTP. + +The content list (`/pulp/api/v3/content/python/packages/`) returns **one row per distribution file** (wheel, sdist, …). For catalog UIs and automation that need **one row per package name**, plus repository metrics, use the repository package index. + +These endpoints default to the **latest complete repository version**. `{pulp_id}` is the repository UUID. Pass `repository_version` (HREF or PRN) to read a specific version of that repository. + +## List packages + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/?limit=10" +``` + +Pagination `count` is the number of **distinct packages** (`name_normalized`), not files. + +Each row includes both a simple version list and per-version metadata: + +```json +{ + "name": "shelf-reader", + "name_normalized": "shelf-reader", + "last_updated": "2026-08-10T10:45:08.099362Z", + "versions": ["0.1"], + "latest_releases": [ + { + "version": "0.1", + "release": "", + "created_at": "2026-08-10T10:45:08.099362Z" + } + ] +} +``` + +`set(versions)` is always the same as `set(latest_releases[].version)`. Both lists are newest-first using PEP 440 version order (`1.10` before `1.9` before `1.2`). There is one `latest_releases` entry per **logical version** (after stripping a trailing rebuild suffix `\.[a-zA-Z]+-[^.]+$`), not per wheel or sdist. A rebuild is the last dot-segment that is letters, a dash, then the rest of that segment (for example `5.3.17.rhlw-00001-n0001` → `5.3.17`). Public and predisclosure files of the same `name_normalized` and logical version collapse to that one row. + +`version` is that base. `release` is the stripped suffix without the leading dot (`rhlw-00001` or `rhlw-00001-n0001`) of the newest unit (`pulp_created`) in that group, otherwise empty. + +`created_at` is when that logical version entered the repository: `RepositoryContent.pulp_created` of the selected newest rebuild, falling back to the content unit's `pulp_created`. + +`last_updated` is when the **package** was last updated in this repository version: the latest `RepositoryContent.pulp_created` among **all** Python package units for that `name_normalized` (any rebuild), falling back to the content unit's `pulp_created`. A rebuild of an older version uploaded yesterday updates `last_updated` even if a newer version number already exists. + +### Ordering + +Default order is `name`. Pass `ordering` to change it: + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/" \ + ordering==name +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/" \ + ordering==-last_updated +``` + +Allowed fields: `name`, `name_normalized`, `last_updated`. Prefix with `-` for descending. `last_updated` uses `name` then `name_normalized` as a stable pagination tiebreaker. Unknown fields return 400. + +### Name search + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/" \ + name_normalized__istartswith==shelf +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/" \ + name_normalized__icontains==http +``` + +`name_normalized__istartswith` and `name_normalized__icontains` are case-insensitive: the value is lowercased and matched with `LIKE` against already-canonical `name_normalized`. Each requires **at least 3 characters** (shorter values return 400). `name__istartswith` is still `ILIKE` on the original package name and has no minimum length. Name search belongs on this index, not on the flat content list. + +## Repository metrics + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/metrics/" +``` + +```json +{ + "package_count": 3, + "version_count": 9, + "build_count": 9 +} +``` + +Counts use Python package content units in that repository version (not filtered by `packagetype`): + +| Field | Identity | +|-------|----------| +| `package_count` | distinct `name_normalized` | +| `version_count` | distinct `(name_normalized, base_version)` after rebuild-suffix strip | +| `build_count` | distinct `(name_normalized, full version)` | + +Until rebuild suffixes exist, `version_count` equals `build_count`. + +## List versions of a package + +Use the existing content API. Pass `packagetype=sdist` for one representative file per PEP version (retry with `packagetype=bdist_wheel` if a release is wheel-only). + +`collapse_builds=true` keeps one unit per logical version (`name_normalized` + `base_version`), the one with the latest `pulp_created`. Do not nest rebuilds on this list. Clients can drain Pulp `next` if the page is full. + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/content/python/packages/" \ + name==shelf-reader \ + packagetype==sdist \ + collapse_builds==true \ + repository_version=="${LATEST_VERSION_HREF}" +``` + +Every content row includes `base_version` (stripped version; equal to `version` when there is no suffix). + +## Get one version + +Omit `collapse_builds`. Filter with `name`, `version`, and `packagetype=sdist`: + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/content/python/packages/" \ + name==shelf-reader \ + version==0.1 \ + packagetype==sdist +``` diff --git a/pulp_python/app/catalog.py b/pulp_python/app/catalog.py new file mode 100644 index 000000000..975cf1048 --- /dev/null +++ b/pulp_python/app/catalog.py @@ -0,0 +1,215 @@ +"""Helpers for repository package catalog, metrics, and rebuild collapse.""" + +from collections import defaultdict + +from django.db.models import CharField, Func, Max, Min, Q, Value +from django.db.models.functions import Coalesce + +from pulp_python.app.models import PythonPackageContent +from pulp_python.app.versions import ( + BUILD_SUFFIX_PATTERN, + normalize_package_index_ordering, + rebuild_release, + version_sort_key, +) + + +def base_version_annotation(field_name="version"): + """SQL expression that strips a trailing rebuild suffix from ``version``. + + Uses ``versions.BUILD_SUFFIX_PATTERN`` (POSIX) so Python ``strip_build_suffix`` + and this ``REGEXP_REPLACE`` stay aligned. Implemented with ``REGEXP_REPLACE`` + so it does not depend on Django's ``RegexpReplace`` (not present in every + Django 4.2/5.2 packaging Pulp uses). + """ + return Func( + field_name, + Value(BUILD_SUFFIX_PATTERN), + Value(""), + function="REGEXP_REPLACE", + output_field=CharField(), + ) + + +def collapse_python_builds(queryset): + """Keep one content unit per ``(name_normalized, base_version)``. + + ``base_version`` is ``version`` with a trailing rebuild suffix stripped. + The unit with the latest ``pulp_created`` is kept. Callers that want one + row per logical version (not per wheel/sdist) should also filter + ``packagetype``. + """ + return ( + queryset.prefetch_related(None) + .annotate(_collapse_base_version=base_version_annotation()) + .order_by("name_normalized", "_collapse_base_version", "-pulp_created") + .distinct("name_normalized", "_collapse_base_version") + ) + + +def python_packages_in_version(repository_version): + """Python package content contained in ``repository_version``.""" + if repository_version is None: + return PythonPackageContent.objects.none() + return PythonPackageContent.objects.filter(pk__in=repository_version.content) + + +def apply_package_prefix_filters( + queryset, + name_normalized_prefix=None, + name_prefix=None, + name_normalized_contains=None, +): + """Apply case-insensitive name filters used by the package index.""" + if name_normalized_prefix: + queryset = queryset.filter(name_normalized__startswith=name_normalized_prefix) + if name_normalized_contains: + queryset = queryset.filter(name_normalized__contains=name_normalized_contains) + if name_prefix: + queryset = queryset.filter(name__istartswith=name_prefix) + return queryset + + +def membership_in_version_q(repository, repository_version): + """Q-object matching RepositoryContent rows present in ``repository_version``.""" + return Q( + version_memberships__repository=repository, + version_memberships__version_added__number__lte=repository_version.number, + ) & ( + Q(version_memberships__version_removed__isnull=True) + | Q(version_memberships__version_removed__number__gt=repository_version.number) + ) + + +def last_updated_annotation(repository, repository_version): + """Newest repository-membership time among all package units for a name. + + Uses ``RepositoryContent.pulp_created`` (any rebuild/build), falling back to + the content unit's ``pulp_created``. + """ + return Coalesce( + Max( + "version_memberships__pulp_created", + filter=membership_in_version_q(repository, repository_version), + ), + Max("pulp_created"), + ) + + +def distinct_package_names_qs(content_qs, repository, repository_version, ordering=None): + """One row per distinct ``name_normalized``, ordered for stable pagination.""" + if ordering is None: + ordering = normalize_package_index_ordering([]) + qs = content_qs.order_by().values("name_normalized").annotate(name=Max("name")) + if repository_version is None: + qs = qs.annotate(last_updated=Max("pulp_created")) + else: + qs = qs.annotate(last_updated=last_updated_annotation(repository, repository_version)) + return qs.order_by(*ordering) + + +def assemble_package_index(content_qs, name_rows, repository, repository_version): + """Build package-index dicts for ``name_rows``. + + ``versions`` are distinct logical versions, newest first (PEP 440). + ``latest_releases`` keeps the newest rebuild (latest ``pulp_created``) + per base version in the same order. ``created_at`` is that unit's + repository-membership time (``RepositoryContent.pulp_created``), falling + back to the content unit's ``pulp_created``. ``last_updated`` is the newest + membership among all units for the package (any rebuild), taken from + ``name_rows`` when annotated. + """ + if not name_rows or repository_version is None: + return [] + + names = [row["name_normalized"] for row in name_rows] + name_by_normalized = {row["name_normalized"]: row["name"] for row in name_rows} + + in_this_version = membership_in_version_q(repository, repository_version) + + newest_units = list( + content_qs.filter(name_normalized__in=names) + .prefetch_related(None) + .annotate(_base_version=base_version_annotation()) + .order_by("name_normalized", "_base_version", "-pulp_created") + .distinct("name_normalized", "_base_version") + ) + newest = [ + { + "pk": unit.pk, + "name_normalized": unit.name_normalized, + "version": unit.version, + "_base_version": unit._base_version, + "pulp_created": unit.pulp_created, + } + for unit in newest_units + ] + + memberships = {} + if newest: + memberships = dict( + PythonPackageContent.objects.filter(pk__in=[row["pk"] for row in newest]) + .annotate( + membership_created=Min( + "version_memberships__pulp_created", + filter=in_this_version, + ) + ) + .values_list("pk", "membership_created") + ) + + releases_by_name = defaultdict(list) + for rel in newest: + releases_by_name[rel["name_normalized"]].append(rel) + + result = [] + for row in name_rows: + normalized = row["name_normalized"] + rels = sorted( + releases_by_name.get(normalized, []), + key=lambda item: version_sort_key(item["_base_version"]), + reverse=True, + ) + versions = [item["_base_version"] for item in rels] + latest_releases = [ + { + "version": item["_base_version"], + "release": rebuild_release(item["version"]), + "created_at": memberships.get(item["pk"]) or item["pulp_created"], + } + for item in rels + ] + result.append( + { + "name": name_by_normalized[normalized], + "name_normalized": normalized, + "last_updated": row.get("last_updated"), + "versions": versions, + "latest_releases": latest_releases, + } + ) + return result + + +def repository_metrics(content_qs): + """Distinct package / logical-version / build counts for package content. + + Identity is always ``PythonPackageContent`` (not filtered by packagetype): + + * ``package_count``: distinct ``name_normalized`` + * ``version_count``: distinct ``(name_normalized, base_version)`` + * ``build_count``: distinct ``(name_normalized, version)`` + + Until rebuild suffixes exist, ``version_count`` equals ``build_count``. + """ + content_qs = content_qs.order_by() + return { + "package_count": content_qs.values("name_normalized").distinct().count(), + "version_count": ( + content_qs.annotate(_base_version=base_version_annotation()) + .values("name_normalized", "_base_version") + .distinct() + .count() + ), + "build_count": content_qs.values("name_normalized", "version").distinct().count(), + } diff --git a/pulp_python/app/migrations/0025_pythonpackagecontent_name_normalized_trgm.py b/pulp_python/app/migrations/0025_pythonpackagecontent_name_normalized_trgm.py new file mode 100644 index 000000000..746bc3d8f --- /dev/null +++ b/pulp_python/app/migrations/0025_pythonpackagecontent_name_normalized_trgm.py @@ -0,0 +1,23 @@ +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.operations import AddIndexConcurrently, TrigramExtension +from django.db import migrations + + +class Migration(migrations.Migration): + atomic = False # required for CONCURRENTLY + + dependencies = [ + ("python", "0024_pythonrepository_error_on_reject"), + ] + + operations = [ + TrigramExtension(), + AddIndexConcurrently( + model_name="pythonpackagecontent", + index=GinIndex( + fields=["name_normalized"], + name="python_name_normalized_trgm", + opclasses=["gin_trgm_ops"], + ), + ), + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index e8e6c26dc..c71114086 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -6,6 +6,7 @@ from aiohttp.web import json_response from django.conf import settings from django.contrib.postgres.fields import ArrayField +from django.contrib.postgres.indexes import GinIndex from django.core.exceptions import ObjectDoesNotExist from django.db import models from django_lifecycle import ( @@ -246,6 +247,13 @@ def __str__(self): class Meta: default_related_name = "%(app_label)s_%(model_name)s" unique_together = ("sha256", "_pulp_domain") + indexes = [ + GinIndex( + fields=["name_normalized"], + name="python_name_normalized_trgm", + opclasses=["gin_trgm_ops"], + ), + ] permissions = [ ("upload_python_packages", "Can upload Python packages using synchronous API."), ] diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index daa51b0d1..c8fa9976d 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -340,7 +340,7 @@ def list(self, request, path): if self.should_redirect(repo_version=repo_version): return redirect(urljoin(self.base_content_url, f"{path}/simple/")) names = ( - content.order_by("name_normalized") + content.order_by("name_normalized", "name") .values_list("name", flat=True) .distinct("name_normalized") .iterator() diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 038bb3584..09ac0b6da 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -34,6 +34,7 @@ get_project_metadata_from_file, parse_project_metadata, ) +from pulp_python.app.versions import BUILD_SUFFIX_PATTERN, strip_build_suffix log = logging.getLogger(__name__) PYPI_BASE_URL = urljoin(settings.PYPI_API_HOSTNAME, settings.PYPI_PATH_PREFIX) @@ -231,6 +232,17 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa help_text=_("The packages version number."), read_only=True, ) + base_version = serializers.SerializerMethodField( + help_text=_( + "The package version with a trailing rebuild suffix stripped " + "(matching %s). Equal to version when no suffix is present." + ) + % BUILD_SUFFIX_PATTERN, + ) + + def get_base_version(self, obj): + return strip_build_suffix(obj.version) + # Version 1.1 classifiers = serializers.JSONField( required=False, @@ -518,6 +530,7 @@ class Meta: "platform", "summary", "version", + "base_version", "classifiers", "download_url", "supported_platform", @@ -636,11 +649,79 @@ class Meta: "packagetype", "name", "version", + "base_version", "sha256", ) model = python_models.PythonPackageContent +class PythonPackageReleaseSerializer(serializers.Serializer): + """One logical version on the repository package index.""" + + version = serializers.CharField( + help_text=_("Logical version key (rebuild suffix stripped)."), + ) + release = serializers.CharField( + help_text=_( + "Rebuild/release qualifier within the version line " + "(e.g. rhlw-00001 or rhlw-00001-n0001). " + "Empty when the selected unit has no rebuild suffix." + ), + allow_blank=True, + ) + created_at = serializers.DateTimeField( + help_text=_( + "When this logical version entered the repository: RepositoryContent.pulp_created " + "of the newest rebuild, falling back to the content unit's pulp_created." + ), + ) + + +class PythonRepositoryPackageSerializer(serializers.Serializer): + """One distinct package in a repository version (not per wheel/sdist file).""" + + name = serializers.CharField(help_text=_("A representative project name for this package.")) + name_normalized = serializers.CharField( + help_text=_("PEP 503 normalized package name. Index rows are unique on this field."), + ) + last_updated = serializers.DateTimeField( + help_text=_( + "When this package was last updated in the repository: the latest " + "RepositoryContent.pulp_created among all Python package units for this " + "name (any rebuild), falling back to the content unit's pulp_created." + ), + allow_null=True, + ) + versions = serializers.ListField( + child=serializers.CharField(), + help_text=_( + "Distinct logical version keys after rebuild-suffix strip, newest first " + "(PEP 440). The set of values matches latest_releases[].version." + ), + ) + latest_releases = PythonPackageReleaseSerializer( + many=True, + help_text=_( + "Newest rebuild per logical version (latest pulp_created), newest version first. " + "set(versions) === set(latest_releases[].version)." + ), + ) + + +class PythonRepositoryMetricsSerializer(serializers.Serializer): + """Distinct package / version / build counts for a repository version.""" + + package_count = serializers.IntegerField( + help_text=_("Distinct name_normalized values among Python package content units."), + ) + version_count = serializers.IntegerField( + help_text=_("Distinct (name_normalized, base_version) pairs after rebuild-suffix strip."), + ) + build_count = serializers.IntegerField( + help_text=_("Distinct (name_normalized, full version) pairs among package content units."), + ) + + class PackageProvenanceSerializer(core_serializers.NoArtifactContentUploadSerializer): """ A Serializer for PackageProvenance. diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index 7c872159d..f3db2349c 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -56,11 +56,14 @@ def write_simple_api(publication): domain = get_domain() simple_dir = "simple/" os.mkdir(simple_dir) + # Secondary ORDER BY makes DISTINCT ON pick a stable display name when + # metadata names differ but canonicalize to the same name_normalized + # (e.g. msg-parser vs msg_parser). project_names = ( python_models.PythonPackageContent.objects.filter( pk__in=publication.repository_version.content, _pulp_domain=domain ) - .order_by("name_normalized") + .order_by("name_normalized", "name") .values_list("name", flat=True) .distinct("name_normalized") ) @@ -81,7 +84,7 @@ def write_simple_api(publication): packages = python_models.PythonPackageContent.objects.filter( pk__in=publication.repository_version.content, _pulp_domain=domain ) - releases = packages.order_by("name_normalized").values("name", "filename", "sha256") + releases = packages.order_by("name_normalized", "filename").values("name", "filename", "sha256") ind = 0 current_name = canonicalize_name(project_names[ind]) diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 9e08c77c6..091c6b69e 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -26,7 +26,6 @@ log = logging.getLogger(__name__) - PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" """TODO This serial constant is temporary until Python repositories implements serials""" PYPI_SERIAL_CONSTANT = 1000000000 diff --git a/pulp_python/app/versions.py b/pulp_python/app/versions.py new file mode 100644 index 000000000..6e2997380 --- /dev/null +++ b/pulp_python/app/versions.py @@ -0,0 +1,105 @@ +"""Catalog version helpers with no Django imports. + +CI unit tests run with ``-p no:pulpcore``, so this module must stay importable +before Django apps are loaded. +""" + +import re + +from packaging.version import InvalidVersion, Version + +# Last dot-segment is a rebuild if it is letters, dash, rest of that segment. +# POSIX string shared with SQL REGEXP_REPLACE. Not hard-coded to "rhlw". +BUILD_SUFFIX_PATTERN = r"\.[a-zA-Z]+-[^.]+$" +BUILD_SUFFIX_RE = re.compile(BUILD_SUFFIX_PATTERN) + +PACKAGE_INDEX_ORDERING_FIELDS = frozenset({"name", "name_normalized", "last_updated"}) +DEFAULT_PACKAGE_INDEX_ORDERING = ("name",) +NAME_NORMALIZED_SEARCH_MIN_LENGTH = 3 + + +def strip_build_suffix(version): + """Return ``version`` with a trailing rebuild suffix removed, else unchanged. + + A rebuild is the last dot-segment matching ``BUILD_SUFFIX_PATTERN``. + """ + if not version: + return version + return BUILD_SUFFIX_RE.sub("", version) + + +def rebuild_release(version): + """Return the rebuild qualifier without the leading dot, or an empty string.""" + if not version: + return "" + base = strip_build_suffix(version) + if version == base: + return "" + if version.startswith(base + "."): + return version[len(base) + 1 :] + return "" + + +def version_sort_key(version): + """PEP 440 sort key. Use with ``reverse=True`` for newest first. + + Invalid versions sort after all valid ones when ``reverse=True``. + """ + if not version: + return (-1, "") + try: + return (0, Version(version)) + except InvalidVersion: + return (-1, str(version)) + + +def normalize_package_index_ordering(raw_values): + """Turn ``ordering`` query values into ``order_by`` arguments. + + Default is ``name``. Unknown fields raise ``ValueError``. ``last_updated`` + keeps ``name`` then ``name_normalized`` as a stable pagination tiebreaker. + ``name_normalized`` is always appended when omitted so equal names paginate + stably (rows are unique on that column). + """ + fields = [] + for item in raw_values: + if not item: + continue + for part in str(item).split(","): + part = part.strip() + if part: + fields.append(part) + + if not fields: + fields = list(DEFAULT_PACKAGE_INDEX_ORDERING) + + normalized = [] + seen = set() + for field in fields: + descending = field.startswith("-") + name = field[1:] if descending else field + if name not in PACKAGE_INDEX_ORDERING_FIELDS: + raise ValueError(f"Unknown ordering field: '{name}'.") + if name in seen: + continue + seen.add(name) + normalized.append(f"-{name}" if descending else name) + + have = {term.lstrip("-") for term in normalized} + if "name" not in have and "name_normalized" not in have: + normalized.append("name") + if "name_normalized" not in have: + normalized.append("name_normalized") + return normalized + + +def normalize_name_normalized_search(value): + """Lowercase a ``name_normalized`` search string, or ``None`` if omitted.""" + if value is None: + return None + normalized = value.strip().lower() + if not normalized: + return None + if len(normalized) < NAME_NORMALIZED_SEARCH_MIN_LENGTH: + raise ValueError(f"Must be at least {NAME_NORMALIZED_SEARCH_MIN_LENGTH} characters.") + return normalized diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 6c73575f4..771a7d7eb 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -4,7 +4,13 @@ from django.db import transaction from django_filters import CharFilter from django_filters.rest_framework import filters as drf_filters -from drf_spectacular.utils import extend_schema, extend_schema_view +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import ( + OpenApiParameter, + extend_schema, + extend_schema_view, + inline_serializer, +) from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.utils import canonicalize_name from rest_framework import status @@ -16,7 +22,7 @@ RetrieveModelMixin, ) from rest_framework.response import Response -from rest_framework.serializers import ValidationError +from rest_framework.serializers import IntegerField, URLField, ValidationError from pulpcore.plugin import viewsets as core_viewsets from pulpcore.plugin.actions import ModifyRepositoryActionMixin @@ -32,6 +38,19 @@ from pulp_python.app import models as python_models from pulp_python.app import serializers as python_serializers from pulp_python.app import tasks +from pulp_python.app.catalog import ( + apply_package_prefix_filters, + assemble_package_index, + collapse_python_builds, + distinct_package_names_qs, + python_packages_in_version, + repository_metrics, +) +from pulp_python.app.versions import ( + BUILD_SUFFIX_PATTERN, + normalize_name_normalized_search, + normalize_package_index_ordering, +) class PythonRepositoryViewSet( @@ -64,7 +83,7 @@ class PythonRepositoryViewSet( ], }, { - "action": ["retrieve"], + "action": ["retrieve", "packages", "metrics"], "principal": "authenticated", "effect": "allow", "condition": "has_model_or_domain_or_obj_perms:python.view_pythonrepository", @@ -138,6 +157,22 @@ class PythonRepositoryViewSet( "python.pythonrepository_viewer": ["python.view_pythonrepository"], } + def filter_queryset(self, queryset): + """Do not apply the repository FilterSet to package-index query params.""" + if getattr(self, "action", None) in ("packages", "metrics"): + return queryset + return super().filter_queryset(queryset) + + def _requested_repository_version(self, repository): + """Resolve optional ``repository_version`` href/PRN, else latest complete version.""" + href = self.request.query_params.get("repository_version") + if not href: + return repository.latest_version() + repo_version = self.get_resource(href, RepositoryVersion) + if repo_version.repository_id != repository.pk: + raise ValidationError({"repository_version": "Must be a version of this repository."}) + return repo_version + @extend_schema( description="Trigger an asynchronous task to create a new repository version.", summary="Modify Repository Content", @@ -247,6 +282,179 @@ def sync(self, request, pk, **kwargs): ) return core_viewsets.OperationPostponedResponse(result, request) + @extend_schema( + summary="List packages", + description=( + "Return one row per distinct package name in a repository version " + "(latest complete version if repository_version is omitted). " + "Pagination count is the number of distinct packages, not files. " + "Each row includes last_updated (newest membership among any rebuild), " + "versions (logical version keys after rebuild-suffix strip, newest first), " + "and latest_releases (newest rebuild per logical version, same order). " + "set(versions) === set(latest_releases[].version)." + ), + parameters=[ + OpenApiParameter( + name="repository_version", + type=OpenApiTypes.URI, + location=OpenApiParameter.QUERY, + required=False, + description=( + "HREF or PRN of a version of this repository. " + "Defaults to the latest complete version." + ), + ), + OpenApiParameter( + name="name_normalized__istartswith", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description=( + "Case-insensitive prefix on the PEP 503 normalized package name." + "At least 3 characters required." + ), + ), + OpenApiParameter( + name="name_normalized__icontains", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description=( + "Case-insensitive substring on the PEP 503 normalized package name." + "At least 3 characters required." + ), + ), + OpenApiParameter( + name="name__istartswith", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Case-insensitive prefix on the original package name.", + ), + OpenApiParameter( + name="ordering", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + many=True, + description=( + "Order catalog rows. Allowed: name, name_normalized, last_updated. " + "Prefix with '-' for descending. Default is name." + ), + ), + OpenApiParameter( + name="limit", + type=OpenApiTypes.INT, + location=OpenApiParameter.QUERY, + required=False, + description="Number of results to return per page.", + ), + OpenApiParameter( + name="offset", + type=OpenApiTypes.INT, + location=OpenApiParameter.QUERY, + required=False, + description="The initial index from which to return the results.", + ), + ], + responses={ + 200: inline_serializer( + name="PaginatedPythonRepositoryPackageList", + fields={ + "count": IntegerField(), + "next": URLField(allow_null=True), + "previous": URLField(allow_null=True), + "results": python_serializers.PythonRepositoryPackageSerializer(many=True), + }, + ) + }, + ) + @action( + detail=True, + methods=["get"], + serializer_class=python_serializers.PythonRepositoryPackageSerializer, + ) + def packages(self, request, pk): + """List distinct packages in a repository version.""" + repository = self.get_object() + repo_version = self._requested_repository_version(repository) + content_qs = python_packages_in_version(repo_version) + search_errors = {} + try: + name_normalized_prefix = normalize_name_normalized_search( + request.query_params.get("name_normalized__istartswith") + ) + except ValueError as exc: + search_errors["name_normalized__istartswith"] = str(exc) + name_normalized_prefix = None + try: + name_normalized_contains = normalize_name_normalized_search( + request.query_params.get("name_normalized__icontains") + ) + except ValueError as exc: + search_errors["name_normalized__icontains"] = str(exc) + name_normalized_contains = None + if search_errors: + raise ValidationError(search_errors) + content_qs = apply_package_prefix_filters( + content_qs, + name_normalized_prefix=name_normalized_prefix, + name_prefix=request.query_params.get("name__istartswith"), + name_normalized_contains=name_normalized_contains, + ) + try: + ordering = normalize_package_index_ordering(request.query_params.getlist("ordering")) + except ValueError as exc: + raise ValidationError({"ordering": str(exc)}) from exc + names_qs = distinct_package_names_qs( + content_qs, repository, repo_version, ordering=ordering + ) + page = self.paginate_queryset(names_qs) + rows = assemble_package_index( + content_qs, + page if page is not None else list(names_qs), + repository, + repo_version, + ) + serializer = self.get_serializer(rows, many=True) + if page is not None: + return self.get_paginated_response(serializer.data) + return Response(serializer.data) + + @extend_schema( + summary="Repository metrics", + description=( + "Distinct counts for Python package content in a repository version " + "(latest complete version if repository_version is omitted). " + "package_count is distinct name_normalized. version_count is distinct " + "(name_normalized, base_version) after rebuild-suffix strip. build_count is " + "distinct (name_normalized, full version). Counts are not filtered by " + "packagetype. Until rebuild suffixes exist, version_count equals build_count." + ), + parameters=[ + OpenApiParameter( + name="repository_version", + type=OpenApiTypes.URI, + location=OpenApiParameter.QUERY, + required=False, + description=( + "HREF or PRN of a version of this repository. " + "Defaults to the latest complete version." + ), + ), + ], + responses={200: python_serializers.PythonRepositoryMetricsSerializer}, + ) + @action( + detail=True, + methods=["get"], + serializer_class=python_serializers.PythonRepositoryMetricsSerializer, + ) + def metrics(self, request, pk): + """Return package / version / build counts for a repository version.""" + repository = self.get_object() + repo_version = self._requested_repository_version(repository) + serializer = self.get_serializer( + repository_metrics(python_packages_in_version(repo_version)) + ) + return Response(serializer.data) + class PythonBlocklistEntryViewSet( core_viewsets.NamedModelViewSet, @@ -506,6 +714,25 @@ class PythonPackageContentFilter(core_viewsets.ContentFilter): field_name="version", help_text="Filter by PEP 440 version specifier (e.g., >=2.4,<3.0 or ~=1.26)", ) + collapse_builds = drf_filters.BooleanFilter( + method="filter_collapse_builds", + help_text=( + "When true, collapse rebuilds of the same logical version: strip a trailing " + f"suffix matching {BUILD_SUFFIX_PATTERN} from version, then keep one content unit " + "per (name_normalized, base_version) with the latest pulp_created. " + "Pass packagetype=sdist so wheel and sdist files are not collapsed together. " + "Default false." + ), + ) + + def filter_collapse_builds(self, qs, name, value): + """Documented on the FilterSet; applied in the viewset after ordering. + + DISTINCT ON requires ORDER BY to start with the distinct columns. The + viewset applies collapse after other filter backends so that ordering + cannot break it. + """ + return qs class Meta: model = python_models.PythonPackageContent @@ -539,6 +766,18 @@ class PythonPackageSingleArtifactContentUploadViewSet( minimal_serializer_class = python_serializers.MinimalPythonPackageContentSerializer filterset_class = PythonPackageContentFilter + def filter_queryset(self, queryset): + """Apply ``collapse_builds`` after other backends so DISTINCT ON stays valid.""" + queryset = super().filter_queryset(queryset) + if getattr(self, "action", "") != "list": + return queryset + raw = self.request.query_params.get("collapse_builds") + if raw is None or raw == "": + return queryset + if str(raw).lower() in ("true", "t", "yes", "y", "1"): + return collapse_python_builds(queryset) + return queryset + DEFAULT_ACCESS_POLICY = { "statements": [ { diff --git a/pulp_python/tests/functional/api/test_catalog.py b/pulp_python/tests/functional/api/test_catalog.py new file mode 100644 index 000000000..3975843e4 --- /dev/null +++ b/pulp_python/tests/functional/api/test_catalog.py @@ -0,0 +1,446 @@ +"""Catalog API tests. + +Generated client methods are unavailable until `oci-env generate-client` is rerun. +""" + +import io +import tarfile +import uuid +from datetime import datetime +from urllib.parse import urljoin + +import pytest +import requests + +from pulp_python.tests.functional.constants import PYTHON_SM_PROJECT_SPECIFIER + + +def _parse_dt(value): + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def _api_get(bindings_cfg, path, **params): + url = urljoin(bindings_cfg.host + "/", path.lstrip("/")) + response = requests.get(url, params=params, auth=(bindings_cfg.username, bindings_cfg.password)) + assert response.status_code == 200, response.text + return response.json() + + +def _content_packages_path(repo_href): + marker = "/api/v3/" + idx = repo_href.find(marker) + assert idx != -1, repo_href + return f"{repo_href[: idx + len(marker)]}content/python/packages/" + + +def _assert_package_row(pkg): + assert pkg["name"] + assert pkg["name_normalized"] + assert pkg["last_updated"] + assert pkg["versions"] == [rel["version"] for rel in pkg["latest_releases"]] + for rel in pkg["latest_releases"]: + assert "release" in rel + assert rel["created_at"] + + +def _write_sdist(directory, name, version): + """Write a minimal sdist whose PKG-INFO Name/Version pkginfo can read.""" + pkg_dir = f"{name}-{version}" + filename = f"{pkg_dir}.tar.gz" + path = directory / filename + pkg_info = f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n".encode() + with tarfile.open(path, "w:gz") as tar: + info = tarfile.TarInfo(name=f"{pkg_dir}/PKG-INFO") + info.size = len(pkg_info) + tar.addfile(info, io.BytesIO(pkg_info)) + return filename, str(path) + + +def _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, version): + filename, path = _write_sdist(tmp_path, name, version) + python_content_factory(relative_path=filename, file=path, repository=repo) + return python_bindings.RepositoriesPythonApi.read(repo.pulp_href) + + +@pytest.fixture +def sm_repo(python_repo_with_sync, python_remote_factory): + remote = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER) + return python_repo_with_sync(remote) + + +@pytest.mark.parallel +def test_package_list_grouping_and_pagination(bindings_cfg, sm_repo): + """Package index is one row per name, and count is distinct packages not files.""" + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=1) + assert data["count"] == 3 + assert len(data["results"]) == 1 + _assert_package_row(data["results"][0]) + + page2 = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=1, offset=1) + assert page2["count"] == 3 + assert page2["results"][0]["name_normalized"] != data["results"][0]["name_normalized"] + + all_rows = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=100)["results"] + assert {pkg["name_normalized"] for pkg in all_rows} == {"aiohttp", "celery", "django"} + django = next(pkg for pkg in all_rows if pkg["name_normalized"] == "django") + assert django["versions"] == ["1.10.4", "1.10.3", "1.10.2", "1.10.1"] + # Dual-field contract: one latest_releases entry per logical version, not per wheel/sdist. + assert len(django["latest_releases"]) == 4 + for rel in django["latest_releases"]: + assert rel["release"] == "" + + +@pytest.mark.parallel +def test_package_list_istartswith(bindings_cfg, sm_repo): + """Name prefix and substring search is case-insensitive on the package index.""" + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__istartswith="djan" + ) + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "django" + + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__istartswith="DJAN" + ) + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "django" + + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", name__istartswith="Cel") + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "celery" + + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__istartswith="shelf" + ) + assert data["count"] == 0 + + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__icontains="http" + ) + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "aiohttp" + + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__icontains="JAN") + assert data["count"] == 1 + assert data["results"][0]["name_normalized"] == "django" + + data = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", name_normalized__icontains="shelf" + ) + assert data["count"] == 0 + + +@pytest.mark.parallel +def test_package_list_name_normalized_search_min_length(bindings_cfg, sm_repo): + """name_normalized prefix/substring shorter than 3 characters is rejected.""" + url = urljoin(bindings_cfg.host + "/", f"{sm_repo.pulp_href}packages/".lstrip("/")) + auth = (bindings_cfg.username, bindings_cfg.password) + for param, value in ( + ("name_normalized__istartswith", "dj"), + ("name_normalized__icontains", "ht"), + ): + response = requests.get(url, params={param: value}, auth=auth) + assert response.status_code == 400, response.text + assert param in response.json() + + +@pytest.mark.parallel +def test_package_list_empty_repository(bindings_cfg, python_repo_factory): + repo = python_repo_factory() + data = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert data["count"] == 0 + assert data["results"] == [] + + +@pytest.mark.parallel +def test_repository_metrics(bindings_cfg, sm_repo, python_repo_factory): + """Metrics count distinct packages / logical versions / builds, not files.""" + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}metrics/") + assert data["package_count"] == 3 + # aiohttp 3 + celery 2 + Django 4; no rebuild suffixes in fixtures. + assert data["version_count"] == 9 + assert data["build_count"] == 9 + assert data["version_count"] == data["build_count"] + + empty = _api_get(bindings_cfg, f"{python_repo_factory().pulp_href}metrics/") + assert empty == {"package_count": 0, "version_count": 0, "build_count": 0} + + +@pytest.mark.parallel +def test_packages_and_metrics_repository_version(bindings_cfg, sm_repo, python_repo_factory): + """repository_version selects a snapshot; omitted uses the latest complete version.""" + latest_href = sm_repo.latest_version_href + v0_href = f"{sm_repo.pulp_href}versions/0/" + + default_pkgs = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/") + explicit_pkgs = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=latest_href + ) + assert default_pkgs["count"] == explicit_pkgs["count"] == 3 + + v0_pkgs = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=v0_href) + assert v0_pkgs["count"] == 0 + assert v0_pkgs["results"] == [] + + default_metrics = _api_get(bindings_cfg, f"{sm_repo.pulp_href}metrics/") + explicit_metrics = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}metrics/", repository_version=latest_href + ) + assert default_metrics == explicit_metrics + v0_metrics = _api_get(bindings_cfg, f"{sm_repo.pulp_href}metrics/", repository_version=v0_href) + assert v0_metrics == {"package_count": 0, "version_count": 0, "build_count": 0} + + other = python_repo_factory() + url = urljoin(bindings_cfg.host + "/", f"{sm_repo.pulp_href}packages/".lstrip("/")) + response = requests.get( + url, + params={"repository_version": other.latest_version_href}, + auth=(bindings_cfg.username, bindings_cfg.password), + ) + assert response.status_code == 400, response.text + + +@pytest.mark.parallel +def test_collapse_builds_and_base_version(bindings_cfg, sm_repo): + """collapse_builds keeps one unit per logical version; base_version is always present.""" + path = _content_packages_path(sm_repo.pulp_href) + repo_version = sm_repo.latest_version_href + + expanded = _api_get( + bindings_cfg, + path, + name="Django", + repository_version=repo_version, + collapse_builds="false", + limit=100, + ) + collapsed = _api_get( + bindings_cfg, + path, + name="Django", + repository_version=repo_version, + collapse_builds="true", + limit=100, + ) + # Wheel + sdist per Django version collapse when packagetype is omitted. + assert expanded["count"] == 8 + assert collapsed["count"] == 4 + assert {item["base_version"] for item in collapsed["results"]} == { + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + } + for item in expanded["results"] + collapsed["results"]: + assert item["base_version"] == item["version"] + + sdist_false = _api_get( + bindings_cfg, + path, + name="Django", + packagetype="sdist", + repository_version=repo_version, + collapse_builds="false", + limit=100, + ) + sdist_true = _api_get( + bindings_cfg, + path, + name="Django", + packagetype="sdist", + repository_version=repo_version, + collapse_builds="true", + limit=100, + ) + assert sdist_false["count"] == 4 + assert sdist_true["count"] == 4 + assert {item["version"] for item in sdist_true["results"]} == { + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + } + + +@pytest.mark.parallel +def test_package_get_base_version_without_collapse(bindings_cfg, python_repo_with_sync): + """PackageGet uses the content list without collapse_builds; base_version is still present.""" + repo = python_repo_with_sync() + path = _content_packages_path(repo.pulp_href) + data = _api_get( + bindings_cfg, + path, + name="shelf-reader", + version="0.1", + packagetype="sdist", + ) + assert data["count"] == 1 + item = data["results"][0] + assert item["version"] == "0.1" + assert item["base_version"] == "0.1" + assert "collapse_builds" not in item + + pkgs = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert pkgs["count"] == 1 + row = pkgs["results"][0] + _assert_package_row(row) + assert row["name_normalized"] == "shelf-reader" + assert row["versions"] == ["0.1"] + assert len(row["latest_releases"]) == 1 + assert row["latest_releases"][0]["release"] == "" + + +@pytest.mark.parallel +def test_package_list_ordering_name(bindings_cfg, sm_repo): + """Default order is name; -name reverses it.""" + default = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/")["results"] + names = [pkg["name"] for pkg in default] + assert len(names) == 3 + explicit = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", ordering="name")["results"] + assert [pkg["name"] for pkg in explicit] == names + + reversed_rows = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", ordering="-name")[ + "results" + ] + assert [pkg["name"] for pkg in reversed_rows] == list(reversed(names)) + + +@pytest.mark.parallel +def test_package_list_version_order( + bindings_cfg, python_bindings, python_content_factory, python_repo_factory, tmp_path +): + """versions and latest_releases are newest-first by PEP 440, not lexicographically.""" + repo = python_repo_factory() + name = f"ordered-{uuid.uuid4().hex[:8]}" + for version in ("1.10", "1.9", "1.2"): + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, version) + data = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert data["count"] == 1 + pkg = data["results"][0] + _assert_package_row(pkg) + assert pkg["versions"] == ["1.10", "1.9", "1.2"] + + +@pytest.mark.parallel +def test_package_list_ordering_last_updated( + bindings_cfg, python_bindings, python_content_factory, python_repo_factory, tmp_path +): + """last_updated is newest membership of any rebuild and is a sort key.""" + repo = python_repo_factory() + suffix = uuid.uuid4().hex[:8] + later_name = f"zzz-later-{suffix}" + earlier_name = f"aaa-earlier-{suffix}" + + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, later_name, "2.0.0") + first = _api_get(bindings_cfg, f"{repo.pulp_href}packages/")["results"][0] + first_updated = _parse_dt(first["last_updated"]) + assert first["name_normalized"] == later_name + assert first["last_updated"] == first["latest_releases"][0]["created_at"] + + repo = _add_sdist( + python_content_factory, python_bindings, tmp_path, repo, earlier_name, "1.0.0" + ) + rows = _api_get(bindings_cfg, f"{repo.pulp_href}packages/")["results"] + by_name = {pkg["name_normalized"]: pkg for pkg in rows} + older = by_name[later_name] + newer = by_name[earlier_name] + assert _parse_dt(older["last_updated"]) == first_updated + assert _parse_dt(newer["last_updated"]) > first_updated + + default_names = [pkg["name_normalized"] for pkg in rows] + assert default_names == [earlier_name, later_name] + + by_updated = _api_get( + bindings_cfg, f"{repo.pulp_href}packages/", ordering="-last_updated", limit=100 + )["results"] + assert [pkg["name_normalized"] for pkg in by_updated] == [earlier_name, later_name] + + repo = _add_sdist( + python_content_factory, + python_bindings, + tmp_path, + repo, + later_name, + "1.0.0.rhlw-00003", + ) + after_rebuild = _api_get( + bindings_cfg, f"{repo.pulp_href}packages/", ordering="-last_updated", limit=100 + )["results"] + assert [pkg["name_normalized"] for pkg in after_rebuild] == [later_name, earlier_name] + zzz = after_rebuild[0] + assert _parse_dt(zzz["last_updated"]) > _parse_dt(newer["last_updated"]) + assert set(zzz["versions"]) == {"2.0.0", "1.0.0"} + assert zzz["versions"][0] == "2.0.0" + rebuild_rel = next(rel for rel in zzz["latest_releases"] if rel["version"] == "1.0.0") + assert rebuild_rel["release"] == "rhlw-00003" + assert zzz["last_updated"] == rebuild_rel["created_at"] + public_rel = next(rel for rel in zzz["latest_releases"] if rel["version"] == "2.0.0") + assert public_rel["release"] == "" + + +@pytest.mark.parallel +def test_public_and_predisclosure_collapse_to_logical_version( + bindings_cfg, python_bindings, python_content_factory, python_repo_factory, tmp_path +): + """Public and predisclosure rebuilds of the same name share one logical version.""" + repo = python_repo_factory() + name = f"rebuild-{uuid.uuid4().hex[:8]}" + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, "5.3.17") + repo = _add_sdist( + python_content_factory, + python_bindings, + tmp_path, + repo, + name, + "5.3.17.rhlw-00001-n0001", + ) + + pkgs = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert pkgs["count"] == 1 + pkg = pkgs["results"][0] + _assert_package_row(pkg) + assert pkg["versions"] == ["5.3.17"] + assert len(pkg["latest_releases"]) == 1 + assert pkg["latest_releases"][0]["version"] == "5.3.17" + assert pkg["latest_releases"][0]["release"] == "rhlw-00001-n0001" + + metrics = _api_get(bindings_cfg, f"{repo.pulp_href}metrics/") + assert metrics == {"package_count": 1, "version_count": 1, "build_count": 2} + + path = _content_packages_path(repo.pulp_href) + repo_version = repo.latest_version_href + expanded = _api_get( + bindings_cfg, + path, + name=name, + packagetype="sdist", + repository_version=repo_version, + collapse_builds="false", + limit=100, + ) + collapsed = _api_get( + bindings_cfg, + path, + name=name, + packagetype="sdist", + repository_version=repo_version, + collapse_builds="true", + limit=100, + ) + assert expanded["count"] == 2 + assert {item["base_version"] for item in expanded["results"]} == {"5.3.17"} + assert collapsed["count"] == 1 + kept = collapsed["results"][0] + assert kept["base_version"] == "5.3.17" + assert kept["version"] == "5.3.17.rhlw-00001-n0001" + + +@pytest.mark.parallel +def test_package_list_ordering_invalid(bindings_cfg, sm_repo): + url = urljoin(bindings_cfg.host + "/", f"{sm_repo.pulp_href}packages/".lstrip("/")) + response = requests.get( + url, + params={"ordering": "group_id"}, + auth=(bindings_cfg.username, bindings_cfg.password), + ) + assert response.status_code == 400, response.text diff --git a/pulp_python/tests/functional/api/test_crud_publications.py b/pulp_python/tests/functional/api/test_crud_publications.py index e7941ac18..fb65cbd15 100644 --- a/pulp_python/tests/functional/api/test_crud_publications.py +++ b/pulp_python/tests/functional/api/test_crud_publications.py @@ -1,8 +1,10 @@ +import io import random +import tarfile +import zipfile from urllib.parse import urljoin import pytest -from pypi_simple import PyPISimple from pulp_python.tests.functional.constants import ( PYTHON_EGG_FILENAME, @@ -113,27 +115,57 @@ def test_new_content_is_published(python_publication_workflow, python_distributi assert proper is True, msgs +def _write_sdist_with_name(directory, name, version): + """Write a minimal sdist whose PKG-INFO Name differs from PEP 503 canonical form.""" + pkg_dir = f"{name}-{version}" + filename = f"{pkg_dir}.tar.gz" + path = directory / filename + pkg_info = f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n".encode() + with tarfile.open(path, "w:gz") as tar: + info = tarfile.TarInfo(name=f"{pkg_dir}/PKG-INFO") + info.size = len(pkg_info) + tar.addfile(info, io.BytesIO(pkg_info)) + return filename, str(path) + + +def _write_wheel_with_name(directory, metadata_name, version, filename): + """Write a minimal wheel whose METADATA Name can differ from the sdist Name.""" + dist_info = f"{filename.split('-')[0]}-{version}.dist-info" + metadata = f"Metadata-Version: 2.1\nName: {metadata_name}\nVersion: {version}\n" + wheel = ( + "Wheel-Version: 1.0\nGenerator: pulp-python-test\nRoot-Is-Purelib: true\n" + "Tag: py2.py3-none-any\n" + ) + path = directory / filename + with zipfile.ZipFile(path, "w") as zf: + zf.writestr(f"{dist_info}/METADATA", metadata) + zf.writestr(f"{dist_info}/WHEEL", wheel) + return filename, str(path) + + @pytest.mark.parallel def test_non_matching_canonicalized_name( - python_repo, python_content_factory, python_publication_factory, python_distribution_factory + python_repo, + python_content_factory, + python_publication_factory, + python_distribution_factory, + tmp_path, ): """Ensures a package with dists that have non-matching canonicalized names is published.""" - packages = [] - filenames = ["msg_parser-1.0.0-py2.py3-none-any.whl", "msg_parser-1.0.0.tar.gz"] - with PyPISimple() as client: - page = client.get_project_page("msg-parser") - for pkg in page.packages: - if pkg.filename in filenames: - c = python_content_factory(pkg.filename, url=pkg.url, repository=python_repo) - if c.filename.endswith(".tar.gz"): - # The metadata name in the SDist is not the same as the Wheel's name - assert c.name == "msg_parser" - else: - assert c.name == "msg-parser" - packages.append(c) + version = "1.0.0" + wheel_filename = "msg_parser-1.0.0-py2.py3-none-any.whl" + sdist_filename, sdist_path = _write_sdist_with_name(tmp_path, "msg_parser", version) + _, wheel_path = _write_wheel_with_name(tmp_path, "msg-parser", version, wheel_filename) + + wheel = python_content_factory(wheel_filename, file=wheel_path, repository=python_repo) + sdist = python_content_factory(sdist_filename, file=sdist_path, repository=python_repo) + # The metadata name in the SDist is not the same as the Wheel's name + assert wheel.name == "msg-parser" + assert sdist.name == "msg_parser" + pub = python_publication_factory(repository=python_repo) distro = python_distribution_factory(publication=pub) url = urljoin(distro.base_url, "simple/") - proper, msgs = ensure_simple(url, {"msg-parser": filenames}) + proper, msgs = ensure_simple(url, {"msg-parser": [wheel_filename, sdist_filename]}) assert proper is True, msgs diff --git a/pulp_python/tests/unit/test_catalog.py b/pulp_python/tests/unit/test_catalog.py new file mode 100644 index 000000000..000f8c096 --- /dev/null +++ b/pulp_python/tests/unit/test_catalog.py @@ -0,0 +1,135 @@ +"""Unit tests for catalog version helpers. + +CI runs these with ``pytest -p no:pulpcore``. Import ``versions``, not ``utils`` +or ``catalog``: those pull in Django models and fail collection with +AppRegistryNotReady. +""" + +import pytest + +from pulp_python.app.versions import ( + normalize_name_normalized_search, + normalize_package_index_ordering, + rebuild_release, + strip_build_suffix, + version_sort_key, +) + + +@pytest.mark.parametrize( + "version,expected", + [ + ("0.1", "0.1"), + ("5.3.17", "5.3.17"), + ("5.3.18", "5.3.18"), + ("5.3.180", "5.3.180"), + ("5.3.17.rhlw-00001", "5.3.17"), + ("5.3.18.rhlw-00003", "5.3.18"), + ("5.3.17.rhlw-00001-n0001", "5.3.17"), + ("5.3.18.lw-1", "5.3.18"), + ("1.0.0.abc-1", "1.0.0"), + ("1.0.0.ABC-99", "1.0.0"), + ("1.0.foo-bar", "1.0"), + ("1.0.rhlw-١", "1.0"), + ("4.3.0-redhat-1", "4.3.0-redhat-1"), + ("5.3.18-anything", "5.3.18-anything"), + ("5.3.18.anything", "5.3.18.anything"), + ("1.0.rhlw-00003.extra", "1.0.rhlw-00003.extra"), + ("1.0.rhlw-", "1.0.rhlw-"), + ("", ""), + (None, None), + ], +) +def test_strip_build_suffix(version, expected): + assert strip_build_suffix(version) == expected + + +@pytest.mark.parametrize( + "version,expected", + [ + ("5.3.18", ""), + ("5.3.17.rhlw-00001", "rhlw-00001"), + ("5.3.18.rhlw-00003", "rhlw-00003"), + ("5.3.17.rhlw-00001-n0001", "rhlw-00001-n0001"), + ("5.3.18.lw-1", "lw-1"), + ("0.1.rhlw-00003", "rhlw-00003"), + ("1.0.foo-bar", "foo-bar"), + ("1.0.rhlw-١", "rhlw-١"), + ("5.3.18.anything", ""), + ("4.3.0-redhat-1", ""), + ("5.3.18-anything", ""), + ("1.0.rhlw-", ""), + ("", ""), + (None, ""), + ], +) +def test_rebuild_release(version, expected): + assert rebuild_release(version) == expected + + +@pytest.mark.parametrize( + "versions,expected", + [ + # Lexical descending would put 1.9 before 1.2 before 1.10. + (["1.10", "1.9", "1.2"], ["1.10", "1.9", "1.2"]), + (["1.10.1", "1.10.4", "1.10.2", "1.10.3"], ["1.10.4", "1.10.3", "1.10.2", "1.10.1"]), + ([], []), + ([""], [""]), + ], +) +def test_version_sort_key_newest_first(versions, expected): + assert sorted(versions, key=version_sort_key, reverse=True) == expected + + +def test_version_sort_key_empty_and_invalid(): + assert version_sort_key("") == (-1, "") + assert version_sort_key(None) == (-1, "") + assert sorted(["1.0", "not-a-version"], key=version_sort_key, reverse=True) == [ + "1.0", + "not-a-version", + ] + + +@pytest.mark.parametrize( + "raw,expected", + [ + ([], ["name", "name_normalized"]), + ([""], ["name", "name_normalized"]), + (["name"], ["name", "name_normalized"]), + (["name,name_normalized"], ["name", "name_normalized"]), + (["-name"], ["-name", "name_normalized"]), + (["name_normalized"], ["name_normalized"]), + (["last_updated"], ["last_updated", "name", "name_normalized"]), + (["-last_updated"], ["-last_updated", "name", "name_normalized"]), + (["-last_updated", "name_normalized"], ["-last_updated", "name_normalized"]), + ], +) +def test_normalize_package_index_ordering(raw, expected): + assert normalize_package_index_ordering(raw) == expected + + +def test_normalize_package_index_ordering_rejects_unknown(): + with pytest.raises(ValueError, match="Unknown ordering field"): + normalize_package_index_ordering(["group_id"]) + + +@pytest.mark.parametrize( + "value,expected", + [ + (None, None), + ("", None), + (" ", None), + ("djan", "djan"), + ("DJAN", "djan"), + ("JAN", "jan"), + (" HTTP ", "http"), + ], +) +def test_normalize_name_normalized_search(value, expected): + assert normalize_name_normalized_search(value) == expected + + +@pytest.mark.parametrize("value", ["a", "ab", "DJ", " x "]) +def test_normalize_name_normalized_search_rejects_short(value): + with pytest.raises(ValueError, match="at least 3 characters"): + normalize_name_normalized_search(value) From 9338213ec5eac58f8304386fa82ae6a91796df2d Mon Sep 17 00:00:00 2001 From: TenSt Date: Thu, 24 Sep 2026 15:38:51 +0200 Subject: [PATCH 2/3] Address review: FilterSet catalog params, collapse in FilterSet, tighter docs. Assisted-By: Cursor --- CHANGES/1358.feature | 2 +- CLAUDE.md | 4 - docs/user/guides/catalog.md | 69 ++++++------- pulp_python/app/catalog.py | 21 +--- pulp_python/app/serializers.py | 6 +- pulp_python/app/versions.py | 1 - pulp_python/app/viewsets.py | 177 ++++++++++++++++----------------- 7 files changed, 127 insertions(+), 153 deletions(-) diff --git a/CHANGES/1358.feature b/CHANGES/1358.feature index d5407bec4..56e06b211 100644 --- a/CHANGES/1358.feature +++ b/CHANGES/1358.feature @@ -1 +1 @@ -Added repository package catalog and metrics endpoints, plus ``collapse_builds`` and ``base_version`` on the Python package content API. The catalog includes ``last_updated``, ``ordering``, newest-first PEP 440 ``versions``/``latest_releases``, and ``name_normalized`` prefix/substring search (at least 3 characters). A trailing rebuild suffix is ``\.[a-zA-Z]+-[^.]+$`` (for example ``5.3.17.rhlw-00001-n0001`` groups with ``5.3.17``). ``latest_releases[].release`` is that suffix on the newest unit in the group, or empty when the stored version has none. Existing installs pick up access policy for the new actions on migrate unless the policy was customized. +Added repository package catalog and metrics endpoints, plus `collapse_builds` and `base_version` on the Python package content API. diff --git a/CLAUDE.md b/CLAUDE.md index 9d7809459..525ea0401 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,7 +58,3 @@ When patchback fails to cherry-pick a PR into an older branch, you need to manua ## Contributing When preparing to commit and create a PR you **must** follow our [PR checklist](https://pulpproject.org/pulpcore/docs/dev/guides/pull-request-walkthrough/) Important to note is the AI attribution requirement in our commit messages. Also, note that our changelog entries are markdown. - -## Catalog `strip_build_suffix` and CI unit tests - -CI runs unit tests with ``pytest -p no:pulpcore``. Collection must not import Django-backed modules (``pulp_python.app.utils``, ``catalog``, models, viewsets). Keep ``strip_build_suffix``, ``BUILD_SUFFIX_PATTERN``, ``version_sort_key``, ``normalize_package_index_ordering``, and ``normalize_name_normalized_search`` in ``pulp_python/app/versions.py``. The rebuild suffix is the last dot-segment matching POSIX ``\.[a-zA-Z]+-[^.]+$`` (letters, dash, rest of that segment; not hard-coded to ``rhlw``). Python ``re`` and SQL ``REGEXP_REPLACE`` share ``BUILD_SUFFIX_PATTERN``; ``catalog.py`` may import it. Catalog ``latest_releases`` keeps the newest ``pulp_created`` unit per logical version; ``release`` is ``rebuild_release`` of that stored ``version`` (empty when there is no suffix). Catalog ``name_normalized`` prefix/substring filters lowercase the input, use ``LIKE`` (not ``ILIKE``) against the trigram GIN index, and reject values shorter than 3 characters. Simple-index ``DISTINCT ON (name_normalized)`` must ``ORDER BY name_normalized, name`` so the displayed project name is deterministic when metadata names differ (``msg-parser`` vs ``msg_parser``). Without the secondary sort, ``ensure_simple`` can miss the ``msg-parser`` link even though both files were published. diff --git a/docs/user/guides/catalog.md b/docs/user/guides/catalog.md index 1f78d9d63..c2db91505 100644 --- a/docs/user/guides/catalog.md +++ b/docs/user/guides/catalog.md @@ -1,10 +1,12 @@ # Browse the package catalog -Pulp CLI commands for these endpoints are generated from the OpenAPI spec in a separate package; until that is updated, use HTTP. +The content API lists **one row per file** (wheel, sdist, and so on). Use the repository +package catalog when you want **one row per package name**, for example in a UI that +shows Django once with its versions underneath. -The content list (`/pulp/api/v3/content/python/packages/`) returns **one row per distribution file** (wheel, sdist, …). For catalog UIs and automation that need **one row per package name**, plus repository metrics, use the repository package index. - -These endpoints default to the **latest complete repository version**. `{pulp_id}` is the repository UUID. Pass `repository_version` (HREF or PRN) to read a specific version of that repository. +Both catalog endpoints default to the repository's latest complete version. Pass +`repository_version` (HREF or PRN) to read an older snapshot. `{pulp_id}` is the +repository UUID. ## List packages @@ -12,9 +14,7 @@ These endpoints default to the **latest complete repository version**. `{pulp_id http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/?limit=10" ``` -Pagination `count` is the number of **distinct packages** (`name_normalized`), not files. - -Each row includes both a simple version list and per-version metadata: +`count` is the number of distinct packages, not files. Each row looks like: ```json { @@ -32,27 +32,24 @@ Each row includes both a simple version list and per-version metadata: } ``` -`set(versions)` is always the same as `set(latest_releases[].version)`. Both lists are newest-first using PEP 440 version order (`1.10` before `1.9` before `1.2`). There is one `latest_releases` entry per **logical version** (after stripping a trailing rebuild suffix `\.[a-zA-Z]+-[^.]+$`), not per wheel or sdist. A rebuild is the last dot-segment that is letters, a dash, then the rest of that segment (for example `5.3.17.rhlw-00001-n0001` → `5.3.17`). Public and predisclosure files of the same `name_normalized` and logical version collapse to that one row. - -`version` is that base. `release` is the stripped suffix without the leading dot (`rhlw-00001` or `rhlw-00001-n0001`) of the newest unit (`pulp_created`) in that group, otherwise empty. - -`created_at` is when that logical version entered the repository: `RepositoryContent.pulp_created` of the selected newest rebuild, falling back to the content unit's `pulp_created`. - -`last_updated` is when the **package** was last updated in this repository version: the latest `RepositoryContent.pulp_created` among **all** Python package units for that `name_normalized` (any rebuild), falling back to the content unit's `pulp_created`. A rebuild of an older version uploaded yesterday updates `last_updated` even if a newer version number already exists. +- `versions` is the list of version numbers, newest first (PEP 440, so `1.10` before `1.9`). +- `latest_releases` is the same versions with extra metadata. `release` is filled when + that version has a rebuild (for example `5.3.17.rhlw-00001` is shown as version + `5.3.17` with `release` `rhlw-00001`); otherwise it is empty. +- `created_at` is when that version was added to the repository. +- `last_updated` is when **any** file for the package last changed in this repository + version, including a rebuild of an older version. ### Ordering -Default order is `name`. Pass `ordering` to change it: +Default order is `name`. Allowed fields: `name`, `name_normalized`, `last_updated`. +Prefix with `-` for descending. ```bash -http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/" \ - ordering==name http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/" \ ordering==-last_updated ``` -Allowed fields: `name`, `name_normalized`, `last_updated`. Prefix with `-` for descending. `last_updated` uses `name` then `name_normalized` as a stable pagination tiebreaker. Unknown fields return 400. - ### Name search ```bash @@ -62,7 +59,9 @@ http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/package name_normalized__icontains==http ``` -`name_normalized__istartswith` and `name_normalized__icontains` are case-insensitive: the value is lowercased and matched with `LIKE` against already-canonical `name_normalized`. Each requires **at least 3 characters** (shorter values return 400). `name__istartswith` is still `ILIKE` on the original package name and has no minimum length. Name search belongs on this index, not on the flat content list. +`name_normalized__istartswith` and `name_normalized__icontains` match the PEP 503 +normalized name and require at least 3 characters. `name__istartswith` matches the +original project name and has no minimum length. ## Repository metrics @@ -78,21 +77,19 @@ http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/metrics } ``` -Counts use Python package content units in that repository version (not filtered by `packagetype`): - -| Field | Identity | -|-------|----------| -| `package_count` | distinct `name_normalized` | -| `version_count` | distinct `(name_normalized, base_version)` after rebuild-suffix strip | -| `build_count` | distinct `(name_normalized, full version)` | +| Field | Meaning | +|-------|---------| +| `package_count` | Distinct packages | +| `version_count` | Distinct packages × versions (rebuilds of the same version count as one) | +| `build_count` | Distinct packages × stored version strings (each rebuild counted) | -Until rebuild suffixes exist, `version_count` equals `build_count`. +Until a repository contains rebuilds, `version_count` equals `build_count`. -## List versions of a package +## List files for a package -Use the existing content API. Pass `packagetype=sdist` for one representative file per PEP version (retry with `packagetype=bdist_wheel` if a release is wheel-only). - -`collapse_builds=true` keeps one unit per logical version (`name_normalized` + `base_version`), the one with the latest `pulp_created`. Do not nest rebuilds on this list. Clients can drain Pulp `next` if the page is full. +Use the content API. `packagetype=sdist` returns one sdist per version (retry with +`packagetype=bdist_wheel` if a release is wheel-only). `collapse_builds=true` keeps +the newest rebuild per version so you do not have to page through every rebuild. ```bash http GET "${BASE_ADDR}/pulp/api/v3/content/python/packages/" \ @@ -102,11 +99,11 @@ http GET "${BASE_ADDR}/pulp/api/v3/content/python/packages/" \ repository_version=="${LATEST_VERSION_HREF}" ``` -Every content row includes `base_version` (stripped version; equal to `version` when there is no suffix). - -## Get one version +Each content row includes `base_version`: the version without a rebuild suffix +(equal to `version` when there is none). -Omit `collapse_builds`. Filter with `name`, `version`, and `packagetype=sdist`: +To fetch a single version, omit `collapse_builds` and filter by `name`, `version`, +and `packagetype`: ```bash http GET "${BASE_ADDR}/pulp/api/v3/content/python/packages/" \ diff --git a/pulp_python/app/catalog.py b/pulp_python/app/catalog.py index 975cf1048..b99931f12 100644 --- a/pulp_python/app/catalog.py +++ b/pulp_python/app/catalog.py @@ -39,11 +39,14 @@ def collapse_python_builds(queryset): row per logical version (not per wheel/sdist) should also filter ``packagetype``. """ + # DISTINCT ON cannot reuse pulpcore's list prefetches (cloned lookups / + # JOINs). Drop them, collapse, then prefetch artifacts for the reduced set. return ( queryset.prefetch_related(None) .annotate(_collapse_base_version=base_version_annotation()) .order_by("name_normalized", "_collapse_base_version", "-pulp_created") .distinct("name_normalized", "_collapse_base_version") + .prefetch_related("contentartifact_set") ) @@ -54,22 +57,6 @@ def python_packages_in_version(repository_version): return PythonPackageContent.objects.filter(pk__in=repository_version.content) -def apply_package_prefix_filters( - queryset, - name_normalized_prefix=None, - name_prefix=None, - name_normalized_contains=None, -): - """Apply case-insensitive name filters used by the package index.""" - if name_normalized_prefix: - queryset = queryset.filter(name_normalized__startswith=name_normalized_prefix) - if name_normalized_contains: - queryset = queryset.filter(name_normalized__contains=name_normalized_contains) - if name_prefix: - queryset = queryset.filter(name__istartswith=name_prefix) - return queryset - - def membership_in_version_q(repository, repository_version): """Q-object matching RepositoryContent rows present in ``repository_version``.""" return Q( @@ -129,7 +116,7 @@ def assemble_package_index(content_qs, name_rows, repository, repository_version newest_units = list( content_qs.filter(name_normalized__in=names) - .prefetch_related(None) + .prefetch_related(None) # DISTINCT ON; see collapse_python_builds .annotate(_base_version=base_version_annotation()) .order_by("name_normalized", "_base_version", "-pulp_created") .distinct("name_normalized", "_base_version") diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 09ac0b6da..b6ea999d3 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -695,15 +695,13 @@ class PythonRepositoryPackageSerializer(serializers.Serializer): versions = serializers.ListField( child=serializers.CharField(), help_text=_( - "Distinct logical version keys after rebuild-suffix strip, newest first " - "(PEP 440). The set of values matches latest_releases[].version." + "Distinct logical version keys after rebuild-suffix strip, newest first (PEP 440)." ), ) latest_releases = PythonPackageReleaseSerializer( many=True, help_text=_( - "Newest rebuild per logical version (latest pulp_created), newest version first. " - "set(versions) === set(latest_releases[].version)." + "Newest rebuild per logical version (latest pulp_created), newest version first." ), ) diff --git a/pulp_python/app/versions.py b/pulp_python/app/versions.py index 6e2997380..ef274ae6e 100644 --- a/pulp_python/app/versions.py +++ b/pulp_python/app/versions.py @@ -9,7 +9,6 @@ from packaging.version import InvalidVersion, Version # Last dot-segment is a rebuild if it is letters, dash, rest of that segment. -# POSIX string shared with SQL REGEXP_REPLACE. Not hard-coded to "rhlw". BUILD_SUFFIX_PATTERN = r"\.[a-zA-Z]+-[^.]+$" BUILD_SUFFIX_RE = re.compile(BUILD_SUFFIX_PATTERN) diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 771a7d7eb..2407d58c0 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -3,6 +3,7 @@ from bandersnatch.configuration import BandersnatchConfig from django.db import transaction from django_filters import CharFilter +from django_filters.rest_framework import FilterSet from django_filters.rest_framework import filters as drf_filters from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import ( @@ -39,7 +40,6 @@ from pulp_python.app import serializers as python_serializers from pulp_python.app import tasks from pulp_python.app.catalog import ( - apply_package_prefix_filters, assemble_package_index, collapse_python_builds, distinct_package_names_qs, @@ -53,6 +53,75 @@ ) +class CatalogOrderingFilter(drf_filters.OrderingFilter): + """Validate catalog ``ordering`` without applying it to the content queryset.""" + + def filter(self, qs, value): + return qs + + +class PythonRepositoryPackageFilter(FilterSet): + """Query parameters for ``GET .../repositories/python/python/{pk}/packages/``.""" + + repository_version = CharFilter( + method="filter_noop", + help_text=( + "HREF or PRN of a version of this repository. Defaults to the latest complete version." + ), + ) + name_normalized__istartswith = CharFilter( + method="filter_name_normalized_prefix", + help_text=( + "Case-insensitive prefix on the PEP 503 normalized package name. " + "At least 3 characters required." + ), + ) + name_normalized__icontains = CharFilter( + method="filter_name_normalized_contains", + help_text=( + "Case-insensitive substring on the PEP 503 normalized package name. " + "At least 3 characters required." + ), + ) + name__istartswith = CharFilter( + field_name="name", + lookup_expr="istartswith", + help_text="Case-insensitive prefix on the original package name.", + ) + ordering = CatalogOrderingFilter( + fields=("name", "name_normalized", "last_updated"), + help_text=( + "Order catalog rows. Allowed: name, name_normalized, last_updated. " + "Prefix with '-' for descending. Default is name." + ), + ) + + def filter_noop(self, qs, name, value): + return qs + + def filter_name_normalized_prefix(self, qs, name, value): + try: + value = normalize_name_normalized_search(value) + except ValueError as exc: + raise ValidationError({"name_normalized__istartswith": str(exc)}) from exc + if not value: + return qs + return qs.filter(name_normalized__startswith=value) + + def filter_name_normalized_contains(self, qs, name, value): + try: + value = normalize_name_normalized_search(value) + except ValueError as exc: + raise ValidationError({"name_normalized__icontains": str(exc)}) from exc + if not value: + return qs + return qs.filter(name_normalized__contains=value) + + class Meta: + model = python_models.PythonPackageContent + fields = [] + + class PythonRepositoryViewSet( core_viewsets.RepositoryViewSet, ModifyRepositoryActionMixin, core_viewsets.RolesMixin ): @@ -287,57 +356,10 @@ def sync(self, request, pk, **kwargs): description=( "Return one row per distinct package name in a repository version " "(latest complete version if repository_version is omitted). " - "Pagination count is the number of distinct packages, not files. " - "Each row includes last_updated (newest membership among any rebuild), " - "versions (logical version keys after rebuild-suffix strip, newest first), " - "and latest_releases (newest rebuild per logical version, same order). " - "set(versions) === set(latest_releases[].version)." + "Pagination count is the number of distinct packages, not files." ), parameters=[ - OpenApiParameter( - name="repository_version", - type=OpenApiTypes.URI, - location=OpenApiParameter.QUERY, - required=False, - description=( - "HREF or PRN of a version of this repository. " - "Defaults to the latest complete version." - ), - ), - OpenApiParameter( - name="name_normalized__istartswith", - type=OpenApiTypes.STR, - location=OpenApiParameter.QUERY, - description=( - "Case-insensitive prefix on the PEP 503 normalized package name." - "At least 3 characters required." - ), - ), - OpenApiParameter( - name="name_normalized__icontains", - type=OpenApiTypes.STR, - location=OpenApiParameter.QUERY, - description=( - "Case-insensitive substring on the PEP 503 normalized package name." - "At least 3 characters required." - ), - ), - OpenApiParameter( - name="name__istartswith", - type=OpenApiTypes.STR, - location=OpenApiParameter.QUERY, - description="Case-insensitive prefix on the original package name.", - ), - OpenApiParameter( - name="ordering", - type=OpenApiTypes.STR, - location=OpenApiParameter.QUERY, - many=True, - description=( - "Order catalog rows. Allowed: name, name_normalized, last_updated. " - "Prefix with '-' for descending. Default is name." - ), - ), + PythonRepositoryPackageFilter, OpenApiParameter( name="limit", type=OpenApiTypes.INT, @@ -375,29 +397,10 @@ def packages(self, request, pk): repository = self.get_object() repo_version = self._requested_repository_version(repository) content_qs = python_packages_in_version(repo_version) - search_errors = {} - try: - name_normalized_prefix = normalize_name_normalized_search( - request.query_params.get("name_normalized__istartswith") - ) - except ValueError as exc: - search_errors["name_normalized__istartswith"] = str(exc) - name_normalized_prefix = None - try: - name_normalized_contains = normalize_name_normalized_search( - request.query_params.get("name_normalized__icontains") - ) - except ValueError as exc: - search_errors["name_normalized__icontains"] = str(exc) - name_normalized_contains = None - if search_errors: - raise ValidationError(search_errors) - content_qs = apply_package_prefix_filters( - content_qs, - name_normalized_prefix=name_normalized_prefix, - name_prefix=request.query_params.get("name__istartswith"), - name_normalized_contains=name_normalized_contains, - ) + filterset = PythonRepositoryPackageFilter(data=request.query_params, queryset=content_qs) + if not filterset.is_valid(): + raise ValidationError(filterset.errors) + content_qs = filterset.qs try: ordering = normalize_package_index_ordering(request.query_params.getlist("ordering")) except ValueError as exc: @@ -726,13 +729,19 @@ class PythonPackageContentFilter(core_viewsets.ContentFilter): ) def filter_collapse_builds(self, qs, name, value): - """Documented on the FilterSet; applied in the viewset after ordering. + """No-op during the per-filter loop; applied in ``filter_queryset``.""" + return qs - DISTINCT ON requires ORDER BY to start with the distinct columns. The - viewset applies collapse after other filter backends so that ordering - cannot break it. + def filter_queryset(self, queryset): + """Apply ``collapse_builds`` after other filters, including ordering. + + DISTINCT ON requires ORDER BY to start with the distinct columns, so + collapse must run after ``StableOrderingFilter``. """ - return qs + queryset = super().filter_queryset(queryset) + if self.form.cleaned_data.get("collapse_builds"): + return collapse_python_builds(queryset) + return queryset class Meta: model = python_models.PythonPackageContent @@ -766,18 +775,6 @@ class PythonPackageSingleArtifactContentUploadViewSet( minimal_serializer_class = python_serializers.MinimalPythonPackageContentSerializer filterset_class = PythonPackageContentFilter - def filter_queryset(self, queryset): - """Apply ``collapse_builds`` after other backends so DISTINCT ON stays valid.""" - queryset = super().filter_queryset(queryset) - if getattr(self, "action", "") != "list": - return queryset - raw = self.request.query_params.get("collapse_builds") - if raw is None or raw == "": - return queryset - if str(raw).lower() in ("true", "t", "yes", "y", "1"): - return collapse_python_builds(queryset) - return queryset - DEFAULT_ACCESS_POLICY = { "statements": [ { From 6b7a2b3329c7ef183fb63712e7352bcd3f7f7d76 Mon Sep 17 00:00:00 2001 From: TenSt Date: Thu, 24 Sep 2026 17:58:16 +0200 Subject: [PATCH 3/3] Treat rebuilds as PEP 440 local versions (+test.N) instead of Maven-style .letters-dash suffixes. Python versions group on the public version (everything before +); release is the local identifier. Catalog still keeps the newest pulp_created unit per public version. Assisted-By: Cursor --- docs/user/guides/catalog.md | 8 ++-- pulp_python/app/catalog.py | 4 +- pulp_python/app/serializers.py | 12 +++--- pulp_python/app/versions.py | 15 ++++--- pulp_python/app/viewsets.py | 6 +-- .../tests/functional/api/test_catalog.py | 10 ++--- pulp_python/tests/unit/test_catalog.py | 39 ++++++++++--------- 7 files changed, 48 insertions(+), 46 deletions(-) diff --git a/docs/user/guides/catalog.md b/docs/user/guides/catalog.md index c2db91505..1d2a001ee 100644 --- a/docs/user/guides/catalog.md +++ b/docs/user/guides/catalog.md @@ -34,8 +34,8 @@ http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/package - `versions` is the list of version numbers, newest first (PEP 440, so `1.10` before `1.9`). - `latest_releases` is the same versions with extra metadata. `release` is filled when - that version has a rebuild (for example `5.3.17.rhlw-00001` is shown as version - `5.3.17` with `release` `rhlw-00001`); otherwise it is empty. + that version has a rebuild (for example `5.3.17+test.1` is shown as version + `5.3.17` with `release` `test.1`); otherwise it is empty. - `created_at` is when that version was added to the repository. - `last_updated` is when **any** file for the package last changed in this repository version, including a rebuild of an older version. @@ -99,8 +99,8 @@ http GET "${BASE_ADDR}/pulp/api/v3/content/python/packages/" \ repository_version=="${LATEST_VERSION_HREF}" ``` -Each content row includes `base_version`: the version without a rebuild suffix -(equal to `version` when there is none). +Each content row includes `base_version`: the version without a PEP 440 local +version (equal to `version` when there is none). To fetch a single version, omit `collapse_builds` and filter by `name`, `version`, and `packagetype`: diff --git a/pulp_python/app/catalog.py b/pulp_python/app/catalog.py index b99931f12..1567c3dbf 100644 --- a/pulp_python/app/catalog.py +++ b/pulp_python/app/catalog.py @@ -15,7 +15,7 @@ def base_version_annotation(field_name="version"): - """SQL expression that strips a trailing rebuild suffix from ``version``. + """SQL expression that strips a PEP 440 local version from ``version``. Uses ``versions.BUILD_SUFFIX_PATTERN`` (POSIX) so Python ``strip_build_suffix`` and this ``REGEXP_REPLACE`` stay aligned. Implemented with ``REGEXP_REPLACE`` @@ -34,7 +34,7 @@ def base_version_annotation(field_name="version"): def collapse_python_builds(queryset): """Keep one content unit per ``(name_normalized, base_version)``. - ``base_version`` is ``version`` with a trailing rebuild suffix stripped. + ``base_version`` is ``version`` with a PEP 440 local version stripped. The unit with the latest ``pulp_created`` is kept. Callers that want one row per logical version (not per wheel/sdist) should also filter ``packagetype``. diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index b6ea999d3..3cda647cb 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -234,8 +234,8 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa ) base_version = serializers.SerializerMethodField( help_text=_( - "The package version with a trailing rebuild suffix stripped " - "(matching %s). Equal to version when no suffix is present." + "The package version with a PEP 440 local version stripped " + "(matching %s). Equal to version when no local version is present." ) % BUILD_SUFFIX_PATTERN, ) @@ -659,13 +659,13 @@ class PythonPackageReleaseSerializer(serializers.Serializer): """One logical version on the repository package index.""" version = serializers.CharField( - help_text=_("Logical version key (rebuild suffix stripped)."), + help_text=_("Logical version key (PEP 440 local version stripped)."), ) release = serializers.CharField( help_text=_( - "Rebuild/release qualifier within the version line " - "(e.g. rhlw-00001 or rhlw-00001-n0001). " - "Empty when the selected unit has no rebuild suffix." + "PEP 440 local identifier within the version line " + "(e.g. test.1 or test.1.n1). " + "Empty when the selected unit has no local version." ), allow_blank=True, ) diff --git a/pulp_python/app/versions.py b/pulp_python/app/versions.py index ef274ae6e..c5feb8928 100644 --- a/pulp_python/app/versions.py +++ b/pulp_python/app/versions.py @@ -8,8 +8,8 @@ from packaging.version import InvalidVersion, Version -# Last dot-segment is a rebuild if it is letters, dash, rest of that segment. -BUILD_SUFFIX_PATTERN = r"\.[a-zA-Z]+-[^.]+$" +# PEP 440 local version: ``+`` through the end of the string (POSIX, shared with SQL). +BUILD_SUFFIX_PATTERN = r"\+.*$" BUILD_SUFFIX_RE = re.compile(BUILD_SUFFIX_PATTERN) PACKAGE_INDEX_ORDERING_FIELDS = frozenset({"name", "name_normalized", "last_updated"}) @@ -18,9 +18,9 @@ def strip_build_suffix(version): - """Return ``version`` with a trailing rebuild suffix removed, else unchanged. + """Return ``version`` with a PEP 440 local version removed, else unchanged. - A rebuild is the last dot-segment matching ``BUILD_SUFFIX_PATTERN``. + A rebuild is the local version (``+`` through the end of the string). """ if not version: return version @@ -28,15 +28,14 @@ def strip_build_suffix(version): def rebuild_release(version): - """Return the rebuild qualifier without the leading dot, or an empty string.""" + """Return the PEP 440 local identifier without the leading ``+``, or empty.""" if not version: return "" base = strip_build_suffix(version) if version == base: return "" - if version.startswith(base + "."): - return version[len(base) + 1 :] - return "" + # strip_build_suffix removes ``+local``; skip the ``+``. + return version[len(base) + 1 :] def version_sort_key(version): diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 2407d58c0..d762abe8f 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -720,9 +720,9 @@ class PythonPackageContentFilter(core_viewsets.ContentFilter): collapse_builds = drf_filters.BooleanFilter( method="filter_collapse_builds", help_text=( - "When true, collapse rebuilds of the same logical version: strip a trailing " - f"suffix matching {BUILD_SUFFIX_PATTERN} from version, then keep one content unit " - "per (name_normalized, base_version) with the latest pulp_created. " + "When true, collapse rebuilds of the same logical version: strip a PEP 440 " + f"local version matching {BUILD_SUFFIX_PATTERN} from version, then keep one " + "content unit per (name_normalized, base_version) with the latest pulp_created. " "Pass packagetype=sdist so wheel and sdist files are not collapsed together. " "Default false." ), diff --git a/pulp_python/tests/functional/api/test_catalog.py b/pulp_python/tests/functional/api/test_catalog.py index 3975843e4..bc7915f81 100644 --- a/pulp_python/tests/functional/api/test_catalog.py +++ b/pulp_python/tests/functional/api/test_catalog.py @@ -361,7 +361,7 @@ def test_package_list_ordering_last_updated( tmp_path, repo, later_name, - "1.0.0.rhlw-00003", + "1.0.0+test.3", ) after_rebuild = _api_get( bindings_cfg, f"{repo.pulp_href}packages/", ordering="-last_updated", limit=100 @@ -372,7 +372,7 @@ def test_package_list_ordering_last_updated( assert set(zzz["versions"]) == {"2.0.0", "1.0.0"} assert zzz["versions"][0] == "2.0.0" rebuild_rel = next(rel for rel in zzz["latest_releases"] if rel["version"] == "1.0.0") - assert rebuild_rel["release"] == "rhlw-00003" + assert rebuild_rel["release"] == "test.3" assert zzz["last_updated"] == rebuild_rel["created_at"] public_rel = next(rel for rel in zzz["latest_releases"] if rel["version"] == "2.0.0") assert public_rel["release"] == "" @@ -392,7 +392,7 @@ def test_public_and_predisclosure_collapse_to_logical_version( tmp_path, repo, name, - "5.3.17.rhlw-00001-n0001", + "5.3.17+test.1.n1", ) pkgs = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") @@ -402,7 +402,7 @@ def test_public_and_predisclosure_collapse_to_logical_version( assert pkg["versions"] == ["5.3.17"] assert len(pkg["latest_releases"]) == 1 assert pkg["latest_releases"][0]["version"] == "5.3.17" - assert pkg["latest_releases"][0]["release"] == "rhlw-00001-n0001" + assert pkg["latest_releases"][0]["release"] == "test.1.n1" metrics = _api_get(bindings_cfg, f"{repo.pulp_href}metrics/") assert metrics == {"package_count": 1, "version_count": 1, "build_count": 2} @@ -432,7 +432,7 @@ def test_public_and_predisclosure_collapse_to_logical_version( assert collapsed["count"] == 1 kept = collapsed["results"][0] assert kept["base_version"] == "5.3.17" - assert kept["version"] == "5.3.17.rhlw-00001-n0001" + assert kept["version"] == "5.3.17+test.1.n1" @pytest.mark.parallel diff --git a/pulp_python/tests/unit/test_catalog.py b/pulp_python/tests/unit/test_catalog.py index 000f8c096..c2e325d00 100644 --- a/pulp_python/tests/unit/test_catalog.py +++ b/pulp_python/tests/unit/test_catalog.py @@ -16,6 +16,9 @@ ) +# A rebuild is a PEP 440 local version: from ``+`` through the end of the +# string. The public version is everything before ``+``. ``release`` is the +# local identifier without the leading ``+``. @pytest.mark.parametrize( "version,expected", [ @@ -23,19 +26,19 @@ ("5.3.17", "5.3.17"), ("5.3.18", "5.3.18"), ("5.3.180", "5.3.180"), - ("5.3.17.rhlw-00001", "5.3.17"), - ("5.3.18.rhlw-00003", "5.3.18"), - ("5.3.17.rhlw-00001-n0001", "5.3.17"), - ("5.3.18.lw-1", "5.3.18"), - ("1.0.0.abc-1", "1.0.0"), - ("1.0.0.ABC-99", "1.0.0"), - ("1.0.foo-bar", "1.0"), - ("1.0.rhlw-١", "1.0"), + ("5.3.17+test.1", "5.3.17"), + ("5.3.18+test.2", "5.3.18"), + ("5.3.17+test.1.n1", "5.3.17"), + ("5.3.17+test.1.n1.hf2", "5.3.17"), + ("1.0.0+foo.bar", "1.0.0"), + ("1.2.3a1+test.1", "1.2.3a1"), + ("1!1.2.3+test.1", "1!1.2.3"), + ("1.0+", "1.0"), ("4.3.0-redhat-1", "4.3.0-redhat-1"), ("5.3.18-anything", "5.3.18-anything"), ("5.3.18.anything", "5.3.18.anything"), - ("1.0.rhlw-00003.extra", "1.0.rhlw-00003.extra"), - ("1.0.rhlw-", "1.0.rhlw-"), + ("5.3.17.test-00001", "5.3.17.test-00001"), + ("1.0.test-1.extra", "1.0.test-1.extra"), ("", ""), (None, None), ], @@ -48,17 +51,17 @@ def test_strip_build_suffix(version, expected): "version,expected", [ ("5.3.18", ""), - ("5.3.17.rhlw-00001", "rhlw-00001"), - ("5.3.18.rhlw-00003", "rhlw-00003"), - ("5.3.17.rhlw-00001-n0001", "rhlw-00001-n0001"), - ("5.3.18.lw-1", "lw-1"), - ("0.1.rhlw-00003", "rhlw-00003"), - ("1.0.foo-bar", "foo-bar"), - ("1.0.rhlw-١", "rhlw-١"), + ("5.3.17+test.1", "test.1"), + ("5.3.18+test.2", "test.2"), + ("5.3.17+test.1.n1", "test.1.n1"), + ("5.3.17+test.1.n1.hf2", "test.1.n1.hf2"), + ("1.0.0+foo.bar", "foo.bar"), + ("1.2.3a1+test.1", "test.1"), + ("1.0+", ""), ("5.3.18.anything", ""), ("4.3.0-redhat-1", ""), ("5.3.18-anything", ""), - ("1.0.rhlw-", ""), + ("5.3.17.test-00001", ""), ("", ""), (None, ""), ],