Support on-demand content in repair_metadata

closes #849

Author: jobselko

CHANGES/849.feature

@@ -0,0 +1 @@
+Added support for on-demand content to `repair_metadata` endpoint.

pulp_python/app/tasks/repair.py

@@ -2,12 +2,18 @@
 import uuid
 from gettext import gettext as _
 
+from requests.exceptions import RequestException
 from django.db.models.query import QuerySet
 from pulpcore.plugin.models import ProgressReport
 from pulpcore.plugin.util import get_domain
 
 from pulp_python.app.models import PythonPackageContent, PythonRepository
-from pulp_python.app.utils import artifact_to_python_content_data
+from pulp_python.app.utils import (
+    artifact_to_python_content_data,
+    fetch_json_release_metadata,
+    parse_metadata,
+)
+from itertools import groupby
 
 log = logging.getLogger(__name__)
 
@@ -47,8 +53,17 @@ def repair_metadata(content: QuerySet[PythonPackageContent]) -> int:
     Returns:
         int: The number of packages that were repaired.
     """
-    # TODO: Add on_demand content repair
-    immediate_content = content.filter(contentartifact__artifact__isnull=False)
+    immediate_content = (
+        content.filter(contentartifact__artifact__isnull=False)
+        .distinct()
+        .prefetch_related("_artifacts")
+    )
+    on_demand_content = (
+        content.filter(contentartifact__artifact__isnull=True)
+        .distinct()
+        .prefetch_related("contentartifact_set__remoteartifact_set")
+        .order_by("name", "version")
+    )
     domain = get_domain()
 
     batch = []
@@ -58,12 +73,12 @@ def repair_metadata(content: QuerySet[PythonPackageContent]) -> int:
     progress_report = ProgressReport(
         message="Repairing packages' metadata",
         code="repair.metadata",
-        total=immediate_content.count(),
+        total=content.count(),
     )
     progress_report.save()
     with progress_report:
         for package in progress_report.iter(
-            immediate_content.prefetch_related("_artifacts").iterator(chunk_size=1000)
+            immediate_content.iterator(chunk_size=1000)
         ):
             new_data = artifact_to_python_content_data(
                 package.filename, package._artifacts.get(), domain
@@ -82,6 +97,85 @@ def repair_metadata(content: QuerySet[PythonPackageContent]) -> int:
                 batch = []
                 set_of_update_fields.clear()
 
+        # For on-demand content, we expect that:
+        # 1. PythonPackageContent always has correct name and version, and one ContentArtifact
+        # 2. RemoteArtifact always has correct sha256
+        # Repair is only supported if all PythonPackageContent items with the same name and
+        # version (i.e. group) share the same remote URL. Otherwise, the entire group is skipped
+        for (name, version), group in groupby(
+            on_demand_content.iterator(chunk_size=1000),
+            key=lambda x: (x.name, x.version),
+        ):
+            group = list(group)
+            remotes = set(
+                remote
+                for content in group
+                for remote in content.contentartifact_set.get()
+                .remoteartifact_set.all()
+                .values_list("remote__url", flat=True)
+            )
+            if len(remotes) != 1:
+                log.warning(
+                    _("Only one remote url is supported for {} {}").format(
+                        name, version
+                    )
+                )
+                continue
+            remote_url = remotes.pop()
+
+            # Retrieve data with all distributions for the given package version
+            try:
+                json_data = fetch_json_release_metadata(name, version, remote_url)
+            except RequestException as exc:
+                log.warning(
+                    _("Could not fetch metadata for {} {} from {}. Error: {}").format(
+                        name, version, remote_url, exc
+                    )
+                )
+                continue
+
+            for package in progress_report.iter(group):
+                remote_artifacts = (
+                    package.contentartifact_set.get().remoteartifact_set.all()
+                )
+                # Extract data only for the specific distribution being checked
+                dist_data = next(
+                    (
+                        dist
+                        for ra in remote_artifacts
+                        for dist in json_data["urls"]
+                        if ra.sha256 == dist["digests"]["sha256"]
+                    ),
+                    None,
+                )
+                if not dist_data:
+                    log.warning(
+                        _(
+                            "Could not fetch distribution for {} {} with sha256 {}."
+                        ).format(name, version, package.sha256)
+                    )
+                    continue
+
+                new_data = parse_metadata(json_data["info"], package.version, dist_data)
+                new_data.pop("url")  # belongs to RemoteArtifact
+                new_data["pulp_domain"] = domain
+                new_data["_pulp_domain"] = new_data["pulp_domain"]
+                changed = False
+                for field, value in new_data.items():
+                    if getattr(package, field) != value:
+                        setattr(package, field, value)
+                        set_of_update_fields.add(field)
+                        changed = True
+                if changed:
+                    batch.append(package)
+                if len(batch) == 1000:
+                    total_repaired += len(batch)
+                    PythonPackageContent.objects.bulk_update(
+                        batch, set_of_update_fields
+                    )
+                    batch = []
+                    set_of_update_fields.clear()
+
     if batch:
         total_repaired += len(batch)
         PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)

pulp_python/app/utils.py

@@ -1,5 +1,6 @@
 import pkginfo
 import re
+import requests
 import shutil
 import tempfile
 import json
@@ -189,6 +190,19 @@ def artifact_to_python_content_data(filename, artifact, domain=None):
     return data
 
 
+def fetch_json_release_metadata(name: str, version: str, remote_url: str) -> dict:
+    """
+    Fetches metadata for a specific release from PyPI's JSON API. A release can contain
+    multiple distributions. See https://docs.pypi.org/api/json/#get-a-release for more details.
+
+    Returns dict containing "info", "last_serial", "urls", and "vulnerabilities" keys.
+    """
+    url = f"{remote_url}pypi/{name}/{version}/json"
+    response = requests.get(url, timeout=10)
+    response.raise_for_status()
+    return response.json()
+
+
 def python_content_to_json(base_path, content_query, version=None, domain=None):
     """
     Converts a QuerySet of PythonPackageContent into the PyPi JSON format

pulp_python/t_model.py

@@ -0,0 +1,73 @@
+from django.db import models
+from pulpcore.plugin.models import Content
+
+from pulpcore.plugin.util import get_domain_pk
+from pulp_python.app.models import PACKAGE_TYPES, NormalizeName
+
+
+class TestClass:
+    ## Required
+    metadata_version = models.TextField()  # mandatory
+    name = models.TextField()
+    name.register_lookup(NormalizeName)
+    version = models.TextField()
+
+    ## Optional
+    # not: Dynamic, License-Expression, License-File, Provides-Extra,
+    platform = models.TextField()
+    supported_platform = models.TextField()
+    summary = models.TextField()
+    description = models.TextField()
+    description_content_type = models.TextField()
+    keywords = models.TextField()
+    author = models.TextField()
+    author_email = models.TextField()
+    maintainer = models.TextField()
+    maintainer_email = models.TextField()
+    license = models.TextField()
+    classifiers = models.JSONField(default=list)
+    requires_dist = models.JSONField(default=list)
+    requires_python = models.TextField()
+    requires_external = models.JSONField(default=list)
+    project_url = models.TextField()
+    provides_dist = models.JSONField(default=list)
+    obsoletes_dist = models.JSONField(default=list)
+    ## not specified in core metadata, but should be
+    project_urls = models.JSONField(default=dict)
+
+    ## Deprecated
+    # not: Requires, Provides, Obsoletes,
+    home_page = models.TextField()
+    download_url = models.TextField()
+
+    ############### NOT MATCHING "Core metadata specifications"
+    ## From the Content model
+    PROTECTED_FROM_RECLAIM = False
+    TYPE = "python"
+    repo_key_fields = ("filename",)
+    _pulp_domain = models.ForeignKey(
+        "core.Domain", default=get_domain_pk, on_delete=models.PROTECT
+    )
+
+    ## ??
+    # Required metadata
+    filename = models.TextField(db_index=True)
+    packagetype = models.TextField(choices=PACKAGE_TYPES)
+    sha256 = models.CharField(db_index=True, max_length=64)
+    # Optional metadata
+    python_version = models.TextField()
+
+
+class PythonPackageContent(Content):
+    """
+    A Content Type representing Python's Distribution Package.
+
+    As defined in pep-0426 and pep-0345.
+
+    https://www.python.org/dev/peps/pep-0491/
+    https://www.python.org/dev/peps/pep-0345/
+    """
+
+    # todo: 426 withdrawn, replaced by https://packaging.python.org/en/latest/specifications/core-metadata/#core-metadata
+    # todo: 345 was before 426
+    # todo: 491 OK, The Wheel Binary Package Format 1.9

pulp_python/test.py

@@ -0,0 +1,70 @@
+import json
+import re
+import shutil
+import tempfile
+from collections import defaultdict
+
+import pkginfo
+import requests
+from django.conf import settings
+from jinja2 import Template
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
+from packaging.version import InvalidVersion, parse
+from pulpcore.plugin.models import Remote
+
+
+def fetch_json_release_metadata(
+    name: str, version: str, remote: Remote, session_obj: Session
+) -> dict:
+    """
+    Fetches metadata for a specific release from PyPI's JSON API. A release can contain
+    multiple distributions. See https://docs.pypi.org/api/json/#get-a-release for more details.
+
+    Returns dict containing "info", "last_serial", "urls", and "vulnerabilities" keys.
+    """
+    url = f"{remote.url}pypi/{name}/{version}0/json"
+    response = session_obj.get(url, timeout=10)
+    response.raise_for_status()
+    return response.json()
+
+
+def test_metadata_repair_endpoint_on_demand(
+    create_content_remote,
+    monitor_task,
+    move_to_repository,
+    python_bindings,
+    python_remote_factory,
+    python_repo_factory,
+):
+    python_egg_filename = "scipy-1.1.0.tar.gz"
+    python_egg_url = urljoin(
+        urljoin(PYTHON_FIXTURES_URL, "packages/"), python_egg_filename
+    )
+    python_egg_sha256 = (
+        "878352408424dffaa695ffedf2f9f92844e116686923ed9aa8626fc30d32cfd1"
+    )
+    data = {
+        "name": "scipy",
+        "version": "1.1.0",
+        # Wrong metadata
+        "author": "ME",
+        "packagetype": "bdist",
+        "requires_python": ">=3.8",
+    }
+    remote = python_remote_factory(includes=["scipy"])
+    repo = python_repo_factory(remote=remote)
+
+    content = create_content_remote(
+        python_egg_filename, python_egg_url, python_egg_sha256, data, remote
+    )
+    move_to_repository(repo.pulp_href, [content.pulp_href])
+
+    response = python_bindings.RepositoriesPythonApi.repair_metadata(repo.pulp_href)
+    # monitor_task(response.task)
+
+    new_content = python_bindings.ContentPackagesApi.read(content.pulp_href)
+    assert new_content.author == ""
+    assert new_content.packagetype == "sdist"
+    assert new_content.requires_python == ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"
+    assert new_content.version == "1.1.0"