Add new repair_metadata endpoint

closes #805

Author: jobselko

CHANGES/805.feature

@@ -0,0 +1 @@
+Added new `repair_metadata` endpoint to `Repository` for fixing packages' metadata.

pulp_python/app/tasks/__init__.py

@@ -3,5 +3,6 @@
 """
 
 from .publish import publish  # noqa:F401
+from .repair import repair  # noqa:F401
 from .sync import sync  # noqa:F401
 from .upload import upload, upload_group  # noqa:F401

pulp_python/app/tasks/repair.py

@@ -0,0 +1,88 @@
+import os
+from gettext import gettext as _
+import logging
+
+import uuid
+from django.conf import settings
+
+from pulp_python.app.models import PythonPackageContent, PythonRepository
+from pulp_python.app.utils import artifact_to_python_content_data
+
+log = logging.getLogger(__name__)
+
+
+def repair(repository_pk: uuid.UUID) -> None:
+    """
+    Repairs metadata of all packages for the specified repository.
+
+    Args:
+        repository_pk (uuid.UUID): The primary key of the repository to repair.
+
+    Returns:
+        None
+    """
+    repository = PythonRepository.objects.get(pk=repository_pk)
+
+    log.info(
+        _(
+            "Repairing packages' metadata for the latest version of repository {}."
+        ).format(repository.name)
+    )
+    content_set = repository.latest_version().content.values_list("pk", flat=True)
+    content = PythonPackageContent.objects.filter(pk__in=content_set)
+    num_repaired = repair_metadata(content)
+    log.info(
+        _("{} packages processed, {} package metadata repaired.").format(
+            len(content_set), num_repaired
+        )
+    )
+
+
+# TODO: include ContentQuerySet in pulpcore/pulpcore/app/models/__init__.py to enable type hint
+def repair_metadata(content) -> int:
+    """
+    Repairs metadata for a queryset of PythonPackageContent objects.
+
+    Args:
+        content (ContentQuerySet): The queryset of PythonPackageContent items to repair.
+
+    Returns:
+        int: The number of packages that were repaired.
+    """
+    # TODO: Add on_demand content repair
+    os.chdir(settings.WORKING_DIRECTORY)
+    content = content.select_related("pulp_domain")
+    immediate_content = content.filter(contentartifact__artifact__isnull=False)
+
+    batch = []
+    set_of_update_fields = set()
+    total_repaired = 0
+
+    for package in immediate_content.prefetch_related("_artifacts").iterator(
+        chunk_size=1000
+    ):
+        import epdb
+
+        # epdb.serve()
+        new_data = artifact_to_python_content_data(
+            package.filename, package._artifacts.get(), package.pulp_domain
+        )
+        changed = False
+        for field, value in new_data.items():
+            if getattr(package, field) != value:
+                setattr(package, field, value)
+                set_of_update_fields.add(field)
+                changed = True
+        if changed:
+            batch.append(package)
+        if len(batch) == 1000:
+            total_repaired += len(batch)
+            PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
+            batch = []
+            set_of_update_fields.clear()
+
+    if batch:
+        total_repaired += len(batch)
+        PythonPackageContent.objects.bulk_update(batch, set_of_update_fields)
+
+    return total_repaired

pulp_python/app/viewsets.py

@@ -83,7 +83,7 @@ class PythonRepositoryViewSet(
                 ],
             },
             {
-                "action": ["modify"],
+                "action": ["modify", "repair_metadata"],
                 "principal": "authenticated",
                 "effect": "allow",
                 "condition": [
@@ -122,6 +122,26 @@ class PythonRepositoryViewSet(
         "python.pythonrepository_viewer": ["python.view_pythonrepository"],
     }
 
+    @extend_schema(
+        summary="Repair metadata",
+        responses={202: AsyncOperationResponseSerializer},
+    )
+    @action(detail=True, methods=["post"], serializer_class=None)
+    def repair_metadata(self, request, pk):
+        """
+        <!-- User-facing documentation, rendered as html-->
+        Trigger an asynchronous task to repair Python metadata. This task will repair metadata
+        of all packages for the specified `Repository`, without creating a new `RepositoryVersion`.
+        """
+        repository = self.get_object()
+
+        result = dispatch(
+            tasks.repair,
+            exclusive_resources=[repository],
+            kwargs={"repository_pk": str(repository.pk)},
+        )
+        return core_viewsets.OperationPostponedResponse(result, request)
+
     @extend_schema(
         summary="Sync from remote",
         responses={202: AsyncOperationResponseSerializer}

pulp_python/tests/functional/api/test_repair.py

@@ -1,7 +1,11 @@
 import pytest
 import subprocess
+from urllib.parse import urljoin
 
-from pulp_python.tests.functional.constants import PYTHON_EGG_FILENAME
+from pulp_python.tests.functional.constants import (
+    PYTHON_EGG_FILENAME,
+    PYTHON_FIXTURES_URL,
+)
 
 
 @pytest.fixture
@@ -76,3 +80,47 @@ def test_metadata_repair_command(
     assert content.packagetype == "sdist"
     assert content.requires_python == ""  # technically null
     assert content.author == "Austin Macdonald"
+
+
+def test_metadata_repair_endpoint(
+    create_content_direct,
+    download_python_file,
+    monitor_task,
+    move_to_repository,
+    python_bindings,
+    python_repo,
+):
+    """
+    Test repairing of package metadata via `Repositories.repair_metadata` endpoint.
+    """
+    python_egg_filename = "scipy-1.1.0.tar.gz"
+    python_egg_url = urljoin(
+        urljoin(PYTHON_FIXTURES_URL, "packages/"), python_egg_filename
+    )
+    python_file = download_python_file(python_egg_filename, python_egg_url)
+
+    data = {
+        "name": "scipy",
+        # Wrong metadata
+        "author": "ME",
+        "packagetype": "bdist",
+        "requires_python": ">=3.8",
+        "version": "0.2",
+    }
+    content = create_content_direct(python_file, python_egg_filename, data)
+    for field, wrong_value in data.items():
+        if field == "python_version":
+            continue
+        assert getattr(content, field) == wrong_value
+    move_to_repository(python_repo.pulp_href, [content.pulp_href])
+
+    response = python_bindings.RepositoriesPythonApi.repair_metadata(
+        python_repo.pulp_href
+    )
+    _ = monitor_task(response.task)
+
+    content = python_bindings.ContentPackagesApi.read(content.pulp_href)
+    assert content.version == "1.1.0"
+    assert content.packagetype == "sdist"
+    assert content.requires_python == ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"
+    assert content.author == ""