Add synchronous upload API

closes #933

Author: jobselko

pulp_python/app/models.py

@@ -227,6 +227,10 @@ def __str__(self):
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("sha256", "_pulp_domain")
+        # todo
+        # permissions = [
+        #     ("upload_python_packages", "Can upload Python packages using synchronous API."),
+        # ]
 
 
 class PythonPublication(Publication, AutoAddObjPermsMixin):

pulp_python/app/serializers.py

@@ -1,5 +1,7 @@
+import logging
 from gettext import gettext as _
 from django.conf import settings
+from django.db.utils import IntegrityError
 from packaging.requirements import Requirement
 from rest_framework import serializers
 
@@ -8,7 +10,13 @@
 from pulpcore.plugin.util import get_domain
 
 from pulp_python.app import models as python_models
-from pulp_python.app.utils import artifact_to_python_content_data
+from pulp_python.app.utils import (
+    DIST_EXTENSIONS,
+    artifact_to_python_content_data,
+    get_project_metadata_from_file,
+)
+
+log = logging.getLogger(__name__)
 
 
 class PythonRepositorySerializer(core_serializers.RepositorySerializer):
@@ -215,7 +223,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
         required=False,
         allow_blank=True,
         help_text=_(
-            "A string stating the markup syntax (if any) used in the distribution’s"
+            "A string stating the markup syntax (if any) used in the distribution's"
             " description, so that tools can intelligently render the description."
         ),
     )
@@ -357,6 +365,57 @@ class Meta:
         model = python_models.PythonPackageContent
 
 
+class PythonPackageContentUploadSerializer(PythonPackageContentSerializer):
+    """
+    A serializer for requests to synchronously upload Python packages.
+    """
+
+    class Meta(PythonPackageContentSerializer.Meta):
+        # This API does not support uploading to a repository or using a custom relative_path
+        fields = tuple(
+            f
+            for f in PythonPackageContentSerializer.Meta.fields
+            if f not in ["repository", "relative_path"]
+        )
+        model = python_models.PythonPackageContent
+        # Name used for the OpenAPI request object
+        ref_name = "PythonPackageContentUpload"
+
+    def validate(self, data):
+        file = data.pop("file")
+
+        for ext, packagetype in DIST_EXTENSIONS.items():
+            if file.name.endswith(ext):
+                break
+        else:
+            raise serializers.ValidationError(
+                _(
+                    "Extension on {} is not a valid python extension "
+                    "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)"
+                ).format(file.name)
+            )
+
+        metadata = get_project_metadata_from_file(file)
+        artifact = core_models.Artifact.init_and_validate(file)
+        try:
+            artifact.save()
+        except IntegrityError:
+            artifact = core_models.Artifact.objects.get(
+                sha256=artifact.sha256, pulp_domain=get_domain()
+            )
+            artifact.touch()
+            log.info(f"Artifact for {file.name} already existed in database")
+
+        data["artifact"] = artifact
+        data["sha256"] = artifact.sha256
+        data.update(vars(metadata))
+        # e.g. '/var/lib/pulp/tmp/tmphexlltu1.upload.gz'
+        data["relative_path"] = vars(metadata)["filename"]
+        # e.g. 'shelf-reader-0.1.tar.gz' instead of '/var/lib/pulp/tmp/tmphexlltu1.upload.gz'
+        data["filename"] = file.name
+        return data
+
+
 class MinimalPythonPackageContentSerializer(PythonPackageContentSerializer):
     """
     A Serializer for PythonPackageContent.

pulp_python/app/utils.py

@@ -162,7 +162,7 @@ def parse_metadata(project, version, distribution):
     return package
 
 
-def get_project_metadata_from_artifact(filename, artifact):
+def get_project_metadata_from_file(file):
     """
     Gets the metadata of a Python Package.
 
@@ -171,32 +171,34 @@ def get_project_metadata_from_artifact(filename, artifact):
     extensions = list(DIST_EXTENSIONS.keys())
     # Iterate through extensions since splitext does not support things like .tar.gz
     # If no supported extension is found, ValueError is raised here
-    pkg_type_index = [filename.endswith(ext) for ext in extensions].index(True)
+    pkg_type_index = [file.name.endswith(ext) for ext in extensions].index(True)
     packagetype = DIST_EXTENSIONS[extensions[pkg_type_index]]
-    # Copy file to a temp directory under the user provided filename, we do this
-    # because pkginfo validates that the filename has a valid extension before
-    # reading it
-    with tempfile.NamedTemporaryFile("wb", dir=".", suffix=filename) as temp_file:
-        shutil.copyfileobj(artifact.file, temp_file)
-        temp_file.flush()
-        metadata = DIST_TYPES[packagetype](temp_file.name)
-        metadata.packagetype = packagetype
-        if packagetype == "sdist":
-            metadata.python_version = "source"
-        else:
-            pyver = ""
-            regex = DIST_REGEXES[extensions[pkg_type_index]]
-            if bdist_name := regex.match(filename):
-                pyver = bdist_name.group("pyver") or ""
-            metadata.python_version = pyver
-        return metadata
+
+    # todo (note): file.name was replaced by file.file.name
+    metadata = DIST_TYPES[packagetype](file.file.name)
+    metadata.packagetype = packagetype
+    if packagetype == "sdist":
+        metadata.python_version = "source"
+    else:
+        pyver = ""
+        regex = DIST_REGEXES[extensions[pkg_type_index]]
+        if bdist_name := regex.match(file.name):
+            pyver = bdist_name.group("pyver") or ""
+        metadata.python_version = pyver
+    return metadata
 
 
 def artifact_to_python_content_data(filename, artifact, domain=None):
     """
     Takes the artifact/filename and returns the metadata needed to create a PythonPackageContent.
     """
-    metadata = get_project_metadata_from_artifact(filename, artifact)
+    # Copy file to a temp directory under the user provided filename, we do this
+    # because pkginfo validates that the filename has a valid extension before
+    # reading it
+    with tempfile.NamedTemporaryFile("wb", dir=".", suffix=filename) as temp_file:
+        shutil.copyfileobj(artifact.file, temp_file)
+        temp_file.flush()
+        metadata = get_project_metadata_from_file(temp_file)
     data = parse_project_metadata(vars(metadata))
     data["sha256"] = artifact.sha256
     data["filename"] = filename

pulp_python/app/viewsets.py

@@ -1,4 +1,5 @@
 from bandersnatch.configuration import BandersnatchConfig
+from django.db import transaction
 from drf_spectacular.utils import extend_schema
 from rest_framework import status
 from rest_framework.decorators import action
@@ -355,10 +356,51 @@ class PythonPackageSingleArtifactContentUploadViewSet(
                     "has_upload_param_model_or_domain_or_obj_perms:core.change_upload",
                 ],
             },
+            {
+                "action": ["upload"],
+                "principal": "authenticated",
+                "effect": "allow",
+                # todo: proper conditions (copied from "create" for now)
+                "condition": [
+                    "has_required_repo_perms_on_upload:python.modify_pythonrepository",
+                    "has_required_repo_perms_on_upload:python.view_pythonrepository",
+                    "has_upload_param_model_or_domain_or_obj_perms:core.change_upload",
+                ],
+            },
         ],
         "queryset_scoping": {"function": "scope_queryset"},
     }
 
+    # todo (plus perms in PPC model)
+    # LOCKED_ROLES = {
+    #     "python.python_package_uploader": [
+    #         "python.upload_python_packages",
+    #     ],
+    # }
+
+    @extend_schema(
+        summary="Synchronous Python package upload",
+        request=python_serializers.PythonPackageContentUploadSerializer,
+        responses={201: python_serializers.PythonPackageContentSerializer},
+    )
+    @action(
+        detail=False,
+        methods=["post"],
+        serializer_class=python_serializers.PythonPackageContentUploadSerializer,
+    )
+    def upload(self, request):
+        """Create a PPC."""
+        serializer = self.get_serializer(data=request.data)
+
+        with transaction.atomic():
+            # Create the artifact
+            serializer.is_valid(raise_exception=True)
+            # Create the Package (ContentArtifact, Content, PPC)
+            serializer.save()
+
+        headers = self.get_success_headers(serializer.data)
+        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
+
 
 class PythonRemoteViewSet(core_viewsets.RemoteViewSet, core_viewsets.RolesMixin):
     """

pulp_python/tests/functional/api/test_upload.py

@@ -0,0 +1,36 @@
+from pulp_python.tests.functional.constants import (
+    PYTHON_EGG_FILENAME,
+    PYTHON_EGG_URL,
+)
+
+
+def test_synchronous_package_upload(
+    delete_orphans_pre, download_python_file, gen_user, python_bindings
+):
+    """
+    Test synchronously uploading a Python package with labels.
+    """
+    # todo: gen_user and perms?
+    download_python_file(PYTHON_EGG_FILENAME, PYTHON_EGG_URL)
+
+    # Upload a unit wih labels
+    labels = {"key_1": "value_1"}
+    content_body = {"file": PYTHON_EGG_FILENAME, "pulp_labels": labels}
+    package = python_bindings.ContentPackagesApi.upload(**content_body)
+    assert package.pulp_labels == labels
+    assert package.name == "shelf-reader"
+    assert package.filename == "shelf-reader-0.1.tar.gz"
+
+    # Check that uploading the same unit again with different (or same) labels does nothing
+    labels_2 = {"key_2": "value_2"}
+    content_body_2 = {"file": PYTHON_EGG_FILENAME, "pulp_labels": labels_2}
+    duplicate_package = python_bindings.ContentPackagesApi.upload(**content_body_2)
+    assert duplicate_package.pulp_href == package.pulp_href
+    assert duplicate_package.pulp_labels == package.pulp_labels
+    assert duplicate_package.pulp_labels != labels_2
+
+    # todo: should work (i.e. fail) once perms are used correctly in the code
+    # # Check that the upload fails if user does not have perms
+    # with pytest.raises(python_bindings.ApiException) as ctx:
+    #     duplicate_package_2 = python_bindings.ContentPackagesApi.upload(**content_body)
+    # assert ctx.value.status == 403