diff --git a/CHANGES/933.feature b/CHANGES/933.feature new file mode 100644 index 00000000..a43275cd --- /dev/null +++ b/CHANGES/933.feature @@ -0,0 +1 @@ +Added a synchronous upload API. diff --git a/pulp_python/app/migrations/0015_alter_pythonpackagecontent_options.py b/pulp_python/app/migrations/0015_alter_pythonpackagecontent_options.py new file mode 100644 index 00000000..40bf3208 --- /dev/null +++ b/pulp_python/app/migrations/0015_alter_pythonpackagecontent_options.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.23 on 2025-08-19 17:10 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("python", "0014_pythonpackagecontent_dynamic_and_more"), + ] + + operations = [ + migrations.AlterModelOptions( + name="pythonpackagecontent", + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + "permissions": [ + ("upload_python_packages", "Can upload Python packages using synchronous API.") + ], + }, + ), + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 8535233b..3bd9d605 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -227,6 +227,9 @@ def __str__(self): class Meta: default_related_name = "%(app_label)s_%(model_name)s" unique_together = ("sha256", "_pulp_domain") + permissions = [ + ("upload_python_packages", "Can upload Python packages using synchronous API."), + ] class PythonPublication(Publication, AutoAddObjPermsMixin): diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 179e3656..0f769484 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -1,5 +1,8 @@ +import logging +import os from gettext import gettext as _ from django.conf import settings +from django.db.utils import IntegrityError from packaging.requirements import Requirement from rest_framework import serializers @@ -8,7 +11,15 @@ from pulpcore.plugin.util import get_domain from pulp_python.app import models as python_models -from pulp_python.app.utils import artifact_to_python_content_data +from pulp_python.app.utils import ( + DIST_EXTENSIONS, + artifact_to_python_content_data, + get_project_metadata_from_file, + parse_project_metadata, +) + + +log = logging.getLogger(__name__) class PythonRepositorySerializer(core_serializers.RepositorySerializer): @@ -207,7 +218,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_( - "The Python version(s) that the distribution is guaranteed to be " "compatible with." + "The Python version(s) that the distribution is guaranteed to be compatible with." ), ) # Version 2.1 @@ -215,7 +226,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa required=False, allow_blank=True, help_text=_( - "A string stating the markup syntax (if any) used in the distribution’s" + "A string stating the markup syntax (if any) used in the distribution's" " description, so that tools can intelligently render the description." ), ) @@ -256,7 +267,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa ) packagetype = serializers.CharField( help_text=_( - "The type of the distribution package " "(e.g. sdist, bdist_wheel, bdist_egg, etc)" + "The type of the distribution package (e.g. sdist, bdist_wheel, bdist_egg, etc)" ), read_only=True, ) @@ -357,6 +368,70 @@ class Meta: model = python_models.PythonPackageContent +class PythonPackageContentUploadSerializer(PythonPackageContentSerializer): + """ + A serializer for requests to synchronously upload Python packages. + """ + + def validate(self, data): + """ + Validates an uploaded Python package file, extracts its metadata, + and creates or retrieves an associated Artifact. + + Returns updated data with artifact and metadata details. + """ + file = data.pop("file") + filename = file.name + + for ext, packagetype in DIST_EXTENSIONS.items(): + if filename.endswith(ext): + break + else: + raise serializers.ValidationError( + _( + "Extension on {} is not a valid python extension " + "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)" + ).format(filename) + ) + + # Replace the incorrect file name in the file path with the original file name + original_filepath = file.file.name + path_to_file, tmp_str = original_filepath.rsplit("/", maxsplit=1) + tmp_str = tmp_str.split(".", maxsplit=1)[0] # Remove e.g. ".upload.gz" suffix + new_filepath = f"{path_to_file}/{tmp_str}{filename}" + os.rename(original_filepath, new_filepath) + + metadata = get_project_metadata_from_file(new_filepath) + artifact = core_models.Artifact.init_and_validate(new_filepath) + try: + artifact.save() + except IntegrityError: + artifact = core_models.Artifact.objects.get( + sha256=artifact.sha256, pulp_domain=get_domain() + ) + artifact.touch() + log.info(f"Artifact for {file.name} already existed in database") + + data["artifact"] = artifact + data["sha256"] = artifact.sha256 + data["relative_path"] = filename + data.update(parse_project_metadata(vars(metadata))) + # Overwrite filename from metadata + data["filename"] = filename + return data + + class Meta(PythonPackageContentSerializer.Meta): + # This API does not support uploading to a repository or using a custom relative_path + fields = tuple( + f + for f in PythonPackageContentSerializer.Meta.fields + if f not in ["repository", "relative_path"] + ) + model = python_models.PythonPackageContent + # Name used for the OpenAPI request object + ref_name = "PythonPackageContentUpload" + + class MinimalPythonPackageContentSerializer(PythonPackageContentSerializer): """ A Serializer for PythonPackageContent. @@ -503,7 +578,7 @@ class PythonPublicationSerializer(core_serializers.PublicationSerializer): distributions = core_serializers.DetailRelatedField( help_text=_( - "This publication is currently being hosted as configured by these " "distributions." + "This publication is currently being hosted as configured by these distributions." ), source="distribution_set", view_name="pythondistributions-detail", diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 41702f95..dfe03bfb 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -1,4 +1,5 @@ from bandersnatch.configuration import BandersnatchConfig +from django.db import transaction from drf_spectacular.utils import extend_schema from rest_framework import status from rest_framework.decorators import action @@ -355,10 +356,49 @@ class PythonPackageSingleArtifactContentUploadViewSet( "has_upload_param_model_or_domain_or_obj_perms:core.change_upload", ], }, + { + "action": ["upload"], + "principal": "authenticated", + "effect": "allow", + "condition": [ + "has_model_or_domain_perms:python.upload_python_packages", + ], + }, ], "queryset_scoping": {"function": "scope_queryset"}, } + LOCKED_ROLES = { + "python.python_package_uploader": [ + "python.upload_python_packages", + ], + } + + @extend_schema( + summary="Synchronous Python package upload", + request=python_serializers.PythonPackageContentUploadSerializer, + responses={201: python_serializers.PythonPackageContentSerializer}, + ) + @action( + detail=False, + methods=["post"], + serializer_class=python_serializers.PythonPackageContentUploadSerializer, + ) + def upload(self, request): + """ + Create a Python package. + """ + serializer = self.get_serializer(data=request.data) + + with transaction.atomic(): + # Create the artifact + serializer.is_valid(raise_exception=True) + # Create the package + serializer.save() + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + class PythonRemoteViewSet(core_viewsets.RemoteViewSet, core_viewsets.RolesMixin): """ diff --git a/pulp_python/tests/functional/api/test_upload.py b/pulp_python/tests/functional/api/test_upload.py new file mode 100644 index 00000000..9071cf58 --- /dev/null +++ b/pulp_python/tests/functional/api/test_upload.py @@ -0,0 +1,44 @@ +import pytest +from pulp_python.tests.functional.constants import ( + PYTHON_EGG_FILENAME, + PYTHON_EGG_URL, + PYTHON_WHEEL_FILENAME, + PYTHON_WHEEL_URL, +) + + +@pytest.mark.parametrize( + "pkg_filename, pkg_url", + [(PYTHON_WHEEL_FILENAME, PYTHON_WHEEL_URL), (PYTHON_EGG_FILENAME, PYTHON_EGG_URL)], +) +def test_synchronous_package_upload( + delete_orphans_pre, download_python_file, gen_user, python_bindings, pkg_filename, pkg_url +): + """ + Test synchronously uploading a Python package with labels. + """ + python_file = download_python_file(pkg_filename, pkg_url) + + # Upload a unit with labels + with gen_user(model_roles=["python.python_package_uploader"]): + labels = {"key_1": "value_1"} + content_body = {"file": python_file, "pulp_labels": labels} + package = python_bindings.ContentPackagesApi.upload(**content_body) + assert package.pulp_labels == labels + assert package.name == "shelf-reader" + assert package.filename == pkg_filename + + # Check that uploading the same unit again with different (or same) labels has no effect + with gen_user(model_roles=["python.python_package_uploader"]): + labels_2 = {"key_2": "value_2"} + content_body_2 = {"file": python_file, "pulp_labels": labels_2} + duplicate_package = python_bindings.ContentPackagesApi.upload(**content_body_2) + assert duplicate_package.pulp_href == package.pulp_href + assert duplicate_package.pulp_labels == package.pulp_labels + assert duplicate_package.pulp_labels != labels_2 + + # Check that the upload fails if the user does not have the required permissions + with gen_user(model_roles=[]): + with pytest.raises(python_bindings.ApiException) as ctx: + python_bindings.ContentPackagesApi.upload(**content_body) + assert ctx.value.status == 403 diff --git a/pyproject.toml b/pyproject.toml index 9af9375a..8085acf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers=[ ] requires-python = ">=3.11" dependencies = [ - "pulpcore>=3.49.0,<3.100", + "pulpcore>=3.81.0,<3.100", "pkginfo>=1.12.0,<1.13.0", "bandersnatch>=6.3.0,<6.6", # 6.6 has breaking changes "pypi-simple>=1.5.0,<2.0",