From c52ecd7de0417d111e21fa499b6fbc3ce292fe9a Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Tue, 11 Nov 2025 17:12:49 -0500 Subject: [PATCH] Implement PEP 700 fixes: #996 --- CHANGES/996.feature | 1 + .../0017_pythonpackagecontent_size.py | 50 +++++++++++++++++++ pulp_python/app/models.py | 1 + pulp_python/app/pypi/views.py | 12 ++++- pulp_python/app/serializers.py | 6 +++ pulp_python/app/utils.py | 24 +++++++-- .../api/test_pypi_simple_json_api.py | 8 ++- 7 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 CHANGES/996.feature create mode 100644 pulp_python/app/migrations/0017_pythonpackagecontent_size.py diff --git a/CHANGES/996.feature b/CHANGES/996.feature new file mode 100644 index 00000000..bd8e582a --- /dev/null +++ b/CHANGES/996.feature @@ -0,0 +1 @@ +Implemented PEP 700 support, adding `versions`, `size` and `upload-time` to the Simple JSON API. diff --git a/pulp_python/app/migrations/0017_pythonpackagecontent_size.py b/pulp_python/app/migrations/0017_pythonpackagecontent_size.py new file mode 100644 index 00000000..a30497a1 --- /dev/null +++ b/pulp_python/app/migrations/0017_pythonpackagecontent_size.py @@ -0,0 +1,50 @@ +# Generated by Django 4.2.26 on 2025-11-11 21:43 + +from django.db import migrations, models, transaction + + +def add_size_to_current_models(apps, schema_editor): + """Adds the size to current PythonPackageContent models.""" + PythonPackageContent = apps.get_model("python", "PythonPackageContent") + RemoteArtifact = apps.get_model("core", "RemoteArtifact") + package_bulk = [] + for python_package in PythonPackageContent.objects.only("pk", "size").iterator(): + content_artifact = python_package.contentartifact_set.first() + if content_artifact.artifact: + artifact = content_artifact.artifact + else: + artifact = RemoteArtifact.objects.filter(content_artifact=content_artifact).first() + python_package.size = artifact.size or 0 + package_bulk.append(python_package) + if len(package_bulk) == 100000: + with transaction.atomic(): + PythonPackageContent.objects.bulk_update( + package_bulk, + [ + "size", + ], + ) + package_bulk = [] + with transaction.atomic(): + PythonPackageContent.objects.bulk_update( + package_bulk, + [ + "size", + ], + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("python", "0016_pythonpackagecontent_metadata_sha256"), + ] + + operations = [ + migrations.AddField( + model_name="pythonpackagecontent", + name="size", + field=models.BigIntegerField(default=0), + ), + migrations.RunPython(add_size_to_current_models, migrations.RunPython.noop, elidable=True), + ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 14059397..d554297f 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -193,6 +193,7 @@ class PythonPackageContent(Content): python_version = models.TextField() sha256 = models.CharField(db_index=True, max_length=64) metadata_sha256 = models.CharField(max_length=64, null=True) + size = models.BigIntegerField(default=0) # yanked and yanked_reason are not implemented because they are mutable # From pulpcore diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index dc4660cd..60c122c9 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -302,6 +302,9 @@ def parse_package(release_package): "sha256": release_package.digests.get("sha256", ""), "requires_python": release_package.requires_python, "metadata_sha256": (release_package.metadata_digests or {}).get("sha256"), + "size": release_package.size, + "upload_time": release_package.upload_time, + "version": release_package.version, } rfilter = get_remote_package_filter(remote) @@ -343,12 +346,19 @@ def retrieve(self, request, path, package): return redirect(urljoin(self.base_content_url, f"{path}/simple/{normalized}/")) if content: packages = content.filter(name__normalize=normalized).values( - "filename", "sha256", "metadata_sha256", "requires_python" + "filename", + "sha256", + "metadata_sha256", + "requires_python", + "size", + "pulp_created", + "version", ) local_releases = { p["filename"]: { **p, "url": urljoin(self.base_content_url, f"{path}/{p['filename']}"), + "upload_time": p["pulp_created"], } for p in packages } diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 2ba95ba4..0998c879 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -277,6 +277,10 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa ), read_only=True, ) + size = serializers.IntegerField( + help_text=_("The size of the package in bytes."), + read_only=True, + ) sha256 = serializers.CharField( default="", help_text=_("The SHA256 digest of this package."), @@ -368,6 +372,7 @@ class Meta: "filename", "packagetype", "python_version", + "size", "sha256", "metadata_sha256", ) @@ -421,6 +426,7 @@ def validate(self, data): data["artifact"] = artifact data["sha256"] = artifact.sha256 data["relative_path"] = filename + data["size"] = artifact.size data.update(parse_project_metadata(vars(metadata))) # Overwrite filename from metadata data["filename"] = filename diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 365de503..0fb6ddbf 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -7,6 +7,7 @@ import json from collections import defaultdict from django.conf import settings +from django.utils import timezone from jinja2 import Template from packaging.utils import canonicalize_name from packaging.requirements import Requirement @@ -18,7 +19,7 @@ """TODO This serial constant is temporary until Python repositories implements serials""" PYPI_SERIAL_CONSTANT = 1000000000 -SIMPLE_API_VERSION = "1.0" +SIMPLE_API_VERSION = "1.1" simple_index_template = """ @@ -161,6 +162,7 @@ def parse_metadata(project, version, distribution): package["sha256"] = distribution.get("digests", {}).get("sha256") or "" package["python_version"] = distribution.get("python_version") or "" package["requires_python"] = distribution.get("requires_python") or "" + package["size"] = distribution.get("size") or 0 return package @@ -223,6 +225,7 @@ def artifact_to_python_content_data(filename, artifact, domain=None): metadata = get_project_metadata_from_file(temp_file.name) data = parse_project_metadata(vars(metadata)) data["sha256"] = artifact.sha256 + data["size"] = artifact.size data["filename"] = filename data["pulp_domain"] = domain or artifact.pulp_domain data["_pulp_domain"] = data["pulp_domain"] @@ -403,7 +406,6 @@ def find_artifact(): components.insert(2, domain.name) url = "/".join(components) md5 = artifact.md5 if artifact and artifact.md5 else "" - size = artifact.size if artifact and artifact.size else 0 return { "comment_text": "", "digests": {"md5": md5, "sha256": content.sha256}, @@ -414,7 +416,7 @@ def find_artifact(): "packagetype": content.packagetype, "python_version": content.python_version, "requires_python": content.requires_python or None, - "size": size, + "size": content.size, "upload_time": str(content.pulp_created), "upload_time_iso_8601": str(content.pulp_created.isoformat()), "url": url, @@ -471,20 +473,32 @@ def write_simple_detail_json(project_name, project_packages): {"sha256": package["metadata_sha256"]} if package["metadata_sha256"] else False ), # yanked and yanked_reason are not implemented because they are mutable + # (v1.1, PEP 700) + "size": package["size"], + "upload-time": format_upload_time(package["upload_time"]), # TODO in the future: - # size, upload-time (v1.1, PEP 700) # core-metadata (PEP 7.14) # provenance (v1.3, PEP 740) } for package in project_packages ], + # (v1.1, PEP 700) + "versions": sorted(set(package["version"] for package in project_packages)), # TODO in the future: - # versions (v1.1, PEP 700) # alternate-locations (v1.2, PEP 708) # project-status (v1.4, PEP 792 - pypi and docs differ) } +def format_upload_time(upload_time): + """Formats the upload time to be in Zulu time. UTC with Z suffix""" + if upload_time: + if upload_time.tzinfo: + dt = upload_time.astimezone(timezone.utc) + return dt.isoformat().replace("+00:00", "Z") + return None + + class PackageIncludeFilter: """A special class to help filter Package's based on a remote's include/exclude""" diff --git a/pulp_python/tests/functional/api/test_pypi_simple_json_api.py b/pulp_python/tests/functional/api/test_pypi_simple_json_api.py index b8c7aa3a..6a0bfb9f 100644 --- a/pulp_python/tests/functional/api/test_pypi_simple_json_api.py +++ b/pulp_python/tests/functional/api/test_pypi_simple_json_api.py @@ -11,7 +11,7 @@ PYTHON_WHEEL_URL, ) -API_VERSION = "1.0" +API_VERSION = "1.1" PYPI_SERIAL_CONSTANT = 1000000000 PYPI_TEXT_HTML = "text/html" @@ -69,6 +69,7 @@ def test_simple_json_detail_api( assert data["meta"] == {"api-version": API_VERSION, "_last-serial": PYPI_SERIAL_CONSTANT} assert data["name"] == "shelf-reader" assert data["files"] + assert data["versions"] == ["0.1"] # Check data of a wheel file_whl = next( @@ -83,7 +84,8 @@ def test_simple_json_detail_api( assert file_whl["data-dist-info-metadata"] == { "sha256": "ed333f0db05d77e933a157b7225b403ada9a2f93318d77b41b662eba78bac350" } - + assert file_whl["size"] == 22455 + assert file_whl["upload-time"] is not None # Check data of a tarball file_tar = next((i for i in data["files"] if i["filename"] == "shelf-reader-0.1.tar.gz"), None) assert file_tar is not None, "tar file not found" @@ -93,6 +95,8 @@ def test_simple_json_detail_api( } assert file_tar["requires-python"] is None assert file_tar["data-dist-info-metadata"] is False + assert file_tar["size"] == 19097 + assert file_tar["upload-time"] is not None @pytest.mark.parallel