Add JSON-based Simple API

jobselko · jobselko · commit 376106bc7808 · 2025-11-04T22:09:46.000+01:00
diff --git a/pulp_python/app/migrations/0016_pythonpackagecontent_metadata_sha256.py b/pulp_python/app/migrations/0016_pythonpackagecontent_metadata_sha256.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.25 on 2025-11-04 07:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("python", "0015_alter_pythonpackagecontent_options"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="pythonpackagecontent",
+            name="metadata_sha256",
+            field=models.CharField(max_length=64, null=True),
+        ),
+    ]
diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py
@@ -192,6 +192,8 @@ class PythonPackageContent(Content):
     packagetype = models.TextField(choices=PACKAGE_TYPES)
     python_version = models.TextField()
     sha256 = models.CharField(db_index=True, max_length=64)
+    metadata_sha256 = models.CharField(max_length=64, null=True)
+    # yanked and yanked_reason are not implemented because they are mutable
 
     # From pulpcore
     PROTECTED_FROM_RECLAIM = False
diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py
@@ -3,7 +3,9 @@
 
 from aiohttp.client_exceptions import ClientError
 from rest_framework.viewsets import ViewSet
+from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer, TemplateHTMLRenderer
 from rest_framework.response import Response
+from rest_framework.exceptions import NotAcceptable
 from django.core.exceptions import ObjectDoesNotExist
 from django.shortcuts import redirect
 from datetime import datetime, timezone, timedelta
@@ -43,7 +45,9 @@
 )
 from pulp_python.app.utils import (
     write_simple_index,
+    write_simple_index_json,
     write_simple_detail,
+    write_simple_detail_json,
     python_content_to_json,
     PYPI_LAST_SERIAL,
     PYPI_SERIAL_CONSTANT,
@@ -57,6 +61,17 @@
 ORIGIN_HOST = settings.CONTENT_ORIGIN if settings.CONTENT_ORIGIN else settings.PYPI_API_HOSTNAME
 BASE_CONTENT_URL = urljoin(ORIGIN_HOST, settings.CONTENT_PATH_PREFIX)
 
+PYPI_SIMPLE_V1_HTML = "application/vnd.pypi.simple.v1+html"
+PYPI_SIMPLE_V1_JSON = "application/vnd.pypi.simple.v1+json"
+
+
+class PyPISimpleHTMLRenderer(TemplateHTMLRenderer):
+    media_type = PYPI_SIMPLE_V1_HTML
+
+
+class PyPISimpleJSONRenderer(JSONRenderer):
+    media_type = PYPI_SIMPLE_V1_JSON
+
 
 class PyPIMixin:
     """Mixin to get index specific info."""
@@ -235,24 +250,58 @@ class SimpleView(PackageUploadMixin, ViewSet):
         ],
     }
 
+    def perform_content_negotiation(self, request, force=False):
+        """
+        Uses standard content negotiation, defaulting to HTML if no acceptable renderer is found.
+        """
+        try:
+            return super().perform_content_negotiation(request, force)
+        except NotAcceptable:
+            return TemplateHTMLRenderer(), TemplateHTMLRenderer.media_type  # text/html
+
+    def get_renderers(self):
+        """
+        Uses custom renderers for PyPI Simple API endpoints, defaulting to standard ones.
+        """
+        if self.action in ["list", "retrieve"]:
+            # Ordered by priority if multiple content types are present
+            return [TemplateHTMLRenderer(), PyPISimpleHTMLRenderer(), PyPISimpleJSONRenderer()]
+        else:
+            return [JSONRenderer(), BrowsableAPIRenderer()]
+
     @extend_schema(summary="Get index simple page")
     def list(self, request, path):
         """Gets the simple api html page for the index."""
         repo_version, content = self.get_rvc()
         if self.should_redirect(repo_version=repo_version):
             return redirect(urljoin(self.base_content_url, f"{path}/simple/"))
         names = content.order_by("name").values_list("name", flat=True).distinct().iterator()
-        return StreamingHttpResponse(write_simple_index(names, streamed=True))
+        media_type = request.accepted_renderer.media_type
+        headers = {"X-PyPI-Last-Serial": str(PYPI_SERIAL_CONSTANT)}
+
+        if media_type == PYPI_SIMPLE_V1_JSON:
+            index_data = write_simple_index_json(names)
+            return Response(index_data, headers=headers)
+        else:
+            index_data = write_simple_index(names, streamed=True)
+            kwargs = {"content_type": media_type, "headers": headers}
+            return StreamingHttpResponse(index_data, **kwargs)
 
-    def pull_through_package_simple(self, package, path, remote):
+    def pull_through_package_simple(self, package, path, remote, media_type):
         """Gets the package's simple page from remote."""
 
         def parse_package(release_package):
             parsed = urlparse(release_package.url)
             stripped_url = urlunsplit(chain(parsed[:3], ("", "")))
             redirect_path = f"{path}/{release_package.filename}?redirect={stripped_url}"
             d_url = urljoin(self.base_content_url, redirect_path)
-            return release_package.filename, d_url, release_package.digests.get("sha256", "")
+            return {
+                "filename": release_package.filename,
+                "url": d_url,
+                "sha256": release_package.digests.get("sha256", ""),
+                "requires_python": release_package.requires_python,
+                "metadata_sha256": (release_package.metadata_digests or {}).get("sha256", ""),
+            }
 
         rfilter = get_remote_package_filter(remote)
         if not rfilter.filter_project(package):
@@ -269,28 +318,40 @@ def parse_package(release_package):
         except TimeoutException:
             return HttpResponse(f"{remote.url} timed out while fetching {package}.", status=504)
 
-        if d.headers["content-type"] == "application/vnd.pypi.simple.v1+json":
+        if d.headers["content-type"] == PYPI_SIMPLE_V1_JSON:
             page = ProjectPage.from_json_data(json.load(open(d.path, "rb")), base_url=url)
         else:
             page = ProjectPage.from_html(package, open(d.path, "rb").read(), base_url=url)
         packages = [
             parse_package(p) for p in page.packages if rfilter.filter_release(package, p.version)
         ]
-        return HttpResponse(write_simple_detail(package, packages))
+        headers = {"X-PyPI-Last-Serial": str(PYPI_SERIAL_CONSTANT)}
+
+        if media_type == PYPI_SIMPLE_V1_JSON:
+            detail_data = write_simple_detail_json(package, packages)
+            return Response(detail_data, headers=headers)
+        else:
+            detail_data = write_simple_detail(package, packages)
+            kwargs = {"content_type": media_type, "headers": headers}
+            return HttpResponse(detail_data, **kwargs)
 
     @extend_schema(operation_id="pypi_simple_package_read", summary="Get package simple page")
     def retrieve(self, request, path, package):
-        """Retrieves the simple api html page for a package."""
+        """Retrieves the simple api html/json page for a package."""
+        media_type = request.accepted_renderer.media_type
+
         repo_ver, content = self.get_rvc()
         # Should I redirect if the normalized name is different?
         normalized = canonicalize_name(package)
         if self.distribution.remote:
-            return self.pull_through_package_simple(normalized, path, self.distribution.remote)
+            return self.pull_through_package_simple(
+                normalized, path, self.distribution.remote, media_type
+            )
         if self.should_redirect(repo_version=repo_ver):
             return redirect(urljoin(self.base_content_url, f"{path}/simple/{normalized}/"))
         packages = (
             content.filter(name__normalize=normalized)
-            .values_list("filename", "sha256", "name")
+            .values_list("filename", "sha256", "name", "metadata_sha256", "requires_python")
             .iterator()
         )
         try:
@@ -300,8 +361,26 @@ def retrieve(self, request, path, package):
         else:
             packages = chain([present], packages)
             name = present[2]
-        releases = ((f, urljoin(self.base_content_url, f"{path}/{f}"), d) for f, d, _ in packages)
-        return StreamingHttpResponse(write_simple_detail(name, releases, streamed=True))
+        releases = (
+            {
+                "filename": f,
+                "url": urljoin(self.base_content_url, f"{path}/{f}"),
+                "sha256": s,
+                "metadata_sha256": ms,
+                "requires_python": rp,
+            }
+            for f, s, _, ms, rp in packages
+        )
+        media_type = request.accepted_renderer.media_type
+        headers = {"X-PyPI-Last-Serial": str(PYPI_SERIAL_CONSTANT)}
+
+        if media_type == PYPI_SIMPLE_V1_JSON:
+            detail_data = write_simple_detail_json(name, releases)
+            return Response(detail_data, headers=headers)
+        else:
+            detail_data = write_simple_detail(name, releases, streamed=True)
+            kwargs = {"content_type": media_type, "headers": headers}
+            return StreamingHttpResponse(detail_data, **kwargs)
 
     @extend_schema(
         request=PackageUploadSerializer,
diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py
@@ -101,7 +101,7 @@ def write_simple_api(publication):
         relative_path = release["filename"]
         path = f"../../{relative_path}"
         checksum = release["sha256"]
-        package_releases.append((relative_path, path, checksum))
+        package_releases.append({"filename": relative_path, "url": path, "sha256": checksum})
     # Write the final project's page
     write_project_page(
         name=canonicalize_name(current_name),
diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py
@@ -16,32 +16,34 @@
 """TODO This serial constant is temporary until Python repositories implements serials"""
 PYPI_SERIAL_CONSTANT = 1000000000
 
+SIMPLE_API_VERSION = "1.0"
+
 simple_index_template = """<!DOCTYPE html>
 <html>
   <head>
     <title>Simple Index</title>
-    <meta name="api-version" value="2" />
+    <meta name="pypi:repository-version" content="{{ SIMPLE_API_VERSION }}">
   </head>
   <body>
     {% for name, canonical_name in projects %}
-    <a href="{{ canonical_name }}/">{{ name }}</a><br/>
+      <a href="{{ canonical_name }}/">{{ name }}</a><br/>
     {% endfor %}
   </body>
 </html>
 """
 
 simple_detail_template = """<!DOCTYPE html>
 <html>
-<head>
-  <title>Links for {{ project_name }}</title>
-  <meta name="api-version" value="2" />
-</head>
-<body>
+  <head>
+    <title>Links for {{ project_name }}</title>
+    <meta name="pypi:repository-version" content="{{ SIMPLE_API_VERSION }}">
+  </head>
+  <body>
     <h1>Links for {{ project_name }}</h1>
-    {% for name, path, sha256 in project_packages %}
-    <a href="{{ path }}#sha256={{ sha256 }}" rel="internal">{{ name }}</a><br/>
+    {% for pkg in project_packages %}
+      <a href="{{ pkg.url }}#sha256={{ pkg.sha256 }}" rel="internal">{{ pkg.filename }}</a><br/>
     {% endfor %}
-</body>
+  </body>
 </html>
 """
 
@@ -128,6 +130,7 @@ def parse_project_metadata(project):
         # Release metadata
         "packagetype": project.get("packagetype") or "",
         "python_version": project.get("python_version") or "",
+        "metadata_sha256": "",  # TODO
     }
 
 
@@ -158,6 +161,9 @@ def parse_metadata(project, version, distribution):
     package["requires_python"] = distribution.get("requires_python") or package.get(
         "requires_python"
     )  # noqa: E501
+    package["metadata_sha256"] = distribution.get("data-dist-info-metadata", {}).get(
+        "sha256"
+    ) or package.get("metadata_sha256")
 
     return package
 
@@ -403,17 +409,65 @@ def find_artifact():
 def write_simple_index(project_names, streamed=False):
     """Writes the simple index."""
     simple = Template(simple_index_template)
-    context = {"projects": ((x, canonicalize_name(x)) for x in project_names)}
+    context = {
+        "SIMPLE_API_VERSION": SIMPLE_API_VERSION,
+        "projects": ((x, canonicalize_name(x)) for x in project_names),
+    }
     return simple.stream(**context) if streamed else simple.render(**context)
 
 
 def write_simple_detail(project_name, project_packages, streamed=False):
     """Writes the simple detail page of a package."""
     detail = Template(simple_detail_template)
-    context = {"project_name": project_name, "project_packages": project_packages}
+    context = {
+        "SIMPLE_API_VERSION": SIMPLE_API_VERSION,
+        "project_name": project_name,
+        "project_packages": project_packages,
+    }
     return detail.stream(**context) if streamed else detail.render(**context)
 
 
+def write_simple_index_json(project_names):
+    """Writes the simple index in JSON format."""
+    return {
+        "meta": {"api-version": SIMPLE_API_VERSION, "_last-serial": PYPI_SERIAL_CONSTANT},
+        "projects": [
+            {"name": name, "_last-serial": PYPI_SERIAL_CONSTANT} for name in project_names
+        ],
+    }
+
+
+def write_simple_detail_json(project_name, project_packages):
+    """Writes the simple detail page in JSON format."""
+    return {
+        "meta": {"api-version": SIMPLE_API_VERSION, "_last-serial": PYPI_SERIAL_CONSTANT},
+        "name": canonicalize_name(project_name),
+        "files": [
+            {
+                # v1.0, PEP 691
+                "filename": package["filename"],
+                "url": package["url"],
+                "hashes": {"sha256": package["sha256"]},
+                "requires_python": package["requires_python"] or None,
+                # data-dist-info-metadata is deprecated alias for core-metadata
+                "data-dist-info-metadata": (
+                    {"sha256": package["metadata_sha256"]} if package["metadata_sha256"] else False
+                ),
+                # yanked and yanked_reason are not implemented because they are mutable
+                # TODO in the future:
+                # size, upload-time (v1.1, PEP 700)
+                # core-metadata (PEP 7.14)
+                # provenance (v1.3, PEP 740)
+            }
+            for package in project_packages
+        ],
+        # TODO in the future:
+        # versions (v1.1, PEP 700)
+        # alternate-locations (v1.2, PEP 708)
+        # project-status (v1.4, PEP 792 - pypi and docs differ)
+    }
+
+
 class PackageIncludeFilter:
     """A special class to help filter Package's based on a remote's include/exclude"""
 
diff --git a/pulp_python/tests/functional/api/test_full_mirror.py b/pulp_python/tests/functional/api/test_full_mirror.py
@@ -58,6 +58,24 @@ def test_pull_through_simple(python_remote_factory, python_distribution_factory,
         assert PYTHON_XS_FIXTURE_CHECKSUMS[package.filename] == package.digests["sha256"]
 
 
+@pytest.mark.parallel
+@pytest.mark.parametrize("media_type", ["application/vnd.pypi.simple.v1+json", "text/html"])
+def test_pull_through_simple_media_types(
+    media_type, python_remote_factory, python_distribution_factory
+):
+    """Tests pull-through with different media types (JSON and HTML)."""
+    remote = python_remote_factory(url=PYPI_URL, includes=["shelf-reader"])
+    distro = python_distribution_factory(remote=remote.pulp_href)
+
+    url = f"{distro.base_url}simple/shelf-reader/"
+    headers = {"Accept": media_type}
+    response = requests.get(url, headers=headers)
+
+    assert response.status_code == 200
+    assert media_type in response.headers["Content-Type"]
+    assert "X-PyPI-Last-Serial" in response.headers
+
+
 @pytest.mark.parallel
 def test_pull_through_filter(python_remote_factory, python_distribution_factory):
     """Tests that pull-through respects the includes/excludes filter on the remote."""
@@ -66,7 +84,7 @@ def test_pull_through_filter(python_remote_factory, python_distribution_factory)
 
     r = requests.get(f"{distro.base_url}simple/pulpcore/")
     assert r.status_code == 404
-    assert r.json() == {"detail": "pulpcore does not exist."}
+    assert r.text == "404 Not Found"
 
     r = requests.get(f"{distro.base_url}simple/shelf-reader/")
     assert r.status_code == 200
@@ -86,7 +104,7 @@ def test_pull_through_filter(python_remote_factory, python_distribution_factory)
 
     r = requests.get(f"{distro.base_url}simple/django/")
     assert r.status_code == 404
-    assert r.json() == {"detail": "django does not exist."}
+    assert r.text == "404 Not Found"
 
     r = requests.get(f"{distro.base_url}simple/pulpcore/")
     assert r.status_code == 502
diff --git a/pulp_python/tests/functional/api/test_pypi_simple_json_api.py b/pulp_python/tests/functional/api/test_pypi_simple_json_api.py
