Add JSON-based Simple API

closes #625

Author: jobselko

pulp_python/app/pypi/views.py

@@ -3,6 +3,7 @@
 
 from aiohttp.client_exceptions import ClientError
 from rest_framework.viewsets import ViewSet
+from rest_framework.renderers import JSONRenderer, TemplateHTMLRenderer
 from rest_framework.response import Response
 from django.core.exceptions import ObjectDoesNotExist
 from django.shortcuts import redirect
@@ -17,6 +18,7 @@
     HttpResponseBadRequest,
     StreamingHttpResponse,
     HttpResponse,
+    JsonResponse,
 )
 from drf_spectacular.utils import extend_schema
 from dynaconf import settings
@@ -43,7 +45,9 @@
 )
 from pulp_python.app.utils import (
     write_simple_index,
+    write_simple_index_json,
     write_simple_detail,
+    write_simple_detail_json,
     python_content_to_json,
     PYPI_LAST_SERIAL,
     PYPI_SERIAL_CONSTANT,
@@ -57,6 +61,30 @@
 ORIGIN_HOST = settings.CONTENT_ORIGIN if settings.CONTENT_ORIGIN else settings.PYPI_API_HOSTNAME
 BASE_CONTENT_URL = urljoin(ORIGIN_HOST, settings.CONTENT_PATH_PREFIX)
 
+PYPI_TEXT_HTML = "text/html"
+PYPI_SIMPLE_V1_HTML = "application/vnd.pypi.simple.v1+html"
+PYPI_SIMPLE_V1_JSON = "application/vnd.pypi.simple.v1+json"
+
+
+class PyPISimpleHTMLRenderer(TemplateHTMLRenderer):
+    media_type = PYPI_SIMPLE_V1_HTML
+
+
+class PyPISimpleJSONRenderer(JSONRenderer):
+    media_type = PYPI_SIMPLE_V1_JSON
+
+
+def _select_content_type(request):
+    """Select content type based on Accept header."""
+    accept_header = request.META.get("HTTP_ACCEPT", "")
+
+    for content_type in (PYPI_TEXT_HTML, PYPI_SIMPLE_V1_HTML, PYPI_SIMPLE_V1_JSON):
+        if content_type in accept_header:
+            return content_type
+    if not accept_header:
+        return PYPI_TEXT_HTML
+    return None
+
 
 class PyPIMixin:
     """Mixin to get index specific info."""
@@ -235,14 +263,32 @@ class SimpleView(PackageUploadMixin, ViewSet):
         ],
     }
 
+    renderer_classes = [
+        TemplateHTMLRenderer,
+        PyPISimpleHTMLRenderer,
+        PyPISimpleJSONRenderer,
+    ]
+
     @extend_schema(summary="Get index simple page")
     def list(self, request, path):
         """Gets the simple api html page for the index."""
+        content_type = _select_content_type(request)
+        if content_type is None:
+            return HttpResponse("Not Acceptable Content-Type", status=406)
+
         repo_version, content = self.get_rvc()
         if self.should_redirect(repo_version=repo_version):
             return redirect(urljoin(self.base_content_url, f"{path}/simple/"))
         names = content.order_by("name").values_list("name", flat=True).distinct().iterator()
-        return StreamingHttpResponse(write_simple_index(names, streamed=True))
+
+        if content_type == PYPI_SIMPLE_V1_JSON:
+            names_list = list(names)
+            data_dict = write_simple_index_json(names_list)
+            headers = {"X-PyPI-Last-Serial": str(PYPI_SERIAL_CONSTANT)}
+            response = JsonResponse(data_dict, content_type=content_type, headers=headers)
+            return response
+        else:
+            return StreamingHttpResponse(write_simple_index(names, streamed=True))
 
     def pull_through_package_simple(self, package, path, remote):
         """Gets the package's simple page from remote."""
@@ -281,6 +327,10 @@ def parse_package(release_package):
     @extend_schema(operation_id="pypi_simple_package_read", summary="Get package simple page")
     def retrieve(self, request, path, package):
         """Retrieves the simple api html page for a package."""
+        content_type = _select_content_type(request)
+        if content_type is None:
+            return HttpResponse("Not Acceptable Content-Type", status=406)
+
         repo_ver, content = self.get_rvc()
         # Should I redirect if the normalized name is different?
         normalized = canonicalize_name(package)
@@ -301,7 +351,15 @@ def retrieve(self, request, path, package):
             packages = chain([present], packages)
             name = present[2]
         releases = ((f, urljoin(self.base_content_url, f"{path}/{f}"), d) for f, d, _ in packages)
-        return StreamingHttpResponse(write_simple_detail(name, releases, streamed=True))
+
+        if content_type == PYPI_SIMPLE_V1_JSON:
+            releases_list = list(releases)
+            data_dict = write_simple_detail_json(name, releases_list)
+            headers = {"X-PyPI-Last-Serial": str(PYPI_SERIAL_CONSTANT)}
+            response = JsonResponse(data_dict, content_type=content_type, headers=headers)
+            return response
+        else:
+            return StreamingHttpResponse(write_simple_detail(name, releases, streamed=True))
 
     @extend_schema(
         request=PackageUploadSerializer,

pulp_python/app/utils.py

@@ -16,6 +16,10 @@
 """TODO This serial constant is temporary until Python repositories implements serials"""
 PYPI_SERIAL_CONSTANT = 1000000000
 
+# todo: update to reflect supported version
+# https://packaging.python.org/en/latest/specifications/simple-repository-api/#api-version-history
+PYPI_API_VERSION = "1.4"
+
 simple_index_template = """<!DOCTYPE html>
 <html>
   <head>
@@ -414,6 +418,46 @@ def write_simple_detail(project_name, project_packages, streamed=False):
     return detail.stream(**context) if streamed else detail.render(**context)
 
 
+def write_simple_index_json(project_names):
+    """Writes the simple index in JSON format."""
+    projects = [{"_last-serial": PYPI_SERIAL_CONSTANT, "name": name} for name in project_names]
+
+    return {
+        "meta": {"api-version": PYPI_API_VERSION, "_last-serial": PYPI_SERIAL_CONSTANT},
+        "projects": projects,
+    }
+
+
+# todo: fields
+def write_simple_detail_json(project_name, project_packages):
+    """Writes the simple detail page in JSON format."""
+    files = []
+    for filename, url, sha256 in project_packages:
+        files.append(
+            {
+                "filename": filename,
+                "url": url,
+                "hashes": {"sha256": sha256},
+                # requires_python
+                # size / version 1.1
+                # upload-time / 1.1
+                # yanked
+                # data-dist-info-metadata
+                # core-metadata
+                # provenance / 1.3
+            }
+        )
+
+    return {
+        "meta": {"api-version": PYPI_API_VERSION, "_last-serial": PYPI_SERIAL_CONSTANT},
+        "name": project_name,  # should be normalized
+        # project-status / 1.4
+        # versions / 1.1
+        # alternate-locations
+        "files": files,
+    }
+
+
 class PackageIncludeFilter:
     """A special class to help filter Package's based on a remote's include/exclude"""
 

pulp_python/tests/functional/api/test_pypi_apis.py

@@ -279,6 +279,40 @@ def test_simple_correctness_live(
     assert proper is True, msgs
 
 
+@pytest.mark.parallel
+def test_simple_json_api(python_remote_factory, python_repo_with_sync, python_distribution_factory):
+    remote = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER)
+    repo = python_repo_with_sync(remote)
+    distro = python_distribution_factory(repository=repo)
+
+    index_url = urljoin(distro.base_url, "simple/")
+    detail_url = f"{index_url}aiohttp"
+
+    headers = {"Accept": "application/vnd.pypi.simple.v1+json"}  # ok
+    # headers = {"Accept": "application/json"}  # nok
+
+    response_index = requests.get(index_url, headers=headers)
+    data_index = response_index.json()
+    assert data_index["meta"] == {"api-version": "1.4", "_last-serial": 1000000000}
+    assert data_index["projects"]
+    for project in data_index["projects"]:
+        for i in ["_last-serial", "name"]:
+            assert project[i]
+    assert response_index.headers["Content-Type"] == "application/vnd.pypi.simple.v1+json"
+    assert response_index.headers["X-PyPI-Last-Serial"] == "1000000000"
+
+    response_detail = requests.get(detail_url, headers=headers)
+    data_detail = response_detail.json()
+    assert data_detail["meta"] == {"api-version": "1.4", "_last-serial": 1000000000}
+    assert data_detail["name"] == "aiohttp"
+    assert data_detail["files"]
+    for file in data_detail["files"]:
+        for i in ["filename", "url", "hashes"]:
+            assert file[i]
+    assert response_detail.headers["Content-Type"] == "application/vnd.pypi.simple.v1+json"
+    assert response_detail.headers["X-PyPI-Last-Serial"] == "1000000000"
+
+
 @pytest.mark.parallel
 def test_pypi_json(python_remote_factory, python_repo_with_sync, python_distribution_factory):
     """Checks the data of `pypi/{package_name}/json` endpoint."""