Merge pull request #717 from gerrod3/normalize-name-pypi

Fix package name normalization for package pypi json view

(cherry picked from commit fc43b4b)

Author: gerrod3

CHANGES/716.bugfix

@@ -0,0 +1 @@
+Fixed package name normalization issue preventing syncing packages with "." or "_" in their names.

pulp_python/app/models.py

@@ -16,6 +16,7 @@
 
 from pathlib import PurePath
 from .utils import (
+    canonicalize_name,
     get_project_metadata_from_artifact,
     parse_project_metadata,
     python_content_to_json,
@@ -84,6 +85,8 @@ def content_handler(self, path):
                         ).latest("pulp_created")
                     except ObjectDoesNotExist:
                         return None
+                    if len(path.parts) == 2:
+                        path = PurePath(f"simple/{canonicalize_name(path.parts[1])}")
                     rel_path = f"{path}/index.html"
                     try:
                         ca = (
@@ -100,8 +103,9 @@ def content_handler(self, path):
                     return ArtifactResponse(ca.artifact, headers=headers)
 
         if name:
+            normalized = canonicalize_name(name)
             package_content = PythonPackageContent.objects.filter(
-                pk__in=self.publication.repository_version.content, name__iexact=name
+                pk__in=self.publication.repository_version.content, name__normalize=normalized
             )
             # TODO Change this value to the Repo's serial value when implemented
             headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)}

pulp_python/app/pypi/views.py

@@ -278,7 +278,8 @@ def retrieve(self, request, path, meta):
         elif meta_path.match("*/json"):
             name = meta_path.parts[0]
         if name:
-            package_content = content.filter(name__iexact=name)
+            normalized = canonicalize_name(name)
+            package_content = content.filter(name__normalize=normalized)
             # TODO Change this value to the Repo's serial value when implemented
             headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)}
             json_body = python_content_to_json(path, package_content, version=version)

pulp_python/tests/functional/api/test_download_content.py

@@ -152,3 +152,31 @@ def test_full_pulp_to_pulp_sync(self):
 
         repo3 = self._create_repo_and_sync_with_remote(remote)
         self.assertEqual(get_content_summary(repo3.to_dict()), PYTHON_MD_FIXTURE_SUMMARY)
+
+
+def test_pulp2pulp_sync_with_oddities(
+    python_repo_with_sync,
+    python_remote_factory,
+    python_publication_factory,
+    python_distribution_factory,
+    python_content_summary,
+):
+    """Test that Pulp can handle syncing packages with wierd names."""
+    remote = python_remote_factory(includes=["oslo.utils"], url="https://pypi.org")
+    repo = python_repo_with_sync(remote)
+    distro = python_distribution_factory(repository=repo.pulp_href)
+    summary = python_content_summary(repository_version=repo.latest_version_href)
+    # Test pulp 2 pulp full sync w/ live pypi apis
+    remote2 = python_remote_factory(includes=[], url=distro.base_url)
+    repo2 = python_repo_with_sync(remote2)
+    summary2 = python_content_summary(repository_version=repo2.latest_version_href)
+    assert summary2.present["python.python"]["count"] > 0
+    assert summary.present["python.python"]["count"] == summary2.present["python.python"]["count"]
+    # Test w/ publication
+    pub = python_publication_factory(repository=repo)
+    distro2 = python_distribution_factory(publication=pub.pulp_href)
+    remote3 = python_remote_factory(includes=[], url=distro2.base_url)
+    repo3 = python_repo_with_sync(remote3)
+    summary3 = python_content_summary(repository_version=repo3.latest_version_href)
+    assert summary3.present["python.python"]["count"] > 0
+    assert summary.present["python.python"]["count"] == summary3.present["python.python"]["count"]

pulp_python/tests/functional/conftest.py

@@ -1,4 +1,5 @@
 import pytest
+import uuid
 
 from pulp_smash.pulp3.utils import gen_distribution, gen_repo
 from pulp_python.tests.functional.utils import gen_python_remote
@@ -9,6 +10,7 @@
     DistributionsPypiApi,
     PublicationsPypiApi,
     RepositoriesPythonApi,
+    RepositoriesPythonVersionsApi,
     RemotesPythonApi,
 )
 
@@ -29,6 +31,12 @@ def python_repo_api_client(python_bindings_client):
     return RepositoriesPythonApi(python_bindings_client)
 
 
+@pytest.fixture
+def python_repo_version_api_client(python_bindings_client):
+    """Provides the Python Repository Version API client object."""
+    return RepositoriesPythonVersionsApi(python_bindings_client)
+
+
 @pytest.fixture
 def python_distro_api_client(python_bindings_client):
     """Provides the Python Distribution API client object."""
@@ -55,6 +63,16 @@ def python_publication_api_client(python_bindings_client):
 
 # Object Generation Fixtures
 
+@pytest.fixture
+def python_repo_factory(python_repo_api_client, gen_object_with_cleanup):
+    """A factory to generate a Python Repository with auto-cleanup."""
+    def _gen_python_repo(**kwargs):
+        kwargs.setdefault("name", str(uuid.uuid4()))
+        return gen_object_with_cleanup(python_repo_api_client, kwargs)
+
+    return _gen_python_repo
+
+
 @pytest.fixture
 def python_repo(python_repo_api_client, gen_object_with_cleanup):
     """Creates a Python Repository and deletes it at test cleanup time."""
@@ -92,3 +110,43 @@ def _gen_python_remote(**kwargs):
         return gen_object_with_cleanup(python_remote_api_client, body)
 
     yield _gen_python_remote
+
+
+@pytest.fixture
+def python_repo_with_sync(
+    python_repo_api_client, python_repo_factory, python_remote_factory, monitor_task
+):
+    """A factory to generate a Python Repository synced with the passed in Remote."""
+    def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body):
+        kwargs = {}
+        if pulp_domain := body.get("pulp_domain"):
+            kwargs["pulp_domain"] = pulp_domain
+        remote = remote or python_remote_factory(**kwargs)
+        repo = repository or python_repo_factory(**body)
+        sync_body = {"mirror": mirror, "remote": remote.pulp_href}
+        monitor_task(python_repo_api_client.sync(repo.pulp_href, sync_body).task)
+        return python_repo_api_client.read(repo.pulp_href)
+
+    yield _gen_python_repo_sync
+
+
+@pytest.fixture
+def python_content_summary(python_repo_api_client, python_repo_version_api_client):
+    """Get a summary of the repository version's content."""
+    def _gen_summary(repository_version=None, repository=None, version=None):
+        if repository_version is None:
+            repo_href = get_href(repository)
+            if version:
+                repo_ver_href = f"{repo_href}versions/{version}/"
+            else:
+                repo_ver_href = python_repo_api_client.read(repo_href).latest_version_href
+        else:
+            repo_ver_href = get_href(repository_version)
+        return python_repo_version_api_client.read(repo_ver_href).content_summary
+
+    yield _gen_summary
+
+
+def get_href(item):
+    """Tries to get the href from the given item, whether it is a string or object."""
+    return item if isinstance(item, str) else item.pulp_href