diff --git a/CHANGES/716.bugfix b/CHANGES/716.bugfix new file mode 100644 index 00000000..344bbf23 --- /dev/null +++ b/CHANGES/716.bugfix @@ -0,0 +1 @@ +Fixed package name normalization issue preventing syncing packages with "." or "_" in their names. diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 0e8b29fc..7f69aa9b 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -16,6 +16,7 @@ from pathlib import PurePath from .utils import ( + canonicalize_name, get_project_metadata_from_artifact, parse_project_metadata, python_content_to_json, @@ -84,6 +85,8 @@ def content_handler(self, path): ).latest("pulp_created") except ObjectDoesNotExist: return None + if len(path.parts) == 2: + path = PurePath(f"simple/{canonicalize_name(path.parts[1])}") rel_path = f"{path}/index.html" try: ca = ( @@ -100,8 +103,9 @@ def content_handler(self, path): return ArtifactResponse(ca.artifact, headers=headers) if name: + normalized = canonicalize_name(name) package_content = PythonPackageContent.objects.filter( - pk__in=self.publication.repository_version.content, name__iexact=name + pk__in=self.publication.repository_version.content, name__normalize=normalized ) # TODO Change this value to the Repo's serial value when implemented headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index ab276558..d2e34f6b 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -278,7 +278,8 @@ def retrieve(self, request, path, meta): elif meta_path.match("*/json"): name = meta_path.parts[0] if name: - package_content = content.filter(name__iexact=name) + normalized = canonicalize_name(name) + package_content = content.filter(name__normalize=normalized) # TODO Change this value to the Repo's serial value when implemented headers = {PYPI_LAST_SERIAL: str(PYPI_SERIAL_CONSTANT)} json_body = python_content_to_json(path, package_content, version=version) diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index 2f2b2eb6..8800bc9f 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -152,3 +152,31 @@ def test_full_pulp_to_pulp_sync(self): repo3 = self._create_repo_and_sync_with_remote(remote) self.assertEqual(get_content_summary(repo3.to_dict()), PYTHON_MD_FIXTURE_SUMMARY) + + +def test_pulp2pulp_sync_with_oddities( + python_repo_with_sync, + python_remote_factory, + python_publication_factory, + python_distribution_factory, + python_content_summary, +): + """Test that Pulp can handle syncing packages with wierd names.""" + remote = python_remote_factory(includes=["oslo.utils"], url="https://pypi.org") + repo = python_repo_with_sync(remote) + distro = python_distribution_factory(repository=repo.pulp_href) + summary = python_content_summary(repository_version=repo.latest_version_href) + # Test pulp 2 pulp full sync w/ live pypi apis + remote2 = python_remote_factory(includes=[], url=distro.base_url) + repo2 = python_repo_with_sync(remote2) + summary2 = python_content_summary(repository_version=repo2.latest_version_href) + assert summary2.present["python.python"]["count"] > 0 + assert summary.present["python.python"]["count"] == summary2.present["python.python"]["count"] + # Test w/ publication + pub = python_publication_factory(repository=repo) + distro2 = python_distribution_factory(publication=pub.pulp_href) + remote3 = python_remote_factory(includes=[], url=distro2.base_url) + repo3 = python_repo_with_sync(remote3) + summary3 = python_content_summary(repository_version=repo3.latest_version_href) + assert summary3.present["python.python"]["count"] > 0 + assert summary.present["python.python"]["count"] == summary3.present["python.python"]["count"] diff --git a/pulp_python/tests/functional/conftest.py b/pulp_python/tests/functional/conftest.py index d3674530..408f46a9 100644 --- a/pulp_python/tests/functional/conftest.py +++ b/pulp_python/tests/functional/conftest.py @@ -1,4 +1,5 @@ import pytest +import uuid from pulp_smash.pulp3.utils import gen_distribution, gen_repo from pulp_python.tests.functional.utils import gen_python_remote @@ -9,6 +10,7 @@ DistributionsPypiApi, PublicationsPypiApi, RepositoriesPythonApi, + RepositoriesPythonVersionsApi, RemotesPythonApi, ) @@ -29,6 +31,12 @@ def python_repo_api_client(python_bindings_client): return RepositoriesPythonApi(python_bindings_client) +@pytest.fixture +def python_repo_version_api_client(python_bindings_client): + """Provides the Python Repository Version API client object.""" + return RepositoriesPythonVersionsApi(python_bindings_client) + + @pytest.fixture def python_distro_api_client(python_bindings_client): """Provides the Python Distribution API client object.""" @@ -55,6 +63,16 @@ def python_publication_api_client(python_bindings_client): # Object Generation Fixtures +@pytest.fixture +def python_repo_factory(python_repo_api_client, gen_object_with_cleanup): + """A factory to generate a Python Repository with auto-cleanup.""" + def _gen_python_repo(**kwargs): + kwargs.setdefault("name", str(uuid.uuid4())) + return gen_object_with_cleanup(python_repo_api_client, kwargs) + + return _gen_python_repo + + @pytest.fixture def python_repo(python_repo_api_client, gen_object_with_cleanup): """Creates a Python Repository and deletes it at test cleanup time.""" @@ -92,3 +110,43 @@ def _gen_python_remote(**kwargs): return gen_object_with_cleanup(python_remote_api_client, body) yield _gen_python_remote + + +@pytest.fixture +def python_repo_with_sync( + python_repo_api_client, python_repo_factory, python_remote_factory, monitor_task +): + """A factory to generate a Python Repository synced with the passed in Remote.""" + def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): + kwargs = {} + if pulp_domain := body.get("pulp_domain"): + kwargs["pulp_domain"] = pulp_domain + remote = remote or python_remote_factory(**kwargs) + repo = repository or python_repo_factory(**body) + sync_body = {"mirror": mirror, "remote": remote.pulp_href} + monitor_task(python_repo_api_client.sync(repo.pulp_href, sync_body).task) + return python_repo_api_client.read(repo.pulp_href) + + yield _gen_python_repo_sync + + +@pytest.fixture +def python_content_summary(python_repo_api_client, python_repo_version_api_client): + """Get a summary of the repository version's content.""" + def _gen_summary(repository_version=None, repository=None, version=None): + if repository_version is None: + repo_href = get_href(repository) + if version: + repo_ver_href = f"{repo_href}versions/{version}/" + else: + repo_ver_href = python_repo_api_client.read(repo_href).latest_version_href + else: + repo_ver_href = get_href(repository_version) + return python_repo_version_api_client.read(repo_ver_href).content_summary + + yield _gen_summary + + +def get_href(item): + """Tries to get the href from the given item, whether it is a string or object.""" + return item if isinstance(item, str) else item.pulp_href