Merge pull request #623 from gerrod3/export

Add import export support of python content

Author: gerrod3

.github/workflows/scripts/install.sh

@@ -70,7 +70,7 @@ VARSYAML
 
 cat >> vars/main.yaml << VARSYAML
 pulp_env: {}
-pulp_settings: {"orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"}
+pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"}
 pulp_scheme: https
 
 pulp_container_tag: "latest"

CHANGES/579.feature

@@ -0,0 +1 @@
+Added import export support of python content.

pulp_python/app/modelresource.py

@@ -0,0 +1,40 @@
+from pulpcore.plugin.importexport import BaseContentResource
+from pulpcore.plugin.modelresources import RepositoryResource
+from pulp_python.app.models import (
+    PythonPackageContent,
+    PythonRepository,
+)
+
+
+class PythonPackageContentResource(BaseContentResource):
+    """
+    Resource for import/export of python_pythonpackagecontent entities.
+    """
+
+    def set_up_queryset(self):
+        """
+        :return: PythonPackageContent specific to a specified repo-version.
+        """
+        return PythonPackageContent.objects.filter(pk__in=self.repo_version.content)
+
+    class Meta:
+        model = PythonPackageContent
+        import_id_fields = model.natural_key_fields()
+
+
+class PythonRepositoryResource(RepositoryResource):
+    """
+    A resource for importing/exporting python repository entities
+    """
+
+    def set_up_queryset(self):
+        """
+        :return: A queryset containing one repository that will be exported.
+        """
+        return PythonRepository.objects.filter(pk=self.repo_version.repository)
+
+    class Meta:
+        model = PythonRepository
+
+
+IMPORT_ORDER = [PythonPackageContentResource, PythonRepositoryResource]

pulp_python/tests/functional/api/test_export_import.py

@@ -0,0 +1,108 @@
+"""
+Tests PulpExporter and PulpExport functionality.
+
+NOTE: assumes ALLOWED_EXPORT_PATHS setting contains "/tmp" - all tests will fail if this is not
+the case.
+"""
+import pytest
+import uuid
+
+from pulp_python.tests.functional.constants import (
+    PYTHON_XS_PROJECT_SPECIFIER, PYTHON_SM_PROJECT_SPECIFIER
+)
+
+
+@pytest.mark.parallel
+def test_export_then_import(
+    python_repo_factory,
+    python_remote_factory,
+    python_repo_api_client,
+    python_repo_version_api_client,
+    exporters_pulp_api_client,
+    exporters_pulp_exports_api_client,
+    importers_pulp_api_client,
+    importers_pulp_imports_api_client,
+    monitor_task,
+    monitor_task_group,
+    gen_object_with_cleanup,
+):
+    """Issue and evaluate a PulpExport (tests both Create and Read)."""
+    # Prepare content
+    remote_a = python_remote_factory(includes=PYTHON_XS_PROJECT_SPECIFIER, policy="immediate")
+    remote_b = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER, policy="immediate")
+    repo_a = python_repo_factory()
+    repo_b = python_repo_factory()
+    sync_response_a = python_repo_api_client.sync(
+        repo_a.pulp_href, {"remote": remote_a.pulp_href}
+    )
+    sync_response_b = python_repo_api_client.sync(
+        repo_b.pulp_href, {"remote": remote_b.pulp_href}
+    )
+    monitor_task(sync_response_a.task)
+    monitor_task(sync_response_b.task)
+
+    repo_ver_a = python_repo_version_api_client.read(f"{repo_a.pulp_href}versions/1/")
+    repo_ver_b = python_repo_version_api_client.read(f"{repo_b.pulp_href}versions/1/")
+
+    # Prepare export
+    exporter = gen_object_with_cleanup(
+        exporters_pulp_api_client,
+        {
+            "name": str(uuid.uuid4()),
+            "path": f"/tmp/{uuid.uuid4()}/",
+            "repositories": [repo.pulp_href for repo in [repo_a, repo_b]],
+        },
+    )
+
+    # Export
+    task = exporters_pulp_exports_api_client.create(exporter.pulp_href, {}).task
+    task = monitor_task(task)
+    assert len(task.created_resources) == 1
+    export = exporters_pulp_exports_api_client.read(task.created_resources[0])
+    assert export is not None
+    assert len(exporter.repositories) == len(export.exported_resources)
+    assert export.output_file_info is not None
+    for an_export_filename in export.output_file_info.keys():
+        assert "//" not in an_export_filename
+    export_filename = next(
+        f for f in export.output_file_info.keys() if f.endswith("tar.gz") or f.endswith("tar")
+    )
+
+    # Prepare import
+    repo_c = python_repo_factory()
+    repo_d = python_repo_factory()
+    repo_mapping = {repo_a.name: repo_c.name, repo_b.name: repo_d.name}
+    importer = gen_object_with_cleanup(
+        importers_pulp_api_client, {"name": str(uuid.uuid4()), "repo_mapping": repo_mapping}
+    )
+
+    # Import
+    import_response = importers_pulp_imports_api_client.create(
+        importer.pulp_href, {"path": export_filename}
+    )
+    monitor_task_group(import_response.task_group)
+    repo_c = python_repo_api_client.read(repo_c.pulp_href)
+    repo_d = python_repo_api_client.read(repo_d.pulp_href)
+    assert repo_c.latest_version_href == f"{repo_c.pulp_href}versions/1/"
+    assert repo_d.latest_version_href == f"{repo_d.pulp_href}versions/1/"
+    repo_ver_c = python_repo_version_api_client.read(f"{repo_c.pulp_href}versions/1/")
+    repo_ver_d = python_repo_version_api_client.read(f"{repo_d.pulp_href}versions/1/")
+    assert (
+        repo_ver_c.content_summary.added["python.python"]["count"]
+        == repo_ver_a.content_summary.present["python.python"]["count"]
+    )
+    assert (
+        repo_ver_d.content_summary.added["python.python"]["count"]
+        == repo_ver_b.content_summary.present["python.python"]["count"]
+    )
+
+    # Import a second time
+    import_response = importers_pulp_imports_api_client.create(
+        importer.pulp_href, {"path": export_filename}
+    )
+    monitor_task_group(import_response.task_group)
+    assert len(importers_pulp_imports_api_client.list(importer.pulp_href).results) == 2
+    for repo in [repo_c, repo_d]:
+        repo = python_repo_api_client.read(repo.pulp_href)
+        # still only one version as pulp won't create a new version if nothing changed
+        assert repo.latest_version_href == f"{repo.pulp_href}versions/1/"

pulp_python/tests/functional/conftest.py

@@ -1,6 +1,7 @@
 import pytest
+import uuid
 
-from pulp_smash.pulp3.utils import gen_distribution, gen_repo
+from pulp_smash.pulp3.utils import gen_distribution
 from pulp_python.tests.functional.utils import gen_python_remote
 
 from pulpcore.client.pulp_python import (
@@ -9,6 +10,7 @@
     DistributionsPypiApi,
     PublicationsPypiApi,
     RepositoriesPythonApi,
+    RepositoriesPythonVersionsApi,
     RemotesPythonApi,
 )
 
@@ -29,6 +31,12 @@ def python_repo_api_client(python_bindings_client):
     return RepositoriesPythonApi(python_bindings_client)
 
 
+@pytest.fixture
+def python_repo_version_api_client(python_bindings_client):
+    """Provides the Python Repository Version API client object."""
+    return RepositoriesPythonVersionsApi(python_bindings_client)
+
+
 @pytest.fixture
 def python_distro_api_client(python_bindings_client):
     """Provides the Python Distribution API client object."""
@@ -56,9 +64,19 @@ def python_publication_api_client(python_bindings_client):
 # Object Generation Fixtures
 
 @pytest.fixture
-def python_repo(python_repo_api_client, gen_object_with_cleanup):
+def python_repo_factory(python_repo_api_client, gen_object_with_cleanup):
+    """A factory to generate a Python Repository with auto-cleanup."""
+    def _gen_python_repo(**kwargs):
+        kwargs.setdefault("name", str(uuid.uuid4()))
+        return gen_object_with_cleanup(python_repo_api_client, kwargs)
+
+    return _gen_python_repo
+
+
+@pytest.fixture
+def python_repo(python_repo_factory):
     """Creates a Python Repository and deletes it at test cleanup time."""
-    return gen_object_with_cleanup(python_repo_api_client, gen_repo())
+    return python_repo_factory()
 
 
 @pytest.fixture

template_config.yml

@@ -55,6 +55,8 @@ pulp_scheme: https
 pulp_settings:
   orphan_protection_time: 0
   pypi_api_hostname: https://pulp:443
+  allowed_export_paths: /tmp
+  allowed_import_paths: /tmp
 pulp_settings_azure: null
 pulp_settings_gcp: null
 pulp_settings_s3: null