From 9d7ff8da0768481f72eab4011302df21b4dda471 Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Fri, 30 Sep 2022 08:53:39 -0400 Subject: [PATCH 1/4] Started refactoring the pulp exporter tests 818 [noissue] --- .../api/from_pulpcore/test_pulpexport.py | 152 +++++++++++------- pulp_file/tests/functional/conftest.py | 8 + 2 files changed, 104 insertions(+), 56 deletions(-) diff --git a/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py b/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py index 33800674..7eac060f 100644 --- a/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py +++ b/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py @@ -4,6 +4,7 @@ NOTE: assumes ALLOWED_EXPORT_PATHS setting contains "/tmp" - all tests will fail if this is not the case. """ +import pytest import unittest from pulp_smash import api, cli, config, utils from pulp_smash.utils import uuid4 @@ -12,6 +13,7 @@ gen_repo, ) +from pulpcore.app import settings from pulpcore.client.pulpcore import ( ApiClient as CoreApiClient, ExportersPulpApi, @@ -39,6 +41,100 @@ NUM_EXPORTERS = 4 +@pytest.fixture +def pulp_exporter_factory(exporters_pulp_api_client, gen_object_with_cleanup): + def _pulp_exporter_factory(repos): + body = { + "name": uuid4(), + "path": "/tmp/{}/".format(uuid4()), + "repositories": [r.pulp_href for r in repos], + } + + exporter = gen_object_with_cleanup(exporters_pulp_api_client, body) + + return exporter, body + + return _pulp_exporter_factory + +def export_factory(exporters_pulp_exports_api_client, tasks_api_client): + def _export_factory(exporter, body={}): + """Create and read back an export for the specified PulpExporter.""" + export_response = exporters_pulp_exports_api_client.create(exporter.pulp_href, body) + export = monitor_task(export_response.task) + assert export is not None + + task = tasks_api_client.read(export_response.task) + resources = task["created_resources"] + assert len(resources) == 1 + + reports = task["progress_reports"] + found_artifacts = False + found_content = False + for r in reports: + assert r["state"] == TASK_STATES.COMPLETED + found_artifacts |= r["code"] == "export.artifacts" + found_content |= r["code"] == "export.repo.version.content" + self.assertTrue(found_artifacts, "No artifacts exported!") + self.assertTrue(found_content, "No content exported!") + export_href = resources[0] + export = self.exports_api.read(export_href) + self.assertIsNotNone(export) + return export + +@pytest.mark.parallel +def test_pulp_exporter_workflow(basic_manifest_path, exporters_pulp_api_client, file_repo_factory, file_fixture_gen_remote_ssl, pulp_exporter_factory): + # Ensure that exporting to /tmp is allowed + allowed_exports = settings.ALLOWED_EXPORT_PATHS + if not allowed_exports or "/tmp" not in allowed_exports: + pytest.skip("Cannot run export-tests unless /tmp is in ALLOWED_EXPORT_PATHS ({}).".format( + allowed_exports + ), + ) + + # Create and sync a number of repositories to be exported + repositories = [] + remotes = [] + for _ in range(3): + repository = file_repo_factory() + repositories.append(repository) + remote = file_fixture_gen_remote_ssl(manifest_path=basic_manifest_path, policy="immediate") + remotes.append(remote) + + # Create an exporter and assert that the returned exporter fields have correct values + exporter, body = pulp_exporter_factory(repositories) + assert exporter.last_export is None + assert body["name"] == exporter.name + assert body["path"] == exporter.path + assert len(repositories) == len(exporter.repositories) + + # Perform a read on the exporter and assert on field values + exporter = exporters_pulp_api_client.read(exporter.pulp_href) + assert exporter.last_export is None + assert body["name"] == exporter.name + assert body["path"] == exporter.path + assert len(repositories) == len(exporter.repositories) + + # Update the path of the exporter + body = {"path": "/tmp/{}".format(uuid4())} + monitor_task(exporters_pulp_api_client.partial_update(exporter.pulp_href, body).task) + exporter = exporters_pulp_api_client.read(exporter.pulp_href) + assert exporter.path == body["path"] + + # List exporters + exporters = exporters_pulp_api_client.list(name=exporter.name).results + assert len(exporters) == 1 + + # Create another exporter and list the two together + another_exporter, _ = pulp_exporter_factory(repositories) + exporters = exporters_pulp_api_client.list(name__in=[exporter.name, another_exporter.name]).results + assert len(exporters) == 2 + + # Delete the exporter and assert it can't be read + exporters_pulp_api_client.delete(another_exporter.pulp_href) + with pytest.raises(ApiException) as exc: + exporters_pulp_api_client.read(another_exporter.pulp_href) + assert exc.value.status == 404 + class BaseExporterCase(unittest.TestCase): """ Base functionality for Exporter and Export test classes @@ -125,62 +221,6 @@ def _create_exporter(self, cleanup=True, use_repos=None): return exporter, body -class PulpExporterTestCase(BaseExporterCase): - """Test PulpExporter CURDL methods.""" - - def test_workflow(self): - self._create() - self._read() - self._partial_update() - self._list() - self._delete() - - def _create(self): - """Create a PulpExporter.""" - (exporter, body) = self._create_exporter() - self.assertIsNone(exporter.last_export) - self.assertEqual(body["name"], exporter.name) - self.assertEqual(body["path"], exporter.path) - self.assertEqual(len(self.repos), len(exporter.repositories)) - - def _read(self): - """Read a created PulpExporter.""" - (exporter_created, body) = self._create_exporter() - exporter_read = self.exporter_api.read(exporter_created.pulp_href) - self.assertEqual(exporter_created.name, exporter_read.name) - self.assertEqual(exporter_created.path, exporter_read.path) - self.assertEqual(len(exporter_created.repositories), len(exporter_read.repositories)) - - def _partial_update(self): - """Update a PulpExporter's path.""" - (exporter_created, body) = self._create_exporter() - body = {"path": "/tmp/{}".format(uuid4())} - result = self.exporter_api.partial_update(exporter_created.pulp_href, body) - monitor_task(result.task) - exporter_read = self.exporter_api.read(exporter_created.pulp_href) - self.assertNotEqual(exporter_created.path, exporter_read.path) - self.assertEqual(body["path"], exporter_read.path) - - def _list(self): - """Show a set of created PulpExporters.""" - starting_exporters = self.exporter_api.list().results - for x in range(NUM_EXPORTERS): - self._create_exporter() - ending_exporters = self.exporter_api.list().results - self.assertEqual(NUM_EXPORTERS, len(ending_exporters) - len(starting_exporters)) - - def _delete(self): - """Delete a pulpExporter.""" - (exporter_created, body) = self._create_exporter(cleanup=False) - delete_exporter(exporter_created) - try: - self.exporter_api.read(exporter_created.pulp_href) - except ApiException as ae: - self.assertEqual(404, ae.status) - return - self.fail("Found a deleted exporter!") - - class PulpExportTestCase(BaseExporterCase): """Test PulpExport CRDL methods (Update is not allowed).""" diff --git a/pulp_file/tests/functional/conftest.py b/pulp_file/tests/functional/conftest.py index aca586b3..13698882 100644 --- a/pulp_file/tests/functional/conftest.py +++ b/pulp_file/tests/functional/conftest.py @@ -99,6 +99,14 @@ def file_repo(file_repo_api_client, gen_object_with_cleanup): return gen_object_with_cleanup(file_repo_api_client, gen_repo()) +@pytest.fixture +def file_repo_factory(file_repo_api_client, gen_object_with_cleanup): + def _file_repo_factory(): + return gen_object_with_cleanup(file_repo_api_client, gen_repo()) + + return _file_repo_factory + + @pytest.fixture def file_repo_with_auto_publish(file_repo_api_client, gen_object_with_cleanup): return gen_object_with_cleanup(file_repo_api_client, gen_repo(autopublish=True)) From 7dc8e9ebdf0d56051b92d9e17c7910529b98d758 Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Fri, 30 Sep 2022 08:57:42 -0400 Subject: [PATCH 2/4] [noissue] --- .github/workflows/ci.yml | 3 ++- .github/workflows/scripts/script.sh | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec7702d9..7134a232 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,7 +179,8 @@ jobs: run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: SECRETS_CONTEXT: ${{ toJson(secrets) }} - + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 - name: Script run: .github/workflows/scripts/script.sh diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 6083b3bc..6115364c 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -88,6 +88,10 @@ cd $REPO_ROOT CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt | tee -a "$CERTIFI" > /dev/null" +cmd_prefix bash -c "mkdir -p /var/lib/pulp/media/artifact" +cmd_prefix bash -c "chown pulp:pulp /var/lib/pulp/media/artifact" +cmd_prefix bash -c "chmod 775 /var/lib/pulp/media/artifact" + # check for any uncommitted migrations echo "Checking for uncommitted migrations..." cmd_prefix bash -c "django-admin makemigrations --check --dry-run" From 9cb575feb989baa69092cafed6c5635492ffce42 Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Fri, 30 Sep 2022 09:03:17 -0400 Subject: [PATCH 3/4] [noissue] --- .../api/from_pulpcore/test_pulpexport.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py b/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py index 7eac060f..e4d05e73 100644 --- a/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py +++ b/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py @@ -56,6 +56,7 @@ def _pulp_exporter_factory(repos): return _pulp_exporter_factory + def export_factory(exporters_pulp_exports_api_client, tasks_api_client): def _export_factory(exporter, body={}): """Create and read back an export for the specified PulpExporter.""" @@ -81,12 +82,20 @@ def _export_factory(exporter, body={}): self.assertIsNotNone(export) return export + @pytest.mark.parallel -def test_pulp_exporter_workflow(basic_manifest_path, exporters_pulp_api_client, file_repo_factory, file_fixture_gen_remote_ssl, pulp_exporter_factory): +def test_pulp_exporter_workflow( + basic_manifest_path, + exporters_pulp_api_client, + file_repo_factory, + file_fixture_gen_remote_ssl, + pulp_exporter_factory, +): # Ensure that exporting to /tmp is allowed allowed_exports = settings.ALLOWED_EXPORT_PATHS if not allowed_exports or "/tmp" not in allowed_exports: - pytest.skip("Cannot run export-tests unless /tmp is in ALLOWED_EXPORT_PATHS ({}).".format( + pytest.skip( + "Cannot run export-tests unless /tmp is in ALLOWED_EXPORT_PATHS ({}).".format( allowed_exports ), ) @@ -126,7 +135,9 @@ def test_pulp_exporter_workflow(basic_manifest_path, exporters_pulp_api_client, # Create another exporter and list the two together another_exporter, _ = pulp_exporter_factory(repositories) - exporters = exporters_pulp_api_client.list(name__in=[exporter.name, another_exporter.name]).results + exporters = exporters_pulp_api_client.list( + name__in=[exporter.name, another_exporter.name] + ).results assert len(exporters) == 2 # Delete the exporter and assert it can't be read @@ -135,6 +146,7 @@ def test_pulp_exporter_workflow(basic_manifest_path, exporters_pulp_api_client, exporters_pulp_api_client.read(another_exporter.pulp_href) assert exc.value.status == 404 + class BaseExporterCase(unittest.TestCase): """ Base functionality for Exporter and Export test classes From cff2003f5f5bd39180866b0d82927bc9cf8b372a Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Fri, 30 Sep 2022 09:10:54 -0400 Subject: [PATCH 4/4] [noissue] --- .../api/from_pulpcore/test_pulpexport.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py b/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py index e4d05e73..831367be 100644 --- a/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py +++ b/pulp_file/tests/functional/api/from_pulpcore/test_pulpexport.py @@ -57,30 +57,30 @@ def _pulp_exporter_factory(repos): return _pulp_exporter_factory -def export_factory(exporters_pulp_exports_api_client, tasks_api_client): - def _export_factory(exporter, body={}): - """Create and read back an export for the specified PulpExporter.""" - export_response = exporters_pulp_exports_api_client.create(exporter.pulp_href, body) - export = monitor_task(export_response.task) - assert export is not None - - task = tasks_api_client.read(export_response.task) - resources = task["created_resources"] - assert len(resources) == 1 - - reports = task["progress_reports"] - found_artifacts = False - found_content = False - for r in reports: - assert r["state"] == TASK_STATES.COMPLETED - found_artifacts |= r["code"] == "export.artifacts" - found_content |= r["code"] == "export.repo.version.content" - self.assertTrue(found_artifacts, "No artifacts exported!") - self.assertTrue(found_content, "No content exported!") - export_href = resources[0] - export = self.exports_api.read(export_href) - self.assertIsNotNone(export) - return export +# def export_factory(exporters_pulp_exports_api_client, tasks_api_client): +# def _export_factory(exporter, body={}): +# """Create and read back an export for the specified PulpExporter.""" +# export_response = exporters_pulp_exports_api_client.create(exporter.pulp_href, body) +# export = monitor_task(export_response.task) +# assert export is not None +# +# task = tasks_api_client.read(export_response.task) +# resources = task["created_resources"] +# assert len(resources) == 1 +# +# reports = task["progress_reports"] +# found_artifacts = False +# found_content = False +# for r in reports: +# assert r["state"] == TASK_STATES.COMPLETED +# found_artifacts |= r["code"] == "export.artifacts" +# found_content |= r["code"] == "export.repo.version.content" +# self.assertTrue(found_artifacts, "No artifacts exported!") +# self.assertTrue(found_content, "No content exported!") +# export_href = resources[0] +# export = self.exports_api.read(export_href) +# self.assertIsNotNone(export) +# return export @pytest.mark.parallel