diff --git a/.ci/scripts/schema.py b/.ci/scripts/schema.py index 9f56caa6..91191c5e 100644 --- a/.ci/scripts/schema.py +++ b/.ci/scripts/schema.py @@ -7,6 +7,7 @@ But some pulp paths start with curly brackets e.g. {artifact_href} This script modifies drf-spectacular schema validation to accept slashes and curly brackets. """ + import json from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9dddcbdd..4f7181d8 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -43,6 +43,12 @@ jobs: bump-my-version bump --dry-run release bump-my-version show-bump + # run black separately from flake8 to get a diff + - name: "Run black" + run: | + black --version + black --check --diff . + # Lint code. - name: "Run flake8" run: | diff --git a/lint_requirements.txt b/lint_requirements.txt index a2936209..3c2947a0 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,8 +5,10 @@ # # For more info visit https://github.com/pulp/plugin_template +black==24.3.0 bump-my-version check-manifest flake8 +flake8-black packaging yamllint diff --git a/pulp_python/__init__.py b/pulp_python/__init__.py index 5d57d16b..50f87cf3 100644 --- a/pulp_python/__init__.py +++ b/pulp_python/__init__.py @@ -1 +1 @@ -default_app_config = 'pulp_python.app.PulpPythonPluginAppConfig' +default_app_config = "pulp_python.app.PulpPythonPluginAppConfig" diff --git a/pulp_python/app/management/commands/repair-python-metadata.py b/pulp_python/app/management/commands/repair-python-metadata.py index 4582f328..da69414c 100644 --- a/pulp_python/app/management/commands/repair-python-metadata.py +++ b/pulp_python/app/management/commands/repair-python-metadata.py @@ -23,7 +23,7 @@ def repair_metadata(content): batch = [] set_of_update_fields = set() total_repaired = 0 - for package in immediate_content.prefetch_related('_artifacts').iterator(chunk_size=1000): + for package in immediate_content.prefetch_related("_artifacts").iterator(chunk_size=1000): new_data = artifact_to_python_content_data( package.filename, package._artifacts.get(), package.pulp_domain ) @@ -55,7 +55,7 @@ def href_prn_list_handler(value): (?:{settings.API_ROOT}(?:[-_a-zA-Z0-9]+/)?api/v3/repositories/python/python/[-a-f0-9]+/) |(?:prn:python\.pythonrepository:[-a-f0-9]+) """, - re.VERBOSE + re.VERBOSE, ) values = [] for v in value.split(","): diff --git a/pulp_python/app/migrations/0001_initial.py b/pulp_python/app/migrations/0001_initial.py index b67df3d9..5534ed2d 100644 --- a/pulp_python/app/migrations/0001_initial.py +++ b/pulp_python/app/migrations/0001_initial.py @@ -10,87 +10,164 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('core', '0091_systemid'), + ("core", "0091_systemid"), ] operations = [ migrations.CreateModel( - name='PythonPublication', + name="PythonPublication", fields=[ - ('publication_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpublication', serialize=False, to='core.publication')), + ( + "publication_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythonpublication", + serialize=False, + to="core.publication", + ), + ), ], options={ - 'default_related_name': '%(app_label)s_%(model_name)s', + "default_related_name": "%(app_label)s_%(model_name)s", }, - bases=('core.publication',), + bases=("core.publication",), ), migrations.CreateModel( - name='PythonRemote', + name="PythonRemote", fields=[ - ('remote_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonremote', serialize=False, to='core.remote')), - ('prereleases', models.BooleanField(default=False)), - ('includes', django.contrib.postgres.fields.jsonb.JSONField(default=list)), - ('excludes', django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ( + "remote_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythonremote", + serialize=False, + to="core.remote", + ), + ), + ("prereleases", models.BooleanField(default=False)), + ("includes", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("excludes", django.contrib.postgres.fields.jsonb.JSONField(default=list)), ], options={ - 'default_related_name': '%(app_label)s_%(model_name)s', + "default_related_name": "%(app_label)s_%(model_name)s", }, - bases=('core.remote',), + bases=("core.remote",), ), migrations.CreateModel( - name='PythonRepository', + name="PythonRepository", fields=[ - ('repository_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonrepository', serialize=False, to='core.repository')), + ( + "repository_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythonrepository", + serialize=False, + to="core.repository", + ), + ), ], options={ - 'default_related_name': '%(app_label)s_%(model_name)s', + "default_related_name": "%(app_label)s_%(model_name)s", }, - bases=('core.repository',), + bases=("core.repository",), ), migrations.CreateModel( - name='PythonPackageContent', + name="PythonPackageContent", fields=[ - ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythonpackagecontent', serialize=False, to='core.content')), - ('filename', models.TextField(db_index=True, unique=True)), - ('packagetype', models.TextField(choices=[('bdist_dmg', 'bdist_dmg'), ('bdist_dumb', 'bdist_dumb'), ('bdist_egg', 'bdist_egg'), ('bdist_msi', 'bdist_msi'), ('bdist_rpm', 'bdist_rpm'), ('bdist_wheel', 'bdist_wheel'), ('bdist_wininst', 'bdist_wininst'), ('sdist', 'sdist')])), - ('name', models.TextField()), - ('version', models.TextField()), - ('metadata_version', models.TextField()), - ('summary', models.TextField()), - ('description', models.TextField()), - ('keywords', models.TextField()), - ('home_page', models.TextField()), - ('download_url', models.TextField()), - ('author', models.TextField()), - ('author_email', models.TextField()), - ('maintainer', models.TextField()), - ('maintainer_email', models.TextField()), - ('license', models.TextField()), - ('requires_python', models.TextField()), - ('project_url', models.TextField()), - ('platform', models.TextField()), - ('supported_platform', models.TextField()), - ('requires_dist', django.contrib.postgres.fields.jsonb.JSONField(default=list)), - ('provides_dist', django.contrib.postgres.fields.jsonb.JSONField(default=list)), - ('obsoletes_dist', django.contrib.postgres.fields.jsonb.JSONField(default=list)), - ('requires_external', django.contrib.postgres.fields.jsonb.JSONField(default=list)), - ('classifiers', django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ( + "content_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythonpackagecontent", + serialize=False, + to="core.content", + ), + ), + ("filename", models.TextField(db_index=True, unique=True)), + ( + "packagetype", + models.TextField( + choices=[ + ("bdist_dmg", "bdist_dmg"), + ("bdist_dumb", "bdist_dumb"), + ("bdist_egg", "bdist_egg"), + ("bdist_msi", "bdist_msi"), + ("bdist_rpm", "bdist_rpm"), + ("bdist_wheel", "bdist_wheel"), + ("bdist_wininst", "bdist_wininst"), + ("sdist", "sdist"), + ] + ), + ), + ("name", models.TextField()), + ("version", models.TextField()), + ("metadata_version", models.TextField()), + ("summary", models.TextField()), + ("description", models.TextField()), + ("keywords", models.TextField()), + ("home_page", models.TextField()), + ("download_url", models.TextField()), + ("author", models.TextField()), + ("author_email", models.TextField()), + ("maintainer", models.TextField()), + ("maintainer_email", models.TextField()), + ("license", models.TextField()), + ("requires_python", models.TextField()), + ("project_url", models.TextField()), + ("platform", models.TextField()), + ("supported_platform", models.TextField()), + ("requires_dist", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("provides_dist", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("obsoletes_dist", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("requires_external", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("classifiers", django.contrib.postgres.fields.jsonb.JSONField(default=list)), ], options={ - 'default_related_name': '%(app_label)s_%(model_name)s', - 'unique_together': {('filename',)}, + "default_related_name": "%(app_label)s_%(model_name)s", + "unique_together": {("filename",)}, }, - bases=('core.content',), + bases=("core.content",), ), migrations.CreateModel( - name='PythonDistribution', + name="PythonDistribution", fields=[ - ('basedistribution_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythondistribution', serialize=False, to='core.basedistribution')), - ('publication', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='python_pythondistribution', to='core.publication')), + ( + "basedistribution_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythondistribution", + serialize=False, + to="core.basedistribution", + ), + ), + ( + "publication", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="python_pythondistribution", + to="core.publication", + ), + ), ], options={ - 'default_related_name': '%(app_label)s_%(model_name)s', + "default_related_name": "%(app_label)s_%(model_name)s", }, - bases=('core.basedistribution',), + bases=("core.basedistribution",), ), ] diff --git a/pulp_python/app/migrations/0002_pythonpackagecontent_python_version.py b/pulp_python/app/migrations/0002_pythonpackagecontent_python_version.py index 5661deb1..b0601b55 100644 --- a/pulp_python/app/migrations/0002_pythonpackagecontent_python_version.py +++ b/pulp_python/app/migrations/0002_pythonpackagecontent_python_version.py @@ -6,14 +6,14 @@ class Migration(migrations.Migration): dependencies = [ - ('python', '0001_initial'), + ("python", "0001_initial"), ] operations = [ migrations.AddField( - model_name='pythonpackagecontent', - name='python_version', - field=models.TextField(default=''), + model_name="pythonpackagecontent", + name="python_version", + field=models.TextField(default=""), preserve_default=False, ), ] diff --git a/pulp_python/app/migrations/0003_new_sync_filters.py b/pulp_python/app/migrations/0003_new_sync_filters.py index bcc7b644..61c9b17f 100644 --- a/pulp_python/app/migrations/0003_new_sync_filters.py +++ b/pulp_python/app/migrations/0003_new_sync_filters.py @@ -7,23 +7,47 @@ class Migration(migrations.Migration): dependencies = [ - ('python', '0002_pythonpackagecontent_python_version'), + ("python", "0002_pythonpackagecontent_python_version"), ] operations = [ migrations.AddField( - model_name='pythonremote', - name='exclude_platforms', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=10), choices=[('windows', 'windows'), ('macos', 'macos'), ('freebsd', 'freebsd'), ('linux', 'linux')], default=list, size=None), + model_name="pythonremote", + name="exclude_platforms", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(blank=True, max_length=10), + choices=[ + ("windows", "windows"), + ("macos", "macos"), + ("freebsd", "freebsd"), + ("linux", "linux"), + ], + default=list, + size=None, + ), ), migrations.AddField( - model_name='pythonremote', - name='keep_latest_packages', + model_name="pythonremote", + name="keep_latest_packages", field=models.IntegerField(default=0), ), migrations.AddField( - model_name='pythonremote', - name='package_types', - field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=15), choices=[('bdist_dmg', 'bdist_dmg'), ('bdist_dumb', 'bdist_dumb'), ('bdist_egg', 'bdist_egg'), ('bdist_msi', 'bdist_msi'), ('bdist_rpm', 'bdist_rpm'), ('bdist_wheel', 'bdist_wheel'), ('bdist_wininst', 'bdist_wininst'), ('sdist', 'sdist')], default=list, size=None), + model_name="pythonremote", + name="package_types", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(blank=True, max_length=15), + choices=[ + ("bdist_dmg", "bdist_dmg"), + ("bdist_dumb", "bdist_dumb"), + ("bdist_egg", "bdist_egg"), + ("bdist_msi", "bdist_msi"), + ("bdist_rpm", "bdist_rpm"), + ("bdist_wheel", "bdist_wheel"), + ("bdist_wininst", "bdist_wininst"), + ("sdist", "sdist"), + ], + default=list, + size=None, + ), ), ] diff --git a/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py b/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py index 07f443f3..dd658c1c 100644 --- a/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py +++ b/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py @@ -5,9 +5,9 @@ def migrate_data_from_old_model_to_new_model_up(apps, schema_editor): - """ Move objects from PythonDistribution to NewPythonDistribution.""" - PythonDistribution = apps.get_model('python', 'PythonDistribution') - NewPythonDistribution = apps.get_model('python', 'NewPythonDistribution') + """Move objects from PythonDistribution to NewPythonDistribution.""" + PythonDistribution = apps.get_model("python", "PythonDistribution") + NewPythonDistribution = apps.get_model("python", "NewPythonDistribution") for python_distribution in PythonDistribution.objects.all(): with transaction.atomic(): NewPythonDistribution( @@ -19,15 +19,15 @@ def migrate_data_from_old_model_to_new_model_up(apps, schema_editor): base_path=python_distribution.base_path, content_guard=python_distribution.content_guard, remote=python_distribution.remote, - publication=python_distribution.publication + publication=python_distribution.publication, ).save() python_distribution.delete() def migrate_data_from_old_model_to_new_model_down(apps, schema_editor): - """ Move objects from NewPythonDistribution to PythonDistribution.""" - PythonDistribution = apps.get_model('python', 'PythonDistribution') - NewPythonDistribution = apps.get_model('python', 'NewPythonDistribution') + """Move objects from NewPythonDistribution to PythonDistribution.""" + PythonDistribution = apps.get_model("python", "PythonDistribution") + NewPythonDistribution = apps.get_model("python", "NewPythonDistribution") for python_distribution in NewPythonDistribution.objects.all(): with transaction.atomic(): PythonDistribution( @@ -39,7 +39,7 @@ def migrate_data_from_old_model_to_new_model_down(apps, schema_editor): base_path=python_distribution.base_path, content_guard=python_distribution.content_guard, remote=python_distribution.remote, - publication=python_distribution.publication + publication=python_distribution.publication, ).save() python_distribution.delete() @@ -48,29 +48,40 @@ class Migration(migrations.Migration): atomic = False dependencies = [ - ('python', '0003_new_sync_filters'), + ("python", "0003_new_sync_filters"), ] operations = [ migrations.CreateModel( - name='NewPythonDistribution', + name="NewPythonDistribution", fields=[ - ('distribution_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='python_pythondistribution', serialize=False, to='core.Distribution')), + ( + "distribution_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythondistribution", + serialize=False, + to="core.Distribution", + ), + ), ], options={ - 'default_related_name': '%(app_label)s_%(model_name)s', + "default_related_name": "%(app_label)s_%(model_name)s", }, - bases=('core.distribution',), + bases=("core.distribution",), ), migrations.RunPython( code=migrate_data_from_old_model_to_new_model_up, reverse_code=migrate_data_from_old_model_to_new_model_down, ), migrations.DeleteModel( - name='PythonDistribution', + name="PythonDistribution", ), migrations.RenameModel( - old_name='NewPythonDistribution', - new_name='PythonDistribution', + old_name="NewPythonDistribution", + new_name="PythonDistribution", ), ] diff --git a/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py b/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py index e8d7df7e..585cc4ec 100644 --- a/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py +++ b/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py @@ -5,8 +5,8 @@ def add_sha256_to_current_models(apps, schema_editor): """Adds the sha256 to current PythonPackageContent models.""" - PythonPackageContent = apps.get_model('python', 'PythonPackageContent') - RemoteArtifact = apps.get_model('core', 'RemoteArtifact') + PythonPackageContent = apps.get_model("python", "PythonPackageContent") + RemoteArtifact = apps.get_model("core", "RemoteArtifact") package_bulk = [] for python_package in PythonPackageContent.objects.only("pk", "sha256").iterator(): content_artifact = python_package.contentartifact_set.first() @@ -18,24 +18,34 @@ def add_sha256_to_current_models(apps, schema_editor): package_bulk.append(python_package) if len(package_bulk) == 100000: with transaction.atomic(): - PythonPackageContent.objects.bulk_update(package_bulk, ["sha256",]) + PythonPackageContent.objects.bulk_update( + package_bulk, + [ + "sha256", + ], + ) package_bulk = [] with transaction.atomic(): - PythonPackageContent.objects.bulk_update(package_bulk, ["sha256",]) + PythonPackageContent.objects.bulk_update( + package_bulk, + [ + "sha256", + ], + ) class Migration(migrations.Migration): dependencies = [ - ('python', '0004_DATA_swap_distribution_model'), + ("python", "0004_DATA_swap_distribution_model"), ] operations = [ migrations.AddField( - model_name='pythonpackagecontent', - name='sha256', - field=models.CharField(max_length=64, default=''), + model_name="pythonpackagecontent", + name="sha256", + field=models.CharField(max_length=64, default=""), preserve_default=False, ), - migrations.RunPython(add_sha256_to_current_models, migrations.RunPython.noop) + migrations.RunPython(add_sha256_to_current_models, migrations.RunPython.noop), ] diff --git a/pulp_python/app/migrations/0006_pythonrepository_autopublish.py b/pulp_python/app/migrations/0006_pythonrepository_autopublish.py index 58315eff..bf914a93 100644 --- a/pulp_python/app/migrations/0006_pythonrepository_autopublish.py +++ b/pulp_python/app/migrations/0006_pythonrepository_autopublish.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('python', '0005_pythonpackagecontent_sha256'), + ("python", "0005_pythonpackagecontent_sha256"), ] operations = [ migrations.AddField( - model_name='pythonrepository', - name='autopublish', + model_name="pythonrepository", + name="autopublish", field=models.BooleanField(default=False), ), ] diff --git a/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py b/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py index 8a817cf6..9216937c 100644 --- a/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py +++ b/pulp_python/app/migrations/0007_pythonpackagecontent_mv-2-1.py @@ -7,19 +7,19 @@ class Migration(migrations.Migration): dependencies = [ - ('python', '0006_pythonrepository_autopublish'), + ("python", "0006_pythonrepository_autopublish"), ] operations = [ migrations.AddField( - model_name='pythonpackagecontent', - name='description_content_type', - field=models.TextField(default=''), + model_name="pythonpackagecontent", + name="description_content_type", + field=models.TextField(default=""), preserve_default=False, ), migrations.AddField( - model_name='pythonpackagecontent', - name='project_urls', + model_name="pythonpackagecontent", + name="project_urls", field=django.contrib.postgres.fields.jsonb.JSONField(default=dict), ), ] diff --git a/pulp_python/app/migrations/0008_pythonpackagecontent_unique_sha256.py b/pulp_python/app/migrations/0008_pythonpackagecontent_unique_sha256.py index 4c9e736b..3540f907 100644 --- a/pulp_python/app/migrations/0008_pythonpackagecontent_unique_sha256.py +++ b/pulp_python/app/migrations/0008_pythonpackagecontent_unique_sha256.py @@ -6,25 +6,25 @@ class Migration(migrations.Migration): dependencies = [ - ('python', '0007_pythonpackagecontent_mv-2-1'), + ("python", "0007_pythonpackagecontent_mv-2-1"), ] operations = [ migrations.AlterField( - model_name='pythonpackagecontent', - name='filename', + model_name="pythonpackagecontent", + name="filename", field=models.TextField(db_index=True), ), migrations.AlterField( - model_name='pythonpackagecontent', - name='sha256', + model_name="pythonpackagecontent", + name="sha256", field=models.CharField(db_index=True, max_length=64, unique=True), ), migrations.SeparateDatabaseAndState( state_operations=[ migrations.AlterUniqueTogether( - name='pythonpackagecontent', - unique_together={('sha256',)}, + name="pythonpackagecontent", + unique_together={("sha256",)}, ), ], ), diff --git a/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py b/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py index de123d2c..75289480 100644 --- a/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py +++ b/pulp_python/app/migrations/0009_pythondistribution_allow_uploads.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('python', '0008_pythonpackagecontent_unique_sha256'), + ("python", "0008_pythonpackagecontent_unique_sha256"), ] operations = [ migrations.AddField( - model_name='pythondistribution', - name='allow_uploads', + model_name="pythondistribution", + name="allow_uploads", field=models.BooleanField(default=True), ), ] diff --git a/pulp_python/app/migrations/0010_update_json_field.py b/pulp_python/app/migrations/0010_update_json_field.py index b333008c..3fb65746 100644 --- a/pulp_python/app/migrations/0010_update_json_field.py +++ b/pulp_python/app/migrations/0010_update_json_field.py @@ -6,48 +6,48 @@ class Migration(migrations.Migration): dependencies = [ - ('python', '0009_pythondistribution_allow_uploads'), + ("python", "0009_pythondistribution_allow_uploads"), ] operations = [ migrations.AlterField( - model_name='pythonpackagecontent', - name='classifiers', + model_name="pythonpackagecontent", + name="classifiers", field=models.JSONField(default=list), ), migrations.AlterField( - model_name='pythonpackagecontent', - name='obsoletes_dist', + model_name="pythonpackagecontent", + name="obsoletes_dist", field=models.JSONField(default=list), ), migrations.AlterField( - model_name='pythonpackagecontent', - name='project_urls', + model_name="pythonpackagecontent", + name="project_urls", field=models.JSONField(default=dict), ), migrations.AlterField( - model_name='pythonpackagecontent', - name='provides_dist', + model_name="pythonpackagecontent", + name="provides_dist", field=models.JSONField(default=list), ), migrations.AlterField( - model_name='pythonpackagecontent', - name='requires_dist', + model_name="pythonpackagecontent", + name="requires_dist", field=models.JSONField(default=list), ), migrations.AlterField( - model_name='pythonpackagecontent', - name='requires_external', + model_name="pythonpackagecontent", + name="requires_external", field=models.JSONField(default=list), ), migrations.AlterField( - model_name='pythonremote', - name='excludes', + model_name="pythonremote", + name="excludes", field=models.JSONField(default=list), ), migrations.AlterField( - model_name='pythonremote', - name='includes', + model_name="pythonremote", + name="includes", field=models.JSONField(default=list), ), ] diff --git a/pulp_python/app/migrations/0011_alter_pythondistribution_distribution_ptr_and_more.py b/pulp_python/app/migrations/0011_alter_pythondistribution_distribution_ptr_and_more.py index 06c4ff8f..57e91ad7 100644 --- a/pulp_python/app/migrations/0011_alter_pythondistribution_distribution_ptr_and_more.py +++ b/pulp_python/app/migrations/0011_alter_pythondistribution_distribution_ptr_and_more.py @@ -7,34 +7,69 @@ class Migration(migrations.Migration): dependencies = [ - ('core', '0106_alter_artifactdistribution_distribution_ptr_and_more'), - ('python', '0010_update_json_field'), + ("core", "0106_alter_artifactdistribution_distribution_ptr_and_more"), + ("python", "0010_update_json_field"), ] operations = [ migrations.AlterField( - model_name='pythondistribution', - name='distribution_ptr', - field=models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.distribution'), + model_name="pythondistribution", + name="distribution_ptr", + field=models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="core.distribution", + ), ), migrations.AlterField( - model_name='pythonpackagecontent', - name='content_ptr', - field=models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.content'), + model_name="pythonpackagecontent", + name="content_ptr", + field=models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="core.content", + ), ), migrations.AlterField( - model_name='pythonpublication', - name='publication_ptr', - field=models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.publication'), + model_name="pythonpublication", + name="publication_ptr", + field=models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="core.publication", + ), ), migrations.AlterField( - model_name='pythonremote', - name='remote_ptr', - field=models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.remote'), + model_name="pythonremote", + name="remote_ptr", + field=models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="core.remote", + ), ), migrations.AlterField( - model_name='pythonrepository', - name='repository_ptr', - field=models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.repository'), + model_name="pythonrepository", + name="repository_ptr", + field=models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="core.repository", + ), ), ] diff --git a/pulp_python/app/migrations/0013_add_rbac_permissions.py b/pulp_python/app/migrations/0013_add_rbac_permissions.py index 9e407f52..d09aa267 100644 --- a/pulp_python/app/migrations/0013_add_rbac_permissions.py +++ b/pulp_python/app/migrations/0013_add_rbac_permissions.py @@ -6,24 +6,47 @@ class Migration(migrations.Migration): dependencies = [ - ('python', '0012_add_domain'), + ("python", "0012_add_domain"), ] operations = [ migrations.AlterModelOptions( - name='pythondistribution', - options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('manage_roles_pythondistribution', 'Can manage roles on python distributions')]}, + name="pythondistribution", + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + "permissions": [ + ("manage_roles_pythondistribution", "Can manage roles on python distributions") + ], + }, ), migrations.AlterModelOptions( - name='pythonpublication', - options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('manage_roles_pythonpublication', 'Can manage roles on python publications')]}, + name="pythonpublication", + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + "permissions": [ + ("manage_roles_pythonpublication", "Can manage roles on python publications") + ], + }, ), migrations.AlterModelOptions( - name='pythonremote', - options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('manage_roles_pythonremote', 'Can manage roles on python remotes')]}, + name="pythonremote", + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + "permissions": [ + ("manage_roles_pythonremote", "Can manage roles on python remotes") + ], + }, ), migrations.AlterModelOptions( - name='pythonrepository', - options={'default_related_name': '%(app_label)s_%(model_name)s', 'permissions': [('sync_pythonrepository', 'Can start a sync task'), ('modify_pythonrepository', 'Can modify content of the repository'), ('manage_roles_pythonrepository', 'Can manage roles on python repositories'), ('repair_pythonrepository', 'Can repair repository versions')]}, + name="pythonrepository", + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + "permissions": [ + ("sync_pythonrepository", "Can start a sync task"), + ("modify_pythonrepository", "Can modify content of the repository"), + ("manage_roles_pythonrepository", "Can manage roles on python repositories"), + ("repair_pythonrepository", "Can repair repository versions"), + ], + }, ), ] diff --git a/pulp_python/app/migrations/0014_pythonpackagecontent_dynamic_and_more.py b/pulp_python/app/migrations/0014_pythonpackagecontent_dynamic_and_more.py index f3773c7b..8f42b4e2 100644 --- a/pulp_python/app/migrations/0014_pythonpackagecontent_dynamic_and_more.py +++ b/pulp_python/app/migrations/0014_pythonpackagecontent_dynamic_and_more.py @@ -6,29 +6,29 @@ class Migration(migrations.Migration): dependencies = [ - ('python', '0013_add_rbac_permissions'), + ("python", "0013_add_rbac_permissions"), ] operations = [ migrations.AddField( - model_name='pythonpackagecontent', - name='dynamic', + model_name="pythonpackagecontent", + name="dynamic", field=models.JSONField(default=list), ), migrations.AddField( - model_name='pythonpackagecontent', - name='license_expression', - field=models.TextField(default=''), + model_name="pythonpackagecontent", + name="license_expression", + field=models.TextField(default=""), preserve_default=False, ), migrations.AddField( - model_name='pythonpackagecontent', - name='license_file', + model_name="pythonpackagecontent", + name="license_file", field=models.JSONField(default=list), ), migrations.AddField( - model_name='pythonpackagecontent', - name='provides_extras', + model_name="pythonpackagecontent", + name="provides_extras", field=models.JSONField(default=list), ), ] diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index adfa0f99..8535233b 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -150,6 +150,7 @@ class PythonPackageContent(Content): https://packaging.python.org/en/latest/specifications/source-distribution-format/ https://packaging.python.org/en/latest/specifications/binary-distribution-format/ """ + # Core metadata # Version 1.0 author = models.TextField() @@ -195,9 +196,7 @@ class PythonPackageContent(Content): # From pulpcore PROTECTED_FROM_RECLAIM = False TYPE = "python" - _pulp_domain = models.ForeignKey( - "core.Domain", default=get_domain_pk, on_delete=models.PROTECT - ) + _pulp_domain = models.ForeignKey("core.Domain", default=get_domain_pk, on_delete=models.PROTECT) name.register_lookup(NormalizeName) repo_key_fields = ("filename",) diff --git a/pulp_python/app/pypi/serializers.py b/pulp_python/app/pypi/serializers.py index dc99fc80..1692072b 100644 --- a/pulp_python/app/pypi/serializers.py +++ b/pulp_python/app/pypi/serializers.py @@ -46,7 +46,7 @@ class PackageUploadSerializer(serializers.Serializer): action = serializers.CharField( help_text=_("Defaults to `file_upload`, don't change it or request will fail!"), default="file_upload", - source=":action" + source=":action", ) sha256_digest = serializers.CharField( help_text=_("SHA256 of package to validate upload integrity."), @@ -59,17 +59,17 @@ def validate(self, data): """Validates the request.""" action = data.get(":action") if action != "file_upload": - raise serializers.ValidationError( - _("We do not support the :action {}").format(action) - ) + raise serializers.ValidationError(_("We do not support the :action {}").format(action)) file = data.get("content") for ext, packagetype in DIST_EXTENSIONS.items(): if file.name.endswith(ext): break else: - raise serializers.ValidationError(_( - "Extension on {} is not a valid python extension " - "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(file.name) + raise serializers.ValidationError( + _( + "Extension on {} is not a valid python extension " + "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)" + ).format(file.name) ) sha256 = data.get("sha256_digest") digests = {"sha256": sha256} if sha256 else None diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index 0174c937..79033d4e 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -163,10 +163,15 @@ def upload(self, request, path): if settings.PYTHON_GROUP_UPLOADS: return self.upload_package_group(repo, artifact, filename, request.session) - result = dispatch(tasks.upload, exclusive_resources=[artifact, repo], - kwargs={"artifact_sha256": artifact.sha256, - "filename": filename, - "repository_pk": str(repo.pk)}) + result = dispatch( + tasks.upload, + exclusive_resources=[artifact, repo], + kwargs={ + "artifact_sha256": artifact.sha256, + "filename": filename, + "repository_pk": str(repo.pk), + }, + ) return OperationPostponedResponse(result, request) def upload_package_group(self, repo, artifact, filename, session): @@ -180,10 +185,10 @@ def upload_package_group(self, repo, artifact, filename, session): try: with transaction.atomic(): sq.first() - current_start = datetime.fromisoformat(session['start']) + current_start = datetime.fromisoformat(session["start"]) if current_start >= datetime.now(tz=timezone.utc): - session['artifacts'].append((str(artifact.sha256), filename)) - session['start'] = str(start_time) + session["artifacts"].append((str(artifact.sha256), filename)) + session["start"] = str(start_time) session.modified = False session.save() else: @@ -196,14 +201,19 @@ def upload_package_group(self, repo, artifact, filename, session): def create_group_upload_task(self, cur_session, repository, artifact, filename, start_time): """Creates the actual task that adds the packages to the index.""" - cur_session['start'] = str(start_time) - cur_session['artifacts'] = [(str(artifact.sha256), filename)] + cur_session["start"] = str(start_time) + cur_session["artifacts"] = [(str(artifact.sha256), filename)] cur_session.modified = False cur_session.save() - result = dispatch(tasks.upload_group, exclusive_resources=[artifact, repository], - kwargs={"session_pk": str(cur_session.session_key), - "repository_pk": str(repository.pk)}) - return reverse('tasks-detail', args=[result.pk], request=None) + result = dispatch( + tasks.upload_group, + exclusive_resources=[artifact, repository], + kwargs={ + "session_pk": str(cur_session.session_key), + "repository_pk": str(repository.pk), + }, + ) + return reverse("tasks-detail", args=[result.pk], request=None) class SimpleView(PackageUploadMixin, ViewSet): @@ -231,16 +241,17 @@ def list(self, request, path): """Gets the simple api html page for the index.""" repo_version, content = self.get_rvc() if self.should_redirect(repo_version=repo_version): - return redirect(urljoin(self.base_content_url, f'{path}/simple/')) - names = content.order_by('name').values_list('name', flat=True).distinct().iterator() + return redirect(urljoin(self.base_content_url, f"{path}/simple/")) + names = content.order_by("name").values_list("name", flat=True).distinct().iterator() return StreamingHttpResponse(write_simple_index(names, streamed=True)) def pull_through_package_simple(self, package, path, remote): """Gets the package's simple page from remote.""" + def parse_package(release_package): parsed = urlparse(release_package.url) stripped_url = urlunsplit(chain(parsed[:3], ("", ""))) - redirect_path = f'{path}/{release_package.filename}?redirect={stripped_url}' + redirect_path = f"{path}/{release_package.filename}?redirect={stripped_url}" d_url = urljoin(self.base_content_url, redirect_path) return release_package.filename, d_url, release_package.digests.get("sha256", "") @@ -248,7 +259,7 @@ def parse_package(release_package): if not rfilter.filter_project(package): raise Http404(f"{package} does not exist.") - url = remote.get_remote_artifact_url(f'simple/{package}/') + url = remote.get_remote_artifact_url(f"simple/{package}/") remote.headers = remote.headers or [] remote.headers.append({"Accept": ACCEPT_JSON_PREFERRED}) downloader = remote.get_downloader(url=url, max_retries=1) @@ -277,10 +288,10 @@ def retrieve(self, request, path, package): if self.distribution.remote: return self.pull_through_package_simple(normalized, path, self.distribution.remote) if self.should_redirect(repo_version=repo_ver): - return redirect(urljoin(self.base_content_url, f'{path}/simple/{normalized}/')) + return redirect(urljoin(self.base_content_url, f"{path}/simple/{normalized}/")) packages = ( content.filter(name__normalize=normalized) - .values_list('filename', 'sha256', 'name') + .values_list("filename", "sha256", "name") .iterator() ) try: @@ -290,12 +301,14 @@ def retrieve(self, request, path, package): else: packages = chain([present], packages) name = present[2] - releases = ((f, urljoin(self.base_content_url, f'{path}/{f}'), d) for f, d, _ in packages) + releases = ((f, urljoin(self.base_content_url, f"{path}/{f}"), d) for f, d, _ in packages) return StreamingHttpResponse(write_simple_detail(name, releases, streamed=True)) - @extend_schema(request=PackageUploadSerializer, - responses={200: PackageUploadTaskSerializer}, - summary="Upload a package") + @extend_schema( + request=PackageUploadSerializer, + responses={200: PackageUploadTaskSerializer}, + summary="Upload a package", + ) def create(self, request, path): """ Upload package to the index. @@ -320,9 +333,11 @@ class MetadataView(PyPIMixin, ViewSet): ], } - @extend_schema(tags=["Pypi: Metadata"], - responses={200: PackageMetadataSerializer}, - summary="Get package metadata") + @extend_schema( + tags=["Pypi: Metadata"], + responses={200: PackageMetadataSerializer}, + summary="Get package metadata", + ) def retrieve(self, request, path, meta): """ Retrieves the package's core-metadata specified by @@ -368,8 +383,7 @@ class PyPIView(PyPIMixin, ViewSet): ], } - @extend_schema(responses={200: SummarySerializer}, - summary="Get index summary") + @extend_schema(responses={200: SummarySerializer}, summary="Get index summary") def retrieve(self, request, path): """Gets package summary stats of index.""" repo_ver, content = self.get_rvc() @@ -395,9 +409,11 @@ class UploadView(PackageUploadMixin, ViewSet): ], } - @extend_schema(request=PackageUploadSerializer, - responses={200: PackageUploadTaskSerializer}, - summary="Upload a package") + @extend_schema( + request=PackageUploadSerializer, + responses={200: PackageUploadTaskSerializer}, + summary="Upload a package", + ) def create(self, request, path): """ Upload package to the index. diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index b1ef7b9f..179e3656 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -44,15 +44,14 @@ class PythonDistributionSerializer(core_serializers.DistributionSerializer): ) base_url = serializers.SerializerMethodField(read_only=True) allow_uploads = serializers.BooleanField( - default=True, - help_text=_("Allow packages to be uploaded to this index.") + default=True, help_text=_("Allow packages to be uploaded to this index.") ) remote = core_serializers.DetailRelatedField( required=False, - help_text=_('Remote that can be used to fetch content when using pull-through caching.'), + help_text=_("Remote that can be used to fetch content when using pull-through caching."), view_name_pattern=r"remotes(-.*/.*)?-detail", queryset=core_models.Remote.objects.all(), - allow_null=True + allow_null=True, ) def get_base_url(self, obj): @@ -63,7 +62,9 @@ def get_base_url(self, obj): class Meta: fields = core_serializers.DistributionSerializer.Meta.fields + ( - 'publication', "allow_uploads", "remote" + "publication", + "allow_uploads", + "remote", ) model = python_models.PythonDistribution @@ -72,159 +73,202 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa """ A Serializer for PythonPackageContent. """ + # Core metadata # Version 1.0 author = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Text containing the author\'s name. Contact information can also be added,' - ' separated with newlines.') + required=False, + allow_blank=True, + help_text=_( + "Text containing the author's name. Contact information can also be added," + " separated with newlines." + ), ) author_email = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The author\'s e-mail address. ') + required=False, allow_blank=True, help_text=_("The author's e-mail address. ") ) description = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A longer description of the package that can run to several paragraphs.') + required=False, + allow_blank=True, + help_text=_("A longer description of the package that can run to several paragraphs."), ) home_page = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The URL for the package\'s home page.') + required=False, allow_blank=True, help_text=_("The URL for the package's home page.") ) keywords = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Additional keywords to be used to assist searching for the ' - 'package in a larger catalog.') + required=False, + allow_blank=True, + help_text=_( + "Additional keywords to be used to assist searching for the " + "package in a larger catalog." + ), ) license = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Text indicating the license covering the distribution') + required=False, + allow_blank=True, + help_text=_("Text indicating the license covering the distribution"), ) metadata_version = serializers.CharField( - help_text=_('Version of the file format'), + help_text=_("Version of the file format"), read_only=True, ) name = serializers.CharField( - help_text=_('The name of the python project.'), + help_text=_("The name of the python project."), read_only=True, ) platform = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A comma-separated list of platform specifications, ' - 'summarizing the operating systems supported by the package.') + required=False, + allow_blank=True, + help_text=_( + "A comma-separated list of platform specifications, " + "summarizing the operating systems supported by the package." + ), ) summary = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A one-line summary of what the package does.') + required=False, + allow_blank=True, + help_text=_("A one-line summary of what the package does."), ) version = serializers.CharField( - help_text=_('The packages version number.'), + help_text=_("The packages version number."), read_only=True, ) # Version 1.1 classifiers = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing classification values for a Python package.') + required=False, + default=list, + help_text=_("A JSON list containing classification values for a Python package."), ) download_url = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Legacy field denoting the URL from which this package can be downloaded.') + required=False, + allow_blank=True, + help_text=_("Legacy field denoting the URL from which this package can be downloaded."), ) supported_platform = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Field to specify the OS and CPU for which the binary package was compiled. ') + required=False, + allow_blank=True, + help_text=_("Field to specify the OS and CPU for which the binary package was compiled. "), ) # Version 1.2 maintainer = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The maintainer\'s name at a minimum; ' - 'additional contact information may be provided.') + required=False, + allow_blank=True, + help_text=_( + "The maintainer's name at a minimum; " "additional contact information may be provided." + ), ) maintainer_email = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The maintainer\'s e-mail address.') + required=False, allow_blank=True, help_text=_("The maintainer's e-mail address.") ) obsoletes_dist = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing names of a distutils project\'s distribution which ' - 'this distribution renders obsolete, meaning that the two projects should not ' - 'be installed at the same time.') + required=False, + default=list, + help_text=_( + "A JSON list containing names of a distutils project's distribution which " + "this distribution renders obsolete, meaning that the two projects should not " + "be installed at the same time." + ), ) project_url = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A browsable URL for the project and a label for it, separated by a comma.') + required=False, + allow_blank=True, + help_text=_("A browsable URL for the project and a label for it, separated by a comma."), ) project_urls = serializers.JSONField( - required=False, default=dict, - help_text=_('A dictionary of labels and URLs for the project.') + required=False, + default=dict, + help_text=_("A dictionary of labels and URLs for the project."), ) provides_dist = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing names of a Distutils project which is contained' - ' within this distribution.') + required=False, + default=list, + help_text=_( + "A JSON list containing names of a Distutils project which is contained" + " within this distribution." + ), ) requires_external = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing some dependency in the system that the distribution ' - 'is to be used.') + required=False, + default=list, + help_text=_( + "A JSON list containing some dependency in the system that the distribution " + "is to be used." + ), ) requires_dist = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing names of some other distutils project ' - 'required by this distribution.') + required=False, + default=list, + help_text=_( + "A JSON list containing names of some other distutils project " + "required by this distribution." + ), ) requires_python = serializers.CharField( - required=False, allow_blank=True, - help_text=_('The Python version(s) that the distribution is guaranteed to be ' - 'compatible with.') + required=False, + allow_blank=True, + help_text=_( + "The Python version(s) that the distribution is guaranteed to be " "compatible with." + ), ) # Version 2.1 description_content_type = serializers.CharField( - required=False, allow_blank=True, - help_text=_('A string stating the markup syntax (if any) used in the distribution’s' - ' description, so that tools can intelligently render the description.') + required=False, + allow_blank=True, + help_text=_( + "A string stating the markup syntax (if any) used in the distribution’s" + " description, so that tools can intelligently render the description." + ), ) provides_extras = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing names of optional features provided by the package.') + required=False, + default=list, + help_text=_("A JSON list containing names of optional features provided by the package."), ) # Version 2.2 dynamic = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing names of other core metadata fields which are ' - 'permitted to vary between sdist and bdist packages. Fields NOT marked ' - 'dynamic MUST be the same between bdist and sdist.') + required=False, + default=list, + help_text=_( + "A JSON list containing names of other core metadata fields which are " + "permitted to vary between sdist and bdist packages. Fields NOT marked " + "dynamic MUST be the same between bdist and sdist." + ), ) # Version 2.4 license_expression = serializers.CharField( - required=False, allow_blank=True, - help_text=_('Text string that is a valid SPDX license expression.') + required=False, + allow_blank=True, + help_text=_("Text string that is a valid SPDX license expression."), ) license_file = serializers.JSONField( - required=False, default=list, - help_text=_('A JSON list containing names of the paths to license-related files.') + required=False, + default=list, + help_text=_("A JSON list containing names of the paths to license-related files."), ) # Release metadata filename = serializers.CharField( - help_text=_('The name of the distribution package, usually of the format:' - ' {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}' - '-{platform tag}.{packagetype}'), + help_text=_( + "The name of the distribution package, usually of the format:" + " {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}" + "-{platform tag}.{packagetype}" + ), read_only=True, ) packagetype = serializers.CharField( - help_text=_('The type of the distribution package ' - '(e.g. sdist, bdist_wheel, bdist_egg, etc)'), + help_text=_( + "The type of the distribution package " "(e.g. sdist, bdist_wheel, bdist_egg, etc)" + ), read_only=True, ) python_version = serializers.CharField( help_text=_( - 'The tag that indicates which Python implementation or version the package requires.' + "The tag that indicates which Python implementation or version the package requires." ), read_only=True, ) sha256 = serializers.CharField( - default='', - help_text=_('The SHA256 digest of this package.'), + default="", + help_text=_("The SHA256 digest of this package."), ) def deferred_validate(self, data): @@ -243,22 +287,26 @@ def deferred_validate(self, data): try: filename = data["relative_path"] except KeyError: - raise serializers.ValidationError(detail={"relative_path": _('This field is required')}) + raise serializers.ValidationError(detail={"relative_path": _("This field is required")}) artifact = data["artifact"] try: _data = artifact_to_python_content_data(filename, artifact, domain=get_domain()) except ValueError: - raise serializers.ValidationError(_( - "Extension on {} is not a valid python extension " - "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)").format(filename) + raise serializers.ValidationError( + _( + "Extension on {} is not a valid python extension " + "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)" + ).format(filename) ) if data.get("sha256") and data["sha256"] != artifact.sha256: raise serializers.ValidationError( - detail={"sha256": _( - "The uploaded artifact's sha256 checksum does not match the one provided" - )} + detail={ + "sha256": _( + "The uploaded artifact's sha256 checksum does not match the one provided" + ) + } ) data.update(_data) @@ -273,13 +321,38 @@ def retrieve(self, validated_data): class Meta: fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + ( - 'author', 'author_email', 'description', 'home_page', 'keywords', 'license', - 'metadata_version', 'name', 'platform', 'summary', 'version', 'classifiers', - 'download_url', 'supported_platform', 'maintainer', 'maintainer_email', - 'obsoletes_dist', 'project_url', 'project_urls', 'provides_dist', 'requires_external', - 'requires_dist', 'requires_python', 'description_content_type', - 'provides_extras', 'dynamic', 'license_expression', 'license_file', - 'filename', 'packagetype', 'python_version', 'sha256' + "author", + "author_email", + "description", + "home_page", + "keywords", + "license", + "metadata_version", + "name", + "platform", + "summary", + "version", + "classifiers", + "download_url", + "supported_platform", + "maintainer", + "maintainer_email", + "obsoletes_dist", + "project_url", + "project_urls", + "provides_dist", + "requires_external", + "requires_dist", + "requires_python", + "description_content_type", + "provides_extras", + "dynamic", + "license_expression", + "license_file", + "filename", + "packagetype", + "python_version", + "sha256", ) model = python_models.PythonPackageContent @@ -291,7 +364,11 @@ class MinimalPythonPackageContentSerializer(PythonPackageContentSerializer): class Meta: fields = core_serializers.SingleArtifactContentUploadSerializer.Meta.fields + ( - 'filename', 'packagetype', 'name', 'version', 'sha256', + "filename", + "packagetype", + "name", + "version", + "sha256", ) model = python_models.PythonPackageContent @@ -315,47 +392,49 @@ class PythonRemoteSerializer(core_serializers.RemoteSerializer): child=serializers.CharField(allow_blank=False), required=False, allow_empty=True, - help_text=_( - "A list containing project specifiers for Python packages to include." - ), + help_text=_("A list containing project specifiers for Python packages to include."), ) excludes = serializers.ListField( child=serializers.CharField(allow_blank=False), required=False, allow_empty=True, - help_text=_( - "A list containing project specifiers for Python packages to exclude." - ), + help_text=_("A list containing project specifiers for Python packages to exclude."), ) prereleases = serializers.BooleanField( - required=False, - help_text=_('Whether or not to include pre-release packages in the sync.') + required=False, help_text=_("Whether or not to include pre-release packages in the sync.") ) policy = serializers.ChoiceField( - help_text=_("The policy to use when downloading content. The possible values include: " - "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default."), + help_text=_( + "The policy to use when downloading content. The possible values include: " + "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default." + ), choices=core_models.Remote.POLICY_CHOICES, - default=core_models.Remote.ON_DEMAND + default=core_models.Remote.ON_DEMAND, ) package_types = MultipleChoiceArrayField( required=False, - help_text=_("The package types to sync for Python content. Leave blank to get every" - "package type."), + help_text=_( + "The package types to sync for Python content. Leave blank to get every" "package type." + ), choices=python_models.PACKAGE_TYPES, - default=list + default=list, ) keep_latest_packages = serializers.IntegerField( required=False, - help_text=_("The amount of latest versions of a package to keep on sync, includes" - "pre-releases if synced. Default 0 keeps all versions."), - default=0 + help_text=_( + "The amount of latest versions of a package to keep on sync, includes" + "pre-releases if synced. Default 0 keeps all versions." + ), + default=0, ) exclude_platforms = MultipleChoiceArrayField( required=False, - help_text=_("List of platforms to exclude syncing Python packages for. Possible values" - "include: windows, macos, freebsd, and linux."), + help_text=_( + "List of platforms to exclude syncing Python packages for. Possible values" + "include: windows, macos, freebsd, and linux." + ), choices=python_models.PLATFORMS, - default=list + default=list, ) def validate_includes(self, value): @@ -382,7 +461,11 @@ def validate_excludes(self, value): class Meta: fields = core_serializers.RemoteSerializer.Meta.fields + ( - "includes", "excludes", "prereleases", "package_types", "keep_latest_packages", + "includes", + "excludes", + "prereleases", + "package_types", + "keep_latest_packages", "exclude_platforms", ) model = python_models.PythonRemote @@ -404,10 +487,12 @@ class PythonBanderRemoteSerializer(serializers.Serializer): ) policy = serializers.ChoiceField( - help_text=_("The policy to use when downloading content. The possible values include: " - "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default."), + help_text=_( + "The policy to use when downloading content. The possible values include: " + "'immediate', 'on_demand', and 'streamed'. 'on_demand' is the default." + ), choices=core_models.Remote.POLICY_CHOICES, - default=core_models.Remote.ON_DEMAND + default=core_models.Remote.ON_DEMAND, ) @@ -417,8 +502,9 @@ class PythonPublicationSerializer(core_serializers.PublicationSerializer): """ distributions = core_serializers.DetailRelatedField( - help_text=_('This publication is currently being hosted as configured by these ' - 'distributions.'), + help_text=_( + "This publication is currently being hosted as configured by these " "distributions." + ), source="distribution_set", view_name="pythondistributions-detail", many=True, @@ -426,5 +512,5 @@ class PythonPublicationSerializer(core_serializers.PublicationSerializer): ) class Meta: - fields = core_serializers.PublicationSerializer.Meta.fields + ('distributions',) + fields = core_serializers.PublicationSerializer.Meta.fields + ("distributions",) model = python_models.PythonPublication diff --git a/pulp_python/app/settings.py b/pulp_python/app/settings.py index 7084ef10..eab951a5 100644 --- a/pulp_python/app/settings.py +++ b/pulp_python/app/settings.py @@ -1,7 +1,7 @@ import socket PYTHON_GROUP_UPLOADS = False -PYPI_API_HOSTNAME = 'https://' + socket.getfqdn() +PYPI_API_HOSTNAME = "https://" + socket.getfqdn() DRF_ACCESS_POLICY = { "dynaconf_merge_unique": True, diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index 136b511f..e8325a5c 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -25,15 +25,17 @@ def publish(repository_version_pk): """ repository_version = models.RepositoryVersion.objects.get(pk=repository_version_pk) - log.info(_('Publishing: repository={repo}, version={version}').format( - repo=repository_version.repository.name, - version=repository_version.number, - )) + log.info( + _("Publishing: repository={repo}, version={version}").format( + repo=repository_version.repository.name, + version=repository_version.number, + ) + ) with python_models.PythonPublication.create(repository_version, pass_through=True) as pub: write_simple_api(pub) - log.info(_('Publication: {pk} created').format(pk=pub.pk)) + log.info(_("Publication: {pk} created").format(pk=pub.pk)) return pub @@ -51,26 +53,24 @@ def write_simple_api(publication): """ domain = get_domain() - simple_dir = 'simple/' + simple_dir = "simple/" os.mkdir(simple_dir) project_names = ( python_models.PythonPackageContent.objects.filter( pk__in=publication.repository_version.content, _pulp_domain=domain ) - .order_by('name') - .values_list('name', flat=True) + .order_by("name") + .values_list("name", flat=True) .distinct() ) # write the root index, which lists all of the projects for which there is a package available - index_path = '{simple_dir}index.html'.format(simple_dir=simple_dir) - with open(index_path, 'w') as index: + index_path = "{simple_dir}index.html".format(simple_dir=simple_dir) + with open(index_path, "w") as index: index.write(write_simple_index(project_names)) index_metadata = models.PublishedMetadata.create_from_file( - relative_path=index_path, - publication=publication, - file=File(open(index_path, 'rb')) + relative_path=index_path, publication=publication, file=File(open(index_path, "rb")) ) index_metadata.save() @@ -86,41 +86,41 @@ def write_simple_api(publication): current_name = project_names[ind] package_releases = [] for release in releases.iterator(): - if release['name'] != current_name: + if release["name"] != current_name: write_project_page( name=canonicalize_name(current_name), simple_dir=simple_dir, package_releases=package_releases, - publication=publication + publication=publication, ) package_releases = [] ind += 1 current_name = project_names[ind] - relative_path = release['filename'] + relative_path = release["filename"] path = f"../../{relative_path}" - checksum = release['sha256'] + checksum = release["sha256"] package_releases.append((relative_path, path, checksum)) # Write the final project's page write_project_page( name=canonicalize_name(current_name), simple_dir=simple_dir, package_releases=package_releases, - publication=publication + publication=publication, ) def write_project_page(name, simple_dir, package_releases, publication): """Writes a project's simple page.""" - project_dir = f'{simple_dir}{name}/' + project_dir = f"{simple_dir}{name}/" os.mkdir(project_dir) - metadata_relative_path = f'{project_dir}index.html' + metadata_relative_path = f"{project_dir}index.html" - with open(metadata_relative_path, 'w') as simple_metadata: + with open(metadata_relative_path, "w") as simple_metadata: simple_metadata.write(write_simple_detail(name, package_releases)) project_metadata = models.PublishedMetadata.create_from_file( relative_path=metadata_relative_path, publication=publication, - file=File(open(metadata_relative_path, 'rb')) + file=File(open(metadata_relative_path, "rb")), ) project_metadata.save() # change to bulk create when multi-table supported diff --git a/pulp_python/app/tasks/repair.py b/pulp_python/app/tasks/repair.py index 7ba26075..0f32cfa2 100644 --- a/pulp_python/app/tasks/repair.py +++ b/pulp_python/app/tasks/repair.py @@ -34,9 +34,9 @@ def repair(repository_pk: UUID) -> None: repository = PythonRepository.objects.get(pk=repository_pk) log.info( - _( - "Repairing packages' metadata for the latest version of repository {}." - ).format(repository.name) + _("Repairing packages' metadata for the latest version of repository {}.").format( + repository.name + ) ) content_set = repository.latest_version().content.values_list("pk", flat=True) content = PythonPackageContent.objects.filter(pk__in=content_set) @@ -94,9 +94,7 @@ def repair_metadata(content: QuerySet[PythonPackageContent]) -> tuple[int, set[s ) progress_report.save() with progress_report: - for package in progress_report.iter( - immediate_content.iterator(chunk_size=BULK_SIZE) - ): + for package in progress_report.iter(immediate_content.iterator(chunk_size=BULK_SIZE)): new_data = artifact_to_python_content_data( package.filename, package._artifacts.get(), domain ) diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index 317f7b6b..2f009f1b 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -51,9 +51,7 @@ def sync(remote_pk, repository_pk, mirror): repository = Repository.objects.get(pk=repository_pk) if not remote.url: - raise serializers.ValidationError( - detail=_("A remote must have a url attribute to sync.") - ) + raise serializers.ValidationError(detail=_("A remote must have a url attribute to sync.")) first_stage = PythonBanderStage(remote) DeclarativeVersion(first_stage, repository, mirror).create() @@ -146,9 +144,7 @@ async def run(self): ) packages_to_sync = None if self.remote.includes: - packages_to_sync = [ - Requirement(pkg).name for pkg in self.remote.includes - ] + packages_to_sync = [Requirement(pkg).name for pkg in self.remote.includes] await pmirror.synchronize(packages_to_sync) # place back old session so that it is properly closed master.session = old_session @@ -159,9 +155,7 @@ class PulpMirror(Mirror): Pulp Mirror Class to perform syncing using Bandersnatch """ - def __init__( - self, serial, master, workers, deferred_download, python_stage, progress_report - ): + def __init__(self, serial, master, workers, deferred_download, python_stage, progress_report): """Initialize Bandersnatch Mirror""" super().__init__(master=master, workers=workers) self.synced_serial = serial @@ -176,11 +170,7 @@ async def determine_packages_to_sync(self): """ number_xmlrpc_attempts = 3 for attempt in range(number_xmlrpc_attempts): - logger.info( - "Attempt {} to get package list from {}".format( - attempt, self.master.url - ) - ) + logger.info("Attempt {} to get package list from {}".format(attempt, self.master.url)) try: if not self.synced_serial: logger.info("Syncing all packages.") @@ -192,9 +182,7 @@ async def determine_packages_to_sync(self): ) else: logger.info("Syncing based on changelog.") - changed_packages = await self.master.changed_packages( - self.synced_serial - ) + changed_packages = await self.master.changed_packages(self.synced_serial) self.packages_to_sync.update(changed_packages) self.target_serial = max( [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()] diff --git a/pulp_python/app/tasks/upload.py b/pulp_python/app/tasks/upload.py index d573c56a..01fb4ec2 100644 --- a/pulp_python/app/tasks/upload.py +++ b/pulp_python/app/tasks/upload.py @@ -43,10 +43,10 @@ def upload_group(session_pk, repository_pk=None): with transaction.atomic(): session_data = s_query.first().get_decoded() now = datetime.now(tz=timezone.utc) - start_time = datetime.fromisoformat(session_data['start']) + start_time = datetime.fromisoformat(session_data["start"]) if now >= start_time: content_to_add = PythonPackageContent.objects.none() - for artifact_sha256, filename in session_data['artifacts']: + for artifact_sha256, filename in session_data["artifacts"]: pre_check = PythonPackageContent.objects.filter( sha256=artifact_sha256, _pulp_domain=domain ) @@ -81,9 +81,7 @@ def create_content(artifact_sha256, filename, domain): @transaction.atomic() def create(): content = PythonPackageContent.objects.create(**data) - ContentArtifact.objects.create( - artifact=artifact, content=content, relative_path=filename - ) + ContentArtifact.objects.create(artifact=artifact, content=content, relative_path=filename) return content new_content = create() diff --git a/pulp_python/app/urls.py b/pulp_python/app/urls.py index 0a786333..405ab119 100644 --- a/pulp_python/app/urls.py +++ b/pulp_python/app/urls.py @@ -16,17 +16,17 @@ path( PYPI_API_URL + "pypi//", MetadataView.as_view({"get": "retrieve"}), - name="pypi-metadata" + name="pypi-metadata", ), path( PYPI_API_URL + "simple//", SimpleView.as_view({"get": "retrieve"}), - name="simple-package-detail" + name="simple-package-detail", ), path( - PYPI_API_URL + 'simple/', + PYPI_API_URL + "simple/", SimpleView.as_view({"get": "list", "post": "create"}), - name="simple-detail" + name="simple-detail", ), path(PYPI_API_URL, PyPIView.as_view({"get": "retrieve"}), name="pypi-detail"), ] diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 35458718..88c56549 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -60,10 +60,12 @@ r"""^(?P.+?)-(?P.*?) ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) \.whl|\.dist-info)$""", - re.VERBOSE + re.VERBOSE, ), # regex based on https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#filename-embedded-metadata # noqa: E501 - ".egg": re.compile(r"^(?P.+?)-(?P.*?)(-(?P.+?(-(?P.+?))?))?\.egg|\.egg-info$"), # noqa: E501 + ".egg": re.compile( + r"^(?P.+?)-(?P.*?)(-(?P.+?(-(?P.+?))?))?\.egg|\.egg-info$" + ), # noqa: E501 # regex based on https://github.com/python/cpython/blob/v3.7.0/Lib/distutils/command/bdist_wininst.py#L292 # noqa: E501 ".exe": re.compile(r"^(?P.+?)-(?P.*?)\.(?P.+?)(-(?P.+?))?\.exe$"), } @@ -90,42 +92,42 @@ def parse_project_metadata(project): return { # Core metadata # Version 1.0 - 'author': project.get('author') or "", - 'author_email': project.get('author_email') or "", - 'description': project.get('description') or "", - 'home_page': project.get('home_page') or "", - 'keywords': project.get('keywords') or "", - 'license': project.get('license') or "", - 'metadata_version': project.get('metadata_version') or "", - 'name': project.get('name') or "", - 'platform': project.get('platform') or "", - 'summary': project.get('summary') or "", - 'version': project.get('version') or "", + "author": project.get("author") or "", + "author_email": project.get("author_email") or "", + "description": project.get("description") or "", + "home_page": project.get("home_page") or "", + "keywords": project.get("keywords") or "", + "license": project.get("license") or "", + "metadata_version": project.get("metadata_version") or "", + "name": project.get("name") or "", + "platform": project.get("platform") or "", + "summary": project.get("summary") or "", + "version": project.get("version") or "", # Version 1.1 - 'classifiers': json.dumps(project.get('classifiers', [])), - 'download_url': project.get('download_url') or "", - 'supported_platform': project.get('supported_platform') or "", + "classifiers": json.dumps(project.get("classifiers", [])), + "download_url": project.get("download_url") or "", + "supported_platform": project.get("supported_platform") or "", # Version 1.2 - 'maintainer': project.get('maintainer') or "", - 'maintainer_email': project.get('maintainer_email') or "", - 'obsoletes_dist': json.dumps(project.get('obsoletes_dist', [])), - 'project_url': project.get('project_url') or "", - 'project_urls': json.dumps(project.get('project_urls', {})), - 'provides_dist': json.dumps(project.get('provides_dist', [])), - 'requires_external': json.dumps(project.get('requires_external', [])), - 'requires_dist': json.dumps(project.get('requires_dist', [])), - 'requires_python': project.get('requires_python') or "", + "maintainer": project.get("maintainer") or "", + "maintainer_email": project.get("maintainer_email") or "", + "obsoletes_dist": json.dumps(project.get("obsoletes_dist", [])), + "project_url": project.get("project_url") or "", + "project_urls": json.dumps(project.get("project_urls", {})), + "provides_dist": json.dumps(project.get("provides_dist", [])), + "requires_external": json.dumps(project.get("requires_external", [])), + "requires_dist": json.dumps(project.get("requires_dist", [])), + "requires_python": project.get("requires_python") or "", # Version 2.1 - 'description_content_type': project.get('description_content_type') or "", - 'provides_extras': json.dumps(project.get('provides_extras', [])), + "description_content_type": project.get("description_content_type") or "", + "provides_extras": json.dumps(project.get("provides_extras", [])), # Version 2.2 - 'dynamic': json.dumps(project.get('dynamic', [])), + "dynamic": json.dumps(project.get("dynamic", [])), # Version 2.4 - 'license_expression': project.get('license_expression') or "", - 'license_file': json.dumps(project.get('license_file', [])), + "license_expression": project.get("license_expression") or "", + "license_file": json.dumps(project.get("license_file", [])), # Release metadata - 'packagetype': project.get('packagetype') or "", - 'python_version': project.get('python_version') or "", + "packagetype": project.get("packagetype") or "", + "python_version": project.get("python_version") or "", } @@ -147,13 +149,15 @@ def parse_metadata(project, version, distribution): """ package = parse_project_metadata(project) - package['filename'] = distribution.get('filename') or "" - package['packagetype'] = distribution.get('packagetype') or "" - package['version'] = version - package['url'] = distribution.get('url') or "" - package['sha256'] = distribution.get('digests', {}).get('sha256') or "" - package['python_version'] = distribution.get('python_version') or package.get('python_version') - package['requires_python'] = distribution.get('requires_python') or package.get('requires_python') # noqa: E501 + package["filename"] = distribution.get("filename") or "" + package["packagetype"] = distribution.get("packagetype") or "" + package["version"] = version + package["url"] = distribution.get("url") or "" + package["sha256"] = distribution.get("digests", {}).get("sha256") or "" + package["python_version"] = distribution.get("python_version") or package.get("python_version") + package["requires_python"] = distribution.get("requires_python") or package.get( + "requires_python" + ) # noqa: E501 return package @@ -172,7 +176,7 @@ def get_project_metadata_from_artifact(filename, artifact): # Copy file to a temp directory under the user provided filename, we do this # because pkginfo validates that the filename has a valid extension before # reading it - with tempfile.NamedTemporaryFile('wb', dir=".", suffix=filename) as temp_file: + with tempfile.NamedTemporaryFile("wb", dir=".", suffix=filename) as temp_file: shutil.copyfileobj(artifact.file, temp_file) temp_file.flush() metadata = DIST_TYPES[packagetype](temp_file.name) @@ -194,10 +198,10 @@ def artifact_to_python_content_data(filename, artifact, domain=None): """ metadata = get_project_metadata_from_artifact(filename, artifact) data = parse_project_metadata(vars(metadata)) - data['sha256'] = artifact.sha256 - data['filename'] = filename - data['pulp_domain'] = domain or artifact.pulp_domain - data['_pulp_domain'] = data['pulp_domain'] + data["sha256"] = artifact.sha256 + data["filename"] = filename + data["pulp_domain"] = domain or artifact.pulp_domain + data["_pulp_domain"] = data["pulp_domain"] return data @@ -355,10 +359,12 @@ def python_content_to_download_info(content, base_path, domain=None): Takes in a PythonPackageContent and base path of the distribution to create a dictionary of download information for that content. This dictionary is used by Releases and Urls. """ + def find_artifact(): _art = content_artifact.artifact if not _art: from pulpcore.plugin import models + _art = models.RemoteArtifact.objects.filter(content_artifact=content_artifact).first() return _art @@ -389,7 +395,7 @@ def find_artifact(): "upload_time_iso_8601": str(content.pulp_created.isoformat()), "url": url, "yanked": False, - "yanked_reason": None + "yanked_reason": None, } diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 7dccf239..41702f95 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -26,7 +26,7 @@ class PythonRepositoryViewSet( synced, added, or removed. """ - endpoint_name = 'python' + endpoint_name = "python" queryset = python_models.PythonRepository.objects.all() serializer_class = python_serializers.PythonRepositorySerializer queryset_filtering_required_permission = "python.view_pythonrepository" @@ -141,11 +141,8 @@ def repair_metadata(self, request, pk): ) return core_viewsets.OperationPostponedResponse(result, request) - @extend_schema( - summary="Sync from remote", - responses={202: AsyncOperationResponseSerializer} - ) - @action(detail=True, methods=['post'], serializer_class=RepositorySyncURLSerializer) + @extend_schema(summary="Sync from remote", responses={202: AsyncOperationResponseSerializer}) + @action(detail=True, methods=["post"], serializer_class=RepositorySyncURLSerializer) def sync(self, request, pk): """ @@ -155,22 +152,21 @@ def sync(self, request, pk): """ repository = self.get_object() serializer = RepositorySyncURLSerializer( - data=request.data, - context={'request': request, "repository_pk": pk} + data=request.data, context={"request": request, "repository_pk": pk} ) serializer.is_valid(raise_exception=True) - remote = serializer.validated_data.get('remote', repository.remote) - mirror = serializer.validated_data.get('mirror') + remote = serializer.validated_data.get("remote", repository.remote) + mirror = serializer.validated_data.get("mirror") result = dispatch( tasks.sync, exclusive_resources=[repository], shared_resources=[remote], kwargs={ - 'remote_pk': str(remote.pk), - 'repository_pk': str(repository.pk), - 'mirror': mirror - } + "remote_pk": str(remote.pk), + "repository_pk": str(repository.pk), + "mirror": mirror, + }, ) return core_viewsets.OperationPostponedResponse(result, request) @@ -188,8 +184,7 @@ class PythonRepositoryVersionViewSet(core_viewsets.RepositoryVersionViewSet): "action": ["list", "retrieve"], "principal": "authenticated", "effect": "allow", - "condition": - "has_repository_model_or_domain_or_obj_perms:python.view_pythonrepository", + "condition": "has_repository_model_or_domain_or_obj_perms:python.view_pythonrepository", # noqa: E501 }, { "action": ["destroy"], @@ -224,7 +219,7 @@ class PythonDistributionViewSet(core_viewsets.DistributionViewSet, core_viewsets href="./#tag/Content:-Packages">Python Package Content. """ - endpoint_name = 'pypi' + endpoint_name = "pypi" queryset = python_models.PythonDistribution.objects.all() serializer_class = python_serializers.PythonDistributionSerializer queryset_filtering_required_permission = "python.view_pythondistribution" @@ -313,19 +308,20 @@ class PythonPackageContentFilter(core_viewsets.ContentFilter): class Meta: model = python_models.PythonPackageContent fields = { - 'name': ['exact', 'in'], - 'author': ['exact', 'in'], - 'packagetype': ['exact', 'in'], - 'requires_python': ['exact', 'in', "contains"], - 'filename': ['exact', 'in', 'contains'], - 'keywords': ['in', 'contains'], - 'sha256': ['exact', 'in'], - 'version': ['exact', 'gt', 'lt', 'gte', 'lte'] + "name": ["exact", "in"], + "author": ["exact", "in"], + "packagetype": ["exact", "in"], + "requires_python": ["exact", "in", "contains"], + "filename": ["exact", "in", "contains"], + "keywords": ["in", "contains"], + "sha256": ["exact", "in"], + "version": ["exact", "gt", "lt", "gte", "lte"], } class PythonPackageSingleArtifactContentUploadViewSet( - core_viewsets.SingleArtifactContentUploadViewSet): + core_viewsets.SingleArtifactContentUploadViewSet +): """ PythonPackageContent represents each individually installable Python package. In the Python @@ -336,7 +332,7 @@ class PythonPackageSingleArtifactContentUploadViewSet( """ - endpoint_name = 'packages' + endpoint_name = "packages" queryset = python_models.PythonPackageContent.objects.all() serializer_class = python_serializers.PythonPackageContentSerializer minimal_serializer_class = python_serializers.MinimalPythonPackageContentSerializer @@ -373,7 +369,7 @@ class PythonRemoteViewSet(core_viewsets.RemoteViewSet, core_viewsets.RolesMixin) """ - endpoint_name = 'python' + endpoint_name = "python" queryset = python_models.PythonRemote.objects.all() serializer_class = python_serializers.PythonRemoteSerializer queryset_filtering_required_permission = "python.view_pythonremote" @@ -445,8 +441,11 @@ class PythonRemoteViewSet(core_viewsets.RemoteViewSet, core_viewsets.RolesMixin) summary="Create from Bandersnatch", responses={201: python_serializers.PythonRemoteSerializer}, ) - @action(detail=False, methods=["post"], - serializer_class=python_serializers.PythonBanderRemoteSerializer) + @action( + detail=False, + methods=["post"], + serializer_class=python_serializers.PythonBanderRemoteSerializer, + ) def from_bandersnatch(self, request): """ @@ -458,11 +457,12 @@ def from_bandersnatch(self, request): name = serializer.validated_data.get("name") policy = serializer.validated_data.get("policy") bander_config = BandersnatchConfig(bander_config_file.file.name).config - data = {"name": name, - "policy": policy, - "url": bander_config.get("mirror", "master"), - "download_concurrency": bander_config.get("mirror", "workers"), - } + data = { + "name": name, + "policy": policy, + "url": bander_config.get("mirror", "master"), + "download_concurrency": bander_config.get("mirror", "workers"), + } enabled = bander_config.get("plugins", "enabled") enabled_all = "all" in enabled data["prereleases"] = not (enabled_all or "prerelease_release" in enabled) @@ -479,8 +479,9 @@ def from_bandersnatch(self, request): "exclude_platform": ("blocklist", "platforms", "exclude_platforms"), } for plugin, options in plugin_filters.items(): - if (enabled_all or plugin in enabled) and \ - bander_config.has_option(options[0], options[1]): + if (enabled_all or plugin in enabled) and bander_config.has_option( + options[0], options[1] + ): data[options[2]] = bander_config.get(options[0], options[1]).split() remote = python_serializers.PythonRemoteSerializer(data=data, context={"request": request}) remote.is_valid(raise_exception=True) @@ -497,7 +498,7 @@ class PythonPublicationViewSet(core_viewsets.PublicationViewSet, core_viewsets.R """ - endpoint_name = 'pypi' + endpoint_name = "pypi" queryset = python_models.PythonPublication.objects.exclude(complete=False) serializer_class = python_serializers.PythonPublicationSerializer queryset_filtering_required_permission = "python.view_pythonpublication" @@ -561,9 +562,7 @@ class PythonPublicationViewSet(core_viewsets.PublicationViewSet, core_viewsets.R "python.pythonpublication_viewer": ["python.view_pythonpublication"], } - @extend_schema( - responses={202: AsyncOperationResponseSerializer} - ) + @extend_schema(responses={202: AsyncOperationResponseSerializer}) def create(self, request): """ @@ -571,18 +570,16 @@ def create(self, request): """ serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) - repository_version = serializer.validated_data.get('repository_version') + repository_version = serializer.validated_data.get("repository_version") # Safe because version OR repository is enforced by serializer. if not repository_version: - repository = serializer.validated_data.get('repository') + repository = serializer.validated_data.get("repository") repository_version = RepositoryVersion.latest(repository) result = dispatch( tasks.publish, shared_resources=[repository_version.repository], - kwargs={ - 'repository_version_pk': str(repository_version.pk) - } + kwargs={"repository_version_pk": str(repository_version.pk)}, ) return core_viewsets.OperationPostponedResponse(result, request) diff --git a/pulp_python/pytest_plugin.py b/pulp_python/pytest_plugin.py index 346cc951..f19f7c1f 100644 --- a/pulp_python/pytest_plugin.py +++ b/pulp_python/pytest_plugin.py @@ -35,6 +35,7 @@ def python_bindings(_api_client_set, bindings_cfg): @pytest.fixture def python_repo_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Repository with auto-cleanup.""" + def _gen_python_repo(remote=None, pulp_domain=None, **body): body.setdefault("name", str(uuid.uuid4())) kwargs = {} @@ -56,6 +57,7 @@ def python_repo(python_repo_factory): @pytest.fixture def python_distribution_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Distribution with auto-cleanup.""" + def _gen_python_distribution( publication=None, repository=None, version=None, pulp_domain=None, **body ): @@ -85,6 +87,7 @@ def _gen_python_distribution( @pytest.fixture def python_publication_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Publication with auto-cleanup.""" + def _gen_python_publication(repository, version=None, pulp_domain=None): repo_href = get_href(repository) if version: @@ -106,6 +109,7 @@ def _gen_python_publication(repository, version=None, pulp_domain=None): @pytest.fixture def python_remote_factory(python_bindings, gen_object_with_cleanup): """A factory to generate a Python Remote with auto-cleanup.""" + def _gen_python_remote(url=PYTHON_FIXTURE_URL, includes=None, pulp_domain=None, **body): body.setdefault("name", str(uuid.uuid4())) body.setdefault("url", url) @@ -125,6 +129,7 @@ def python_repo_with_sync( python_bindings, python_repo_factory, python_remote_factory, monitor_task ): """A factory to generate a Python Repository synced with the passed in Remote.""" + def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): kwargs = {} if pulp_domain := body.get("pulp_domain"): @@ -141,6 +146,7 @@ def _gen_python_repo_sync(remote=None, mirror=False, repository=None, **body): @pytest.fixture def download_python_file(tmp_path, http_get): """Download a Python file and return its path.""" + def _download_python_file(relative_path, url): file_path = tmp_path / relative_path with open(file_path, mode="wb") as f: @@ -159,6 +165,7 @@ def python_file(download_python_file): @pytest.fixture def python_content_factory(python_bindings, download_python_file, monitor_task): """A factory to create a Python Package Content.""" + def _gen_python_content(relative_path=PYTHON_EGG_FILENAME, url=None, **body): body["relative_path"] = relative_path if url: @@ -191,6 +198,7 @@ def shelf_reader_cleanup(): @pytest.fixture def python_content_summary(python_bindings): """Get a summary of the repository version's content.""" + def _gen_summary(repository_version=None, repository=None, version=None): if repository_version is None: repo_href = get_href(repository) diff --git a/pulp_python/tests/functional/api/test_crud_content_unit.py b/pulp_python/tests/functional/api/test_crud_content_unit.py index 648f27a8..9689735c 100644 --- a/pulp_python/tests/functional/api/test_crud_content_unit.py +++ b/pulp_python/tests/functional/api/test_crud_content_unit.py @@ -102,7 +102,9 @@ def test_content_crud( monitor_task(pulpcore_bindings.OrphansCleanupApi.cleanup({"orphan_protection_time": 0}).task) mismatch_sha256 = PYTHON_SM_FIXTURE_CHECKSUMS["aiohttp-3.3.0.tar.gz"] content_body = { - "relative_path": PYTHON_EGG_FILENAME, "file": python_file, "sha256": mismatch_sha256 + "relative_path": PYTHON_EGG_FILENAME, + "file": python_file, + "sha256": mismatch_sha256, } with pytest.raises(PulpTaskError) as e: response = python_bindings.ContentPackagesApi.create(**content_body) @@ -129,8 +131,8 @@ def test_content_create_new_metadata( python_package_data = { "filename": "setuptools-80.9.0.tar.gz", - "provides_extras": - '["test", "doc", "ssl", "certs", "core", "check", "cover", "enabler", "type"]', + "provides_extras": '["test", "doc", "ssl", "certs",' + ' "core", "check", "cover", "enabler", "type"]', "dynamic": '["license-file"]', "license_expression": "MIT", "license_file": '["LICENSE"]', diff --git a/pulp_python/tests/functional/api/test_crud_publications.py b/pulp_python/tests/functional/api/test_crud_publications.py index 70e8b432..9659d4ee 100644 --- a/pulp_python/tests/functional/api/test_crud_publications.py +++ b/pulp_python/tests/functional/api/test_crud_publications.py @@ -17,6 +17,7 @@ def python_publication_workflow( python_repo_with_sync, python_remote_factory, python_publication_factory ): """Create repo, remote, sync & then publish.""" + def _publish_workflow(repository=None, remote=None, **remote_body): if not remote: remote = python_remote_factory(**remote_body) @@ -55,8 +56,9 @@ def test_all_content_published(python_publication_workflow, python_distribution_ distro = python_distribution_factory(publication=pub) url = urljoin(distro.base_url, "simple/") - proper, msgs = ensure_simple(url, PYTHON_SM_FIXTURE_RELEASES, - sha_digests=PYTHON_SM_FIXTURE_CHECKSUMS) + proper, msgs = ensure_simple( + url, PYTHON_SM_FIXTURE_RELEASES, sha_digests=PYTHON_SM_FIXTURE_CHECKSUMS + ) assert proper is True, msgs diff --git a/pulp_python/tests/functional/api/test_domains.py b/pulp_python/tests/functional/api/test_domains.py index 786e58ec..7209d6d3 100644 --- a/pulp_python/tests/functional/api/test_domains.py +++ b/pulp_python/tests/functional/api/test_domains.py @@ -72,7 +72,9 @@ def test_domain_object_creation( with pytest.raises(python_bindings.ApiException) as e: distro_body = { - "name": str(uuid.uuid4()), "base_path": str(uuid.uuid4()), "repository": repo.pulp_href + "name": str(uuid.uuid4()), + "base_path": str(uuid.uuid4()), + "repository": repo.pulp_href, } python_bindings.DistributionsPypiApi.create(distro_body) assert e.value.status == 400 diff --git a/pulp_python/tests/functional/api/test_export_import.py b/pulp_python/tests/functional/api/test_export_import.py index 15a21e04..e3ecda82 100644 --- a/pulp_python/tests/functional/api/test_export_import.py +++ b/pulp_python/tests/functional/api/test_export_import.py @@ -4,12 +4,14 @@ NOTE: assumes ALLOWED_EXPORT_PATHS setting contains "/tmp" - all tests will fail if this is not the case. """ + import pytest import uuid from pulpcore.app import settings from pulp_python.tests.functional.constants import ( - PYTHON_XS_PROJECT_SPECIFIER, PYTHON_SM_PROJECT_SPECIFIER + PYTHON_XS_PROJECT_SPECIFIER, + PYTHON_SM_PROJECT_SPECIFIER, ) diff --git a/pulp_python/tests/functional/api/test_pypi_apis.py b/pulp_python/tests/functional/api/test_pypi_apis.py index 1e557f36..4294ecc2 100644 --- a/pulp_python/tests/functional/api/test_pypi_apis.py +++ b/pulp_python/tests/functional/api/test_pypi_apis.py @@ -29,6 +29,7 @@ @pytest.fixture def python_empty_repo_distro(python_repo_factory, python_distribution_factory): """Returns an empty repo with and distribution serving it.""" + def _generate_empty_repo_distro(repo_body=None, distro_body=None): repo_body = repo_body or {} distro_body = distro_body or {} @@ -240,7 +241,7 @@ def test_twine_upload( ), capture_output=True, check=True, - text=True + text=True, ) assert output.stdout.count("Skipping") == 2 @@ -322,9 +323,7 @@ def test_pypi_last_serial( repo = python_repo_with_sync(remote) pub = python_publication_factory(repository=repo) distro = python_distribution_factory(publication=pub) - content_url = urljoin( - pulp_content_url, f"{distro.base_path}/pypi/shelf-reader/json" - ) + content_url = urljoin(pulp_content_url, f"{distro.base_path}/pypi/shelf-reader/json") pypi_url = urljoin(distro.base_url, "pypi/shelf-reader/json/") for url in [content_url, pypi_url]: response = requests.get(url) @@ -337,9 +336,7 @@ def assert_pypi_json(package): assert SHELF_PYTHON_JSON["last_serial"] == package["last_serial"] assert SHELF_PYTHON_JSON["info"].items() <= package["info"].items() assert len(SHELF_PYTHON_JSON["urls"]) == len(package["urls"]) - assert_download_info( - SHELF_PYTHON_JSON["urls"], package["urls"], "Failed to match URLS" - ) + assert_download_info(SHELF_PYTHON_JSON["urls"], package["urls"], "Failed to match URLS") assert SHELF_PYTHON_JSON["releases"].keys() <= package["releases"].keys() for version in SHELF_PYTHON_JSON["releases"].keys(): assert_download_info( diff --git a/pulp_python/tests/functional/api/test_rbac.py b/pulp_python/tests/functional/api/test_rbac.py index 125ef368..2c46977e 100644 --- a/pulp_python/tests/functional/api/test_rbac.py +++ b/pulp_python/tests/functional/api/test_rbac.py @@ -238,7 +238,7 @@ def test_pypi_apis( python_distribution_factory, anonymous_user, download_python_file, - try_action + try_action, ): alice, bob, charlie = gen_users(["pythonrepository", "pythondistribution"]) with bob: diff --git a/pulp_python/tests/functional/api/test_repair.py b/pulp_python/tests/functional/api/test_repair.py index cdfac9c6..ede9df46 100644 --- a/pulp_python/tests/functional/api/test_repair.py +++ b/pulp_python/tests/functional/api/test_repair.py @@ -54,9 +54,7 @@ def _create(content, remote, remote_2=None): "ra2.save(); " ) commands += "print(get_url(c))" - process = subprocess.run( - ["pulpcore-manager", "shell", "-c", commands], capture_output=True - ) + process = subprocess.run(["pulpcore-manager", "shell", "-c", commands], capture_output=True) assert process.returncode == 0 content_href = process.stdout.decode().strip() @@ -101,7 +99,7 @@ def test_metadata_repair_command( move_to_repository(python_repo.pulp_href, [content.pulp_href]) process = subprocess.run( ["pulpcore-manager", "repair-python-metadata", "--repositories", python_repo.pulp_href], - capture_output=True + capture_output=True, ) assert process.returncode == 0 output = process.stdout.decode().strip() @@ -136,9 +134,7 @@ def test_metadata_repair_endpoint( # Immediate content scipy_egg_filename = "scipy-1.1.0-cp27-none-win32.whl" - scipy_egg_url = urljoin( - urljoin(PYTHON_FIXTURES_URL, "packages/"), scipy_egg_filename - ) + scipy_egg_url = urljoin(urljoin(PYTHON_FIXTURES_URL, "packages/"), scipy_egg_filename) scipy_file = download_python_file(scipy_egg_filename, scipy_egg_url) scipy_data_0 = { "filename": scipy_egg_filename, @@ -175,16 +171,12 @@ def test_metadata_repair_endpoint( scipy_data_2 = scipy_data_1.copy() scipy_data_2["filename"] = "scipy-1.1.0-cp36-none-win32.whl" - scipy_data_2["sha256"] = ( - "0e9bb7efe5f051ea7212555b290e784b82f21ffd0f655405ac4f87e288b730b3" - ) + scipy_data_2["sha256"] = "0e9bb7efe5f051ea7212555b290e784b82f21ffd0f655405ac4f87e288b730b3" # 2. Create content celery_content = create_content_remote(celery_data, python_remote) scipy_content_0 = create_content_direct(scipy_file, scipy_data_0) - scipy_content_1 = create_content_remote( - scipy_data_1, python_remote, python_remote_bad - ) + scipy_content_1 = create_content_remote(scipy_data_1, python_remote, python_remote_bad) scipy_content_2 = create_content_remote(scipy_data_2, python_remote_bad) content_hrefs = {} @@ -200,9 +192,7 @@ def test_metadata_repair_endpoint( move_to_repository(python_repo.pulp_href, list(content_hrefs.values())) # 3. Repair metadata - response = python_bindings.RepositoriesPythonApi.repair_metadata( - python_repo.pulp_href - ) + response = python_bindings.RepositoriesPythonApi.repair_metadata(python_repo.pulp_href) monitor_task(response.task) # 4. Check new metadata diff --git a/pulp_python/tests/functional/api/test_sync.py b/pulp_python/tests/functional/api/test_sync.py index 0a846cf6..5069b108 100644 --- a/pulp_python/tests/functional/api/test_sync.py +++ b/pulp_python/tests/functional/api/test_sync.py @@ -213,7 +213,7 @@ def test_sync_package_type(python_repo_with_sync, python_remote_factory, python_ @pytest.mark.parallel def test_sync_platform_exclude( - python_repo_with_sync, python_remote_factory, python_content_summary + python_repo_with_sync, python_remote_factory, python_content_summary ): """ Tests for platform specific packages not being synced when specified @@ -226,8 +226,11 @@ def test_sync_platform_exclude( 1 any platform release """ # Tests that no windows packages are synced - remote = python_remote_factory(includes=["scipy"], exclude_platforms=["windows"], - prereleases=True, ) + remote = python_remote_factory( + includes=["scipy"], + exclude_platforms=["windows"], + prereleases=True, + ) repo = python_repo_with_sync(remote) summary = python_content_summary(repository_version=repo.latest_version_href) @@ -235,8 +238,11 @@ def test_sync_platform_exclude( assert summary.present["python.python"]["count"] == diff_count # Tests that no macos packages are synced - remote = python_remote_factory(includes=["scipy"], exclude_platforms=["macos"], - prereleases=True, ) + remote = python_remote_factory( + includes=["scipy"], + exclude_platforms=["macos"], + prereleases=True, + ) repo = python_repo_with_sync(remote) summary = python_content_summary(repository_version=repo.latest_version_href) @@ -244,8 +250,11 @@ def test_sync_platform_exclude( assert summary.present["python.python"]["count"] == diff_count # Tests that no linux packages are synced - remote = python_remote_factory(includes=["scipy"], exclude_platforms=["linux"], - prereleases=True, ) + remote = python_remote_factory( + includes=["scipy"], + exclude_platforms=["linux"], + prereleases=True, + ) repo = python_repo_with_sync(remote) summary = python_content_summary(repository_version=repo.latest_version_href) @@ -273,7 +282,7 @@ def test_sync_multiple_filters( includes=PYTHON_LG_PROJECT_SPECIFIER, package_types=["bdist_wheel"], keep_latest_packages=1, - prereleases=False + prereleases=False, ) repo = python_repo_with_sync(remote) diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index 7214c1a0..d65e62f6 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -25,9 +25,7 @@ "pylint", # matches 0 ] PYTHON_UNAVAILABLE_PACKAGE_COUNT = 5 -PYTHON_UNAVAILABLE_FIXTURE_SUMMARY = { - PYTHON_CONTENT_NAME: PYTHON_UNAVAILABLE_PACKAGE_COUNT -} +PYTHON_UNAVAILABLE_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_UNAVAILABLE_PACKAGE_COUNT} # no "name" field PYTHON_INVALID_SPECIFIER_NO_NAME = [ @@ -51,13 +49,9 @@ "Django", ] PYTHON_WITH_PRERELEASE_COUNT = 46 -PYTHON_WITH_PRERELEASE_FIXTURE_SUMMARY = { - PYTHON_CONTENT_NAME: PYTHON_WITH_PRERELEASE_COUNT -} +PYTHON_WITH_PRERELEASE_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_WITH_PRERELEASE_COUNT} PYTHON_WITHOUT_PRERELEASE_COUNT = 30 -PYTHON_WITHOUT_PRERELEASE_FIXTURE_SUMMARY = { - PYTHON_CONTENT_NAME: PYTHON_WITHOUT_PRERELEASE_COUNT -} +PYTHON_WITHOUT_PRERELEASE_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_WITHOUT_PRERELEASE_COUNT} # Specifier for basic sync / publish tests. PYTHON_XS_PROJECT_SPECIFIER = ["shelf-reader"] # matches 2 @@ -93,21 +87,15 @@ "aiohttp-3.2.1.tar.gz": "1b95d53f8dac13898f0a3e4af76f6f36d540fbfaefc4f4c9f43e436fa0e53d22", "aiohttp-3.2.0.tar.gz": "1be3903fe6a36d20492e74efb326522dd4702bf32b45ffc7acbc0fb34ab240a6", "Django-1.10.4.tar.gz": "fff7f062e510d812badde7cfc57745b7779edb4d209b2bc5ea8d954c22305c2b", - "Django-1.10.4-py2.py3-none-any.whl": - "a8e1a552205cda15023c39ecf17f7e525e96c5b0142e7879e8bd0c445351f2cc", + "Django-1.10.4-py2.py3-none-any.whl": "a8e1a552205cda15023c39ecf17f7e525e96c5b0142e7879e8bd0c445351f2cc", # noqa: E501 "Django-1.10.3.tar.gz": "6f92f08dee8a1bd7680e098a91bf5acd08b5cdfe74137f695b60fd79f4478c30", - "Django-1.10.3-py2.py3-none-any.whl": - "94426cc28d8721fbf13c333053f08d32427671a4ca7986f7030fc82bdf9c88c1", + "Django-1.10.3-py2.py3-none-any.whl": "94426cc28d8721fbf13c333053f08d32427671a4ca7986f7030fc82bdf9c88c1", # noqa: E501 "Django-1.10.2.tar.gz": "e127f12a0bfb34843b6e8c82f91e26fff6445a7ca91d222c0794174cf97cbce1", - "Django-1.10.2-py2.py3-none-any.whl": - "4d48ab8e84a7c8b2bc4b2f4f199bc3a8bfcc9cbdbc29e355ac5c44a501d73a1a", + "Django-1.10.2-py2.py3-none-any.whl": "4d48ab8e84a7c8b2bc4b2f4f199bc3a8bfcc9cbdbc29e355ac5c44a501d73a1a", # noqa: E501 "Django-1.10.1.tar.gz": "d6e6c5b25cb67f46afd7c82f536529b11981183423dad8932e15bce93d1a24f3", - "Django-1.10.1-py2.py3-none-any.whl": - "3d689905cd0635bbb33b87f9a5df7ca70a3db206faae4ec58cda5e7f5f47050d", - "celery-4.2.0-py2.py3-none-any.whl": - "2082cbd82effa8ac8a8a58977d70bb203a9f362817e3b66f4578117b9f93d8a9", - "celery-4.1.1-py2.py3-none-any.whl": - "6fc4678d1692af97e137b2a9f1c04efd8e7e2fb7134c5c5ad60738cdd927762f", + "Django-1.10.1-py2.py3-none-any.whl": "3d689905cd0635bbb33b87f9a5df7ca70a3db206faae4ec58cda5e7f5f47050d", # noqa: E501 + "celery-4.2.0-py2.py3-none-any.whl": "2082cbd82effa8ac8a8a58977d70bb203a9f362817e3b66f4578117b9f93d8a9", # noqa: E501 + "celery-4.1.1-py2.py3-none-any.whl": "6fc4678d1692af97e137b2a9f1c04efd8e7e2fb7134c5c5ad60738cdd927762f", # noqa: E501 } PYTHON_MD_PROJECT_SPECIFIER = [ @@ -156,9 +144,7 @@ # Intended to be used with the XS specifier PYTHON_WHEEL_FILENAME = "shelf_reader-0.1-py2-none-any.whl" -PYTHON_WHEEL_URL = urljoin( - urljoin(PYTHON_FIXTURES_URL, "packages/"), PYTHON_WHEEL_FILENAME -) +PYTHON_WHEEL_URL = urljoin(urljoin(PYTHON_FIXTURES_URL, "packages/"), PYTHON_WHEEL_FILENAME) PYTHON_WHEEL_SHA256 = "2eceb1643c10c5e4a65970baf63bde43b79cbdac7de81dae853ce47ab05197e9" PYTHON_XS_FIXTURE_CHECKSUMS = { @@ -266,7 +252,7 @@ "download_concurrency": 3, "policy": "on_demand", "prereleases": False, - "excludes": ["example1", "example2"] + "excludes": ["example1", "example2"], } BANDERSNATCH_CONF = b""" @@ -330,7 +316,7 @@ "requires_python": None, "size": 22455, "yanked": False, - "yanked_reason": None + "yanked_reason": None, } SHELF_SDIST_PYTHON_DOWNLOAD = { @@ -348,7 +334,7 @@ "requires_python": None, "size": 19097, "yanked": False, - "yanked_reason": None + "yanked_reason": None, } SHELF_0DOT1_RELEASE = [SHELF_BDIST_PYTHON_DOWNLOAD, SHELF_SDIST_PYTHON_DOWNLOAD] @@ -356,9 +342,6 @@ SHELF_PYTHON_JSON = { "info": PYTHON_INFO_DATA, "last_serial": 0, - "releases": { - "0.1": SHELF_0DOT1_RELEASE - }, - "urls": SHELF_0DOT1_RELEASE - + "releases": {"0.1": SHELF_0DOT1_RELEASE}, + "urls": SHELF_0DOT1_RELEASE, } diff --git a/pulp_python/tests/functional/utils.py b/pulp_python/tests/functional/utils.py index 619b9bbb..a3a2ede3 100644 --- a/pulp_python/tests/functional/utils.py +++ b/pulp_python/tests/functional/utils.py @@ -16,6 +16,7 @@ def ensure_simple(simple_url, packages, sha_digests=None): in the simple index and thus be accessible from the distribution, but if one can't see it how would one know that it's there?* """ + def explore_links(page_url, page_name, links_found, msgs): legit_found_links = [] page = html.fromstring(requests.get(page_url).text) @@ -48,8 +49,16 @@ def explore_links(page_url, page_name, links_found, msgs): package = package_link.split("/")[-1] if sha_digests[package] != sha: msgs += f"\nRelease has bad sha256 attached to it {package}" - msgs += "".join(map(lambda x: f"\nSimple link not found for {x}", - [name for name, val in packages_found.items() if not val])) - msgs += "".join(map(lambda x: f"\nReleases link not found for {x}", - [name for name, val in releases_found.items() if not val])) + msgs += "".join( + map( + lambda x: f"\nSimple link not found for {x}", + [name for name, val in packages_found.items() if not val], + ) + ) + msgs += "".join( + map( + lambda x: f"\nReleases link not found for {x}", + [name for name, val in releases_found.items() if not val], + ) + ) return len(msgs) == 0, msgs diff --git a/pyproject.toml b/pyproject.toml index 22a04871..429a6ec0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,3 +122,6 @@ replace = "version = \"{new_version}\"" filename = "./pyproject.toml" search = "version = \"{current_version}\"" replace = "version = \"{new_version}\"" + +[tool.black] +line-length = 100 diff --git a/template_config.yml b/template_config.yml index f3e7735e..cdd00e6c 100644 --- a/template_config.yml +++ b/template_config.yml @@ -4,7 +4,7 @@ # generated with plugin_template api_root: /pulp/ -black: false +black: true check_commit_message: true check_gettext: true check_manifest: true @@ -74,4 +74,3 @@ test_reroute: true test_s3: true test_storages_compat_layer: true use_issue_template: true -