diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 7a73ea2f..2d30839a 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -26,10 +26,22 @@ API_ROOT = {{ api_root | repr }} {% endfor %} {% endif %} -{% if s3_test | default(false) %} -MEDIA_ROOT: "" -S3_USE_SIGV4 = True -{% if test_storages_compat_layer is defined and test_storages_compat_layer %} +{# ======================================= +Macros for legacy and new storage settings +========================================== -#} + +{%- macro s3_settings(legacy) -%} + {%- if legacy %} +DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" +AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" +AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" +AWS_S3_REGION_NAME = "eu-central-1" +AWS_S3_ADDRESSING_STYLE = "path" +AWS_S3_SIGNATURE_VERSION = "s3v4" +AWS_STORAGE_BUCKET_NAME = "pulp3" +AWS_S3_ENDPOINT_URL = "http://minio:9000" +AWS_DEFAULT_ACL = "@none None" + {%- else %} STORAGES = { "default": { "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", @@ -48,22 +60,12 @@ STORAGES = { "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", }, } -{% else %} -DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" -AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" -AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" -AWS_S3_REGION_NAME = "eu-central-1" -AWS_S3_ADDRESSING_STYLE = "path" -AWS_S3_SIGNATURE_VERSION = "s3v4" -AWS_STORAGE_BUCKET_NAME = "pulp3" -AWS_S3_ENDPOINT_URL = "http://minio:9000" -AWS_DEFAULT_ACL = "@none None" -{% endif %} -{% endif %} + {%- endif %} +{%- endmacro -%} -{% if azure_test | default(false) %} +{%- macro azure_settings(legacy) -%} + {%- if legacy %} DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" -MEDIA_ROOT = "" AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" AZURE_ACCOUNT_NAME = "devstoreaccount1" AZURE_CONTAINER = "pulp-test" @@ -71,11 +73,80 @@ AZURE_LOCATION = "pulp3" AZURE_OVERWRITE_FILES = True AZURE_URL_EXPIRATION_SECS = 120 AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' -{% endif %} + {%- else %} +STORAGES = { + "default": { + "BACKEND": "storages.backends.azure_storage.AzureStorage", + "OPTIONS": { + "account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + "account_name": "devstoreaccount1", + "location": "pulp3", + "azure_container": "pulp-test", + "overwrite_files": True, + "expiration_secs": 120, + "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", + }, + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, +} + {%- endif %} +{%- endmacro -%} -{% if gcp_test | default(false) %} +{%- macro gcp_settings(legacy) -%} + {%- if legacy %} DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" -MEDIA_ROOT = "" GS_BUCKET_NAME = "gcppulp" GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443" + {%- else %} +STORAGES = { + "default": { + "BACKEND": "storages.backends.gcloud.GoogleCloudStorage", + "OPTIONS": { + "bucket_name": "gcppulp", + "custom_endpoint": "http://ci-gcp:4443", + }, + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, +} + {%- endif %} +{%- endmacro -%} + +{#- ========================================== +Render according to test_storages_compat_layer +============================================== + +Case 1) test_storages_compat_layer is unset +- All storages render the legacy setting +- Branches using pulpcore <3.70 must leave this key unset (use legacy) + +Case 2) test_storages_compat_layer is True +- To tests both work, only one setting uses the new storage setting +- Branches using pulpcore >=3.70,<3.85 must set this key to True (test both) + +Case 3) test_storages_compat_layer is False +- All storages render the new setting +- Branches using pulpcore >=3.85 must set this key to False (use new) +-#} + +{% if s3_test | default(false) or azure_test | default(false) or gcp_test | default(false)%} +MEDIA_ROOT="" {% endif %} +{%- if test_storages_compat_layer is not defined -%} + {%- if s3_test | default(false) -%}{{ s3_settings(legacy=True) }}{%- endif -%} + {%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%} + {%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%} +{%- else -%} + {%- if test_storages_compat_layer is true -%} + {%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%} + {%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%} + {%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%} + {%- elif test_storages_compat_layer is false -%} + {%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%} + {%- if azure_test | default(false) -%}{{ azure_settings(legacy=False) }}{%- endif -%} + {%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=False) }}{%- endif -%} + {%- endif -%} +{%- endif -%} diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index c63631d8..7a0fed78 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -97,7 +97,7 @@ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ pulp_scenario_settings: {"domain_enabled": true}\ pulp_scenario_env: {}\ -test_storages_compat_layer: true\ +test_storages_compat_layer: false\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" fi @@ -112,6 +112,7 @@ if [ "$TEST" = "azure" ]; then sed -i -e '$a azure_test: true\ pulp_scenario_settings: {"content_origin": null, "domain_enabled": true}\ pulp_scenario_env: {}\ +test_storages_compat_layer: false\ ' vars/main.yaml fi diff --git a/CHANGES/+3.85-compat.feature b/CHANGES/+3.85-compat.feature new file mode 100644 index 00000000..a5a08e2e --- /dev/null +++ b/CHANGES/+3.85-compat.feature @@ -0,0 +1 @@ +Bump pulpcore upperbound to <3.100. pulp_python is now a Python >=3.11 project. diff --git a/pulp_python/app/migrations/0001_squashed_0010_update_json_field.py b/pulp_python/app/migrations/0001_squashed_0010_update_json_field.py new file mode 100644 index 00000000..fe0bc2e3 --- /dev/null +++ b/pulp_python/app/migrations/0001_squashed_0010_update_json_field.py @@ -0,0 +1,261 @@ +# Generated by Django 4.2.23 on 2025-08-12 18:24 + +import django.contrib.postgres.fields +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + replaces = [ + ("python", "0001_initial"), + ("python", "0002_pythonpackagecontent_python_version"), + ("python", "0003_new_sync_filters"), + ("python", "0004_DATA_swap_distribution_model"), + ("python", "0005_pythonpackagecontent_sha256"), + ("python", "0006_pythonrepository_autopublish"), + ("python", "0007_pythonpackagecontent_mv-2-1"), + ("python", "0008_pythonpackagecontent_unique_sha256"), + ("python", "0009_pythondistribution_allow_uploads"), + ("python", "0010_update_json_field"), + ] + + initial = True + + dependencies = [ + ("core", "0091_systemid"), + ] + + operations = [ + migrations.CreateModel( + name="PythonRemote", + fields=[ + ( + "remote_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythonremote", + serialize=False, + to="core.remote", + ), + ), + ("prereleases", models.BooleanField(default=False)), + ("includes", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("excludes", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ( + "exclude_platforms", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(blank=True, max_length=10), + choices=[ + ("windows", "windows"), + ("macos", "macos"), + ("freebsd", "freebsd"), + ("linux", "linux"), + ], + default=list, + size=None, + ), + ), + ("keep_latest_packages", models.IntegerField(default=0)), + ( + "package_types", + django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(blank=True, max_length=15), + choices=[ + ("bdist_dmg", "bdist_dmg"), + ("bdist_dumb", "bdist_dumb"), + ("bdist_egg", "bdist_egg"), + ("bdist_msi", "bdist_msi"), + ("bdist_rpm", "bdist_rpm"), + ("bdist_wheel", "bdist_wheel"), + ("bdist_wininst", "bdist_wininst"), + ("sdist", "sdist"), + ], + default=list, + size=None, + ), + ), + ], + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + }, + bases=("core.remote",), + ), + migrations.CreateModel( + name="PythonPublication", + fields=[ + ( + "publication_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythonpublication", + serialize=False, + to="core.publication", + ), + ), + ], + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + }, + bases=("core.publication",), + ), + migrations.CreateModel( + name="PythonDistribution", + fields=[ + ( + "distribution_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythondistribution", + serialize=False, + to="core.distribution", + ), + ), + ], + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + }, + bases=("core.distribution",), + ), + migrations.CreateModel( + name="PythonRepository", + fields=[ + ( + "repository_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythonrepository", + serialize=False, + to="core.repository", + ), + ), + ("autopublish", models.BooleanField(default=False)), + ], + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + }, + bases=("core.repository",), + ), + migrations.CreateModel( + name="PythonPackageContent", + fields=[ + ( + "content_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + related_name="python_pythonpackagecontent", + serialize=False, + to="core.content", + ), + ), + ("filename", models.TextField(db_index=True)), + ( + "packagetype", + models.TextField( + choices=[ + ("bdist_dmg", "bdist_dmg"), + ("bdist_dumb", "bdist_dumb"), + ("bdist_egg", "bdist_egg"), + ("bdist_msi", "bdist_msi"), + ("bdist_rpm", "bdist_rpm"), + ("bdist_wheel", "bdist_wheel"), + ("bdist_wininst", "bdist_wininst"), + ("sdist", "sdist"), + ] + ), + ), + ("name", models.TextField()), + ("version", models.TextField()), + ("metadata_version", models.TextField()), + ("summary", models.TextField()), + ("description", models.TextField()), + ("keywords", models.TextField()), + ("home_page", models.TextField()), + ("download_url", models.TextField()), + ("author", models.TextField()), + ("author_email", models.TextField()), + ("maintainer", models.TextField()), + ("maintainer_email", models.TextField()), + ("license", models.TextField()), + ("requires_python", models.TextField()), + ("project_url", models.TextField()), + ("platform", models.TextField()), + ("supported_platform", models.TextField()), + ("requires_dist", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("provides_dist", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("obsoletes_dist", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("requires_external", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("classifiers", django.contrib.postgres.fields.jsonb.JSONField(default=list)), + ("python_version", models.TextField()), + ("sha256", models.CharField(db_index=True, max_length=64, unique=True)), + ("description_content_type", models.TextField()), + ("project_urls", django.contrib.postgres.fields.jsonb.JSONField(default=dict)), + ], + options={ + "default_related_name": "%(app_label)s_%(model_name)s", + "unique_together": {("sha256",)}, + }, + bases=("core.content",), + ), + migrations.AddField( + model_name="pythondistribution", + name="allow_uploads", + field=models.BooleanField(default=True), + ), + migrations.AlterField( + model_name="pythonpackagecontent", + name="classifiers", + field=models.JSONField(default=list), + ), + migrations.AlterField( + model_name="pythonpackagecontent", + name="obsoletes_dist", + field=models.JSONField(default=list), + ), + migrations.AlterField( + model_name="pythonpackagecontent", + name="project_urls", + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name="pythonpackagecontent", + name="provides_dist", + field=models.JSONField(default=list), + ), + migrations.AlterField( + model_name="pythonpackagecontent", + name="requires_dist", + field=models.JSONField(default=list), + ), + migrations.AlterField( + model_name="pythonpackagecontent", + name="requires_external", + field=models.JSONField(default=list), + ), + migrations.AlterField( + model_name="pythonremote", + name="excludes", + field=models.JSONField(default=list), + ), + migrations.AlterField( + model_name="pythonremote", + name="includes", + field=models.JSONField(default=list), + ), + ] diff --git a/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py b/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py index dd658c1c..e6948d12 100644 --- a/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py +++ b/pulp_python/app/migrations/0004_DATA_swap_distribution_model.py @@ -76,6 +76,7 @@ class Migration(migrations.Migration): migrations.RunPython( code=migrate_data_from_old_model_to_new_model_up, reverse_code=migrate_data_from_old_model_to_new_model_down, + elidable=True, ), migrations.DeleteModel( name="PythonDistribution", diff --git a/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py b/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py index 585cc4ec..af48b1ab 100644 --- a/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py +++ b/pulp_python/app/migrations/0005_pythonpackagecontent_sha256.py @@ -47,5 +47,7 @@ class Migration(migrations.Migration): field=models.CharField(max_length=64, default=""), preserve_default=False, ), - migrations.RunPython(add_sha256_to_current_models, migrations.RunPython.noop), + migrations.RunPython( + add_sha256_to_current_models, migrations.RunPython.noop, elidable=True + ), ] diff --git a/pyproject.toml b/pyproject.toml index d4bb83ef..e82983ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,16 +20,15 @@ classifiers=[ "Framework :: Django", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] -requires-python = ">=3.9" +requires-python = ">=3.11" dependencies = [ - "pulpcore>=3.49.0,<3.85", + "pulpcore>=3.49.0,<3.100", "pkginfo>=1.12.0,<1.13.0", - "bandersnatch>=6.3.0,<6.4", # Anything >=6.4 requires Python 3.10+ + "bandersnatch>=6.3.0,<6.6", # 6.6 has breaking changes "pypi-simple>=1.5.0,<2.0", ] diff --git a/template_config.yml b/template_config.yml index 705fa106..217abf83 100644 --- a/template_config.yml +++ b/template_config.yml @@ -73,6 +73,6 @@ test_lowerbounds: true test_performance: false test_reroute: true test_s3: true -test_storages_compat_layer: true +test_storages_compat_layer: false use_issue_template: true