From 4f35e418ac704f3a7af50f55a8855072cdeccda8 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 21 Aug 2025 17:59:34 -0400 Subject: [PATCH] Update Azure and S3 storage settings for 3.18 Set storage settings in template_config.yml based on pulpcore requirement (>=3.49.0,<3.100). Storage formats: Azure (new STORAGES), S3 (new STORAGES) This addresses the recent change where plugin-template stopped automatically setting test scenario storage settings and now requires plugins to define them in their template_config.yml. --- .ci/ansible/settings.py.j2 | 124 --------------------------- .github/workflows/scripts/install.sh | 6 +- template_config.yml | 30 ++++++- 3 files changed, 31 insertions(+), 129 deletions(-) diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 2d30839a..dfe2851d 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -26,127 +26,3 @@ API_ROOT = {{ api_root | repr }} {% endfor %} {% endif %} -{# ======================================= -Macros for legacy and new storage settings -========================================== -#} - -{%- macro s3_settings(legacy) -%} - {%- if legacy %} -DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" -AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" -AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" -AWS_S3_REGION_NAME = "eu-central-1" -AWS_S3_ADDRESSING_STYLE = "path" -AWS_S3_SIGNATURE_VERSION = "s3v4" -AWS_STORAGE_BUCKET_NAME = "pulp3" -AWS_S3_ENDPOINT_URL = "http://minio:9000" -AWS_DEFAULT_ACL = "@none None" - {%- else %} -STORAGES = { - "default": { - "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", - "OPTIONS": { - "access_key": "{{ minio_access_key }}", - "secret_key": "{{ minio_secret_key }}", - "region_name": "eu-central-1", - "addressing_style": "path", - "signature_version": "s3v4", - "bucket_name": "pulp3", - "endpoint_url": "http://minio:9000", - "default_acl": "@none None", - }, - }, - "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", - }, -} - {%- endif %} -{%- endmacro -%} - -{%- macro azure_settings(legacy) -%} - {%- if legacy %} -DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" -AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" -AZURE_ACCOUNT_NAME = "devstoreaccount1" -AZURE_CONTAINER = "pulp-test" -AZURE_LOCATION = "pulp3" -AZURE_OVERWRITE_FILES = True -AZURE_URL_EXPIRATION_SECS = 120 -AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' - {%- else %} -STORAGES = { - "default": { - "BACKEND": "storages.backends.azure_storage.AzureStorage", - "OPTIONS": { - "account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", - "account_name": "devstoreaccount1", - "location": "pulp3", - "azure_container": "pulp-test", - "overwrite_files": True, - "expiration_secs": 120, - "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", - }, - }, - "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", - }, -} - {%- endif %} -{%- endmacro -%} - -{%- macro gcp_settings(legacy) -%} - {%- if legacy %} -DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage" -GS_BUCKET_NAME = "gcppulp" -GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443" - {%- else %} -STORAGES = { - "default": { - "BACKEND": "storages.backends.gcloud.GoogleCloudStorage", - "OPTIONS": { - "bucket_name": "gcppulp", - "custom_endpoint": "http://ci-gcp:4443", - }, - }, - "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", - }, -} - {%- endif %} -{%- endmacro -%} - -{#- ========================================== -Render according to test_storages_compat_layer -============================================== - -Case 1) test_storages_compat_layer is unset -- All storages render the legacy setting -- Branches using pulpcore <3.70 must leave this key unset (use legacy) - -Case 2) test_storages_compat_layer is True -- To tests both work, only one setting uses the new storage setting -- Branches using pulpcore >=3.70,<3.85 must set this key to True (test both) - -Case 3) test_storages_compat_layer is False -- All storages render the new setting -- Branches using pulpcore >=3.85 must set this key to False (use new) --#} - -{% if s3_test | default(false) or azure_test | default(false) or gcp_test | default(false)%} -MEDIA_ROOT="" -{% endif %} -{%- if test_storages_compat_layer is not defined -%} - {%- if s3_test | default(false) -%}{{ s3_settings(legacy=True) }}{%- endif -%} - {%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%} - {%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%} -{%- else -%} - {%- if test_storages_compat_layer is true -%} - {%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%} - {%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%} - {%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%} - {%- elif test_storages_compat_layer is false -%} - {%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%} - {%- if azure_test | default(false) -%}{{ azure_settings(legacy=False) }}{%- endif -%} - {%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=False) }}{%- endif -%} - {%- endif -%} -{%- endif -%} diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 7a0fed78..1d6404b1 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -95,9 +95,8 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"domain_enabled": true}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true}\ pulp_scenario_env: {}\ -test_storages_compat_layer: false\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" fi @@ -110,9 +109,8 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"content_origin": null, "domain_enabled": true}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "content_origin": null, "domain_enabled": true}\ pulp_scenario_env: {}\ -test_storages_compat_layer: false\ ' vars/main.yaml fi diff --git a/template_config.yml b/template_config.yml index 217abf83..5f86a8c5 100644 --- a/template_config.yml +++ b/template_config.yml @@ -47,10 +47,39 @@ pulp_settings: orphan_protection_time: 0 pypi_api_hostname: https://pulp:443 pulp_settings_azure: + MEDIA_ROOT: '' + STORAGES: + default: + BACKEND: storages.backends.azure_storage.AzureStorage + OPTIONS: + account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + account_name: devstoreaccount1 + azure_container: pulp-test + connection_string: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1; + expiration_secs: 120 + location: pulp3 + overwrite_files: true + staticfiles: + BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage content_origin: null domain_enabled: true pulp_settings_gcp: null pulp_settings_s3: + MEDIA_ROOT: '' + STORAGES: + default: + BACKEND: storages.backends.s3boto3.S3Boto3Storage + OPTIONS: + access_key: AKIAIT2Z5TDYPX3ARJBA + addressing_style: path + bucket_name: pulp3 + default_acl: '@none' + endpoint_url: http://minio:9000 + region_name: eu-central-1 + secret_key: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS + signature_version: s3v4 + staticfiles: + BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage domain_enabled: true pydocstyle: true release_email: pulp-infra@redhat.com @@ -73,6 +102,5 @@ test_lowerbounds: true test_performance: false test_reroute: true test_s3: true -test_storages_compat_layer: false use_issue_template: true