Skip to content

Update Azure and S3 storage settings for main #945

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gerrod3 merged 1 commit into from
Aug 22, 2025
Merged

Update Azure and S3 storage settings for main #945

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 0 additions & 124 deletions .ci/ansible/settings.py.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,127 +26,3 @@ API_ROOT = {{ api_root | repr }}
{% endfor %}
{% endif %}

{# =======================================
Macros for legacy and new storage settings
========================================== -#}

{%- macro s3_settings(legacy) -%}
{%- if legacy %}
DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
AWS_S3_REGION_NAME = "eu-central-1"
AWS_S3_ADDRESSING_STYLE = "path"
AWS_S3_SIGNATURE_VERSION = "s3v4"
AWS_STORAGE_BUCKET_NAME = "pulp3"
AWS_S3_ENDPOINT_URL = "http://minio:9000"
AWS_DEFAULT_ACL = "@none None"
{%- else %}
STORAGES = {
"default": {
"BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
"OPTIONS": {
"access_key": "{{ minio_access_key }}",
"secret_key": "{{ minio_secret_key }}",
"region_name": "eu-central-1",
"addressing_style": "path",
"signature_version": "s3v4",
"bucket_name": "pulp3",
"endpoint_url": "http://minio:9000",
"default_acl": "@none None",
},
},
"staticfiles": {
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
},
}
{%- endif %}
{%- endmacro -%}

{%- macro azure_settings(legacy) -%}
{%- if legacy %}
DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
AZURE_ACCOUNT_NAME = "devstoreaccount1"
AZURE_CONTAINER = "pulp-test"
AZURE_LOCATION = "pulp3"
AZURE_OVERWRITE_FILES = True
AZURE_URL_EXPIRATION_SECS = 120
AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;'
{%- else %}
STORAGES = {
"default": {
"BACKEND": "storages.backends.azure_storage.AzureStorage",
"OPTIONS": {
"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
"account_name": "devstoreaccount1",
"location": "pulp3",
"azure_container": "pulp-test",
"overwrite_files": True,
"expiration_secs": 120,
"connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;",
},
},
"staticfiles": {
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
},
}
{%- endif %}
{%- endmacro -%}

{%- macro gcp_settings(legacy) -%}
{%- if legacy %}
DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
GS_BUCKET_NAME = "gcppulp"
GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
{%- else %}
STORAGES = {
"default": {
"BACKEND": "storages.backends.gcloud.GoogleCloudStorage",
"OPTIONS": {
"bucket_name": "gcppulp",
"custom_endpoint": "http://ci-gcp:4443",
},
},
"staticfiles": {
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
},
}
{%- endif %}
{%- endmacro -%}

{#- ==========================================
Render according to test_storages_compat_layer
==============================================

Case 1) test_storages_compat_layer is unset
- All storages render the legacy setting
- Branches using pulpcore <3.70 must leave this key unset (use legacy)

Case 2) test_storages_compat_layer is True
- To tests both work, only one setting uses the new storage setting
- Branches using pulpcore >=3.70,<3.85 must set this key to True (test both)

Case 3) test_storages_compat_layer is False
- All storages render the new setting
- Branches using pulpcore >=3.85 must set this key to False (use new)
-#}

{% if s3_test | default(false) or azure_test | default(false) or gcp_test | default(false)%}
MEDIA_ROOT=""
{% endif %}
{%- if test_storages_compat_layer is not defined -%}
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=True) }}{%- endif -%}
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%}
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%}
{%- else -%}
{%- if test_storages_compat_layer is true -%}
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%}
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=True) }}{%- endif -%}
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=True) }}{%- endif -%}
{%- elif test_storages_compat_layer is false -%}
{%- if s3_test | default(false) -%}{{ s3_settings(legacy=False) }}{%- endif -%}
{%- if azure_test | default(false) -%}{{ azure_settings(legacy=False) }}{%- endif -%}
{%- if gcp_test | default(false) -%}{{ gcp_settings(legacy=False) }}{%- endif -%}
{%- endif -%}
{%- endif -%}
6 changes: 2 additions & 4 deletions .github/workflows/scripts/install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,8 @@ if [ "$TEST" = "s3" ]; then
sed -i -e '$a s3_test: true\
minio_access_key: "'$MINIO_ACCESS_KEY'"\
minio_secret_key: "'$MINIO_SECRET_KEY'"\
pulp_scenario_settings: {"domain_enabled": true}\
pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true}\
pulp_scenario_env: {}\
test_storages_compat_layer: false\
' vars/main.yaml
export PULP_API_ROOT="/rerouted/djnd/"
fi
Expand All @@ -110,9 +109,8 @@ if [ "$TEST" = "azure" ]; then
- ./azurite:/etc/pulp\
command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml
sed -i -e '$a azure_test: true\
pulp_scenario_settings: {"content_origin": null, "domain_enabled": true}\
pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "content_origin": null, "domain_enabled": true}\
pulp_scenario_env: {}\
test_storages_compat_layer: false\
' vars/main.yaml
fi

Expand Down
30 changes: 29 additions & 1 deletion template_config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,39 @@ pulp_settings:
orphan_protection_time: 0
pypi_api_hostname: https://pulp:443
pulp_settings_azure:
MEDIA_ROOT: ''
STORAGES:
default:
BACKEND: storages.backends.azure_storage.AzureStorage
OPTIONS:
account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
account_name: devstoreaccount1
azure_container: pulp-test
connection_string: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;
expiration_secs: 120
location: pulp3
overwrite_files: true
staticfiles:
BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
content_origin: null
domain_enabled: true
pulp_settings_gcp: null
pulp_settings_s3:
MEDIA_ROOT: ''
STORAGES:
default:
BACKEND: storages.backends.s3boto3.S3Boto3Storage
OPTIONS:
access_key: AKIAIT2Z5TDYPX3ARJBA
addressing_style: path
bucket_name: pulp3
default_acl: '@none'
endpoint_url: http://minio:9000
region_name: eu-central-1
secret_key: fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS
signature_version: s3v4
staticfiles:
BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage
domain_enabled: true
pydocstyle: true
release_email: pulp-infra@redhat.com
Expand All @@ -73,6 +102,5 @@ test_lowerbounds: true
test_performance: false
test_reroute: true
test_s3: true
test_storages_compat_layer: false
use_issue_template: true