Update CI

Author: mdellweg

.bumpversion.cfg

@@ -1,37 +1,47 @@
-FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }}
+FROM {{ ci_base | default(pulp_default_container) }}
 
 # Add source directories to container
 {% for item in plugins %}
-{% if item.source.startswith("./") or item.ci_requirements | default(false) %}
 ADD ./{{ item.name }} ./{{ item.name }}
-{% endif %}
 {% endfor %}
 
 # Install python packages
+# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
-RUN pip3 install
-{%- if stream_test | default(false) -%}
-{{ " " }}django-storages[sftp]
-{%- endif -%}
+# This MUST be the ONLY call to pip install in inside the container.
+RUN pip3 install --upgrade pip setuptools wheel && \
+  rm -rf /root/.cache/pip && \
+  pip3 install
 {%- if s3_test | default(false) -%}
-{{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
-{%- endif -%}
-{%- if azure_test | default(false) -%}
-{{ " " }}django-storages[azure]>=1.12.2
-{%- endif -%}
-{%- if gcp_test | default(false) -%}
-{{ " " }}django-storages[google]>=1.13.2
+{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue
 {%- endif -%}
 {%- for item in plugins -%}
-{%- if item.name == "pulp-certguard" -%}
-{{ " " }}python-dateutil rhsm
-{%- endif -%}
 {{ " " }}{{ item.source }}
+{%- if item.upperbounds | default(false) -%}
+{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt
+{%- endif -%}
+{%- if item.lowerbounds | default(false) -%}
+{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt
+{%- endif -%}
 {%- if item.ci_requirements | default(false) -%}
 {{ " " }}-r ./{{ item.name }}/ci_requirements.txt
 {%- endif -%}
 {%- endfor %}
+{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \
+  rm -rf /root/.cache/pip
+
+{% if pulp_env is defined and pulp_env %}
+{% for key, value in pulp_env.items() %}
+ENV {{ key | upper }}={{ value }}
+{% endfor %}
+{% endif %}
+
+{% if pulp_scenario_env is defined and pulp_scenario_env %}
+{% for key, value in pulp_scenario_env.items() %}
+ENV {{ key | upper }}={{ value }}
+{% endfor %}
+{% endif %}
 
 USER pulp:pulp
 RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \

.ci/ansible/Containerfile.j2

@@ -26,31 +26,40 @@ API_ROOT = {{ api_root | repr }}
 {% endfor %}
 {% endif %}
 
-{% if stream_test | default(false) -%}
-REDIRECT_TO_OBJECT_STORAGE = False
-DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.PulpSFTPStorage"
-MEDIA_ROOT = ""
-SFTP_STORAGE_HOST = "ci-sftp"
-SFTP_STORAGE_ROOT = "/storage/"
-SFTP_STORAGE_PARAMS = {
-    "username": "foo",
-    "key_filename": "/keys/id_ed25519",
-}
-{%- endif %}
-
 {% if s3_test | default(false) %}
+MEDIA_ROOT: ""
+S3_USE_SIGV4 = True
+{% if test_storages_compat_layer is defined and test_storages_compat_layer %}
+STORAGES = {
+    "default": {
+        "BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
+        "OPTIONS": {
+            "access_key": "{{ minio_access_key }}",
+            "secret_key": "{{ minio_secret_key }}",
+            "region_name": "eu-central-1",
+            "addressing_style": "path",
+            "signature_version": "s3v4",
+            "bucket_name": "pulp3",
+            "endpoint_url": "http://minio:9000",
+            "default_acl": "@none None",
+        },
+    },
+    "staticfiles": {
+      "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+    },
+}
+{% else %}
 DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
-MEDIA_ROOT = ""
 AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
 AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
 AWS_S3_REGION_NAME = "eu-central-1"
 AWS_S3_ADDRESSING_STYLE = "path"
-S3_USE_SIGV4 = True
 AWS_S3_SIGNATURE_VERSION = "s3v4"
 AWS_STORAGE_BUCKET_NAME = "pulp3"
 AWS_S3_ENDPOINT_URL = "http://minio:9000"
 AWS_DEFAULT_ACL = "@none None"
 {% endif %}
+{% endif %}
 
 {% if azure_test | default(false) %}
 DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage"
@@ -61,7 +70,7 @@ AZURE_CONTAINER =  "pulp-test"
 AZURE_LOCATION = "pulp3"
 AZURE_OVERWRITE_FILES = True
 AZURE_URL_EXPIRATION_SECS = 120
-AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;'
+AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;'
 {% endif %}
 
 {% if gcp_test | default(false) %}

.ci/ansible/settings.py.j2

@@ -15,14 +15,6 @@
         - ssh
         - ~/.config/pulp_smash
 
-    - name: Generate an OpenSSH keypair
-      community.crypto.openssh_keypair:
-        path: ssh/id_ed25519
-        type: ed25519
-        owner: 700  # pulp in the container
-      become: true
-      when: stream_test | default(false)
-
     - name: "Generate Pulp Settings"
       template:
         src: settings.py.j2
@@ -91,22 +83,14 @@
           command: "docker logs pulp"
           failed_when: true
 
-    - block:
-        - name: "Check version of component being tested"
-          assert:
-            that:
-              - (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] | canonical_semver == (component_version | canonical_semver)
-            fail_msg: |
-              Component {{ component_name }} was expected to be installed in version {{ component_version }}.
-              Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] }}.
-      rescue:
-        - name: "Check version of component being tested (legacy)"
-          assert:
-            that:
-              - (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] | canonical_semver == (component_version | canonical_semver)
-            fail_msg: |
-              Component {{ legacy_component_name }} was expected to be installed in version {{ component_version }}.
-              Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] }}.
+    - name: "Check version of component being tested"
+      assert:
+        that:
+          - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver)
+        fail_msg: |
+          Component {{ item.app_label }} was expected to be installed in version {{ component_version }}.
+          Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}.
+      loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}"
 
     - name: "Set pulp password in .netrc"
       copy:

.ci/ansible/start_container.yaml

@@ -0,0 +1,10 @@
+# Pulpcore versions without the openapi command do no longer work in the CI
+pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*
+
+
+tablib!=3.6.0
+# 3.6.0: This release introduced a regression removing the "html" optional dependency.
+
+
+multidict!=6.3.0
+# This release failed the lower bounds test for some case sensitivity in CIMultiDict.

.ci/assets/ci_constraints.txt

@@ -1,3 +1,3 @@
-bump2version
+bump-my-version
 gitpython
 towncrier

.ci/assets/release_requirements.txt

@@ -0,0 +1,139 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_python' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import argparse
+import fileinput
+import urllib.request
+import sys
+from packaging.requirements import Requirement
+from packaging.version import Version
+import yaml
+
+try:
+    import tomllib
+except ImportError:
+    import tomli as tomllib
+
+
+CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml"
+
+
+def fetch_pulpcore_upper_bound(requirement):
+    with urllib.request.urlopen(CORE_TEMPLATE_URL) as f:
+        template = yaml.safe_load(f.read())
+    supported_versions = template["supported_release_branches"]
+    supported_versions.append(template["latest_release_branch"])
+    applicable_versions = sorted(
+        requirement.specifier.filter((Version(v) for v in supported_versions))
+    )
+    if len(applicable_versions) == 0:
+        raise Exception("No supported pulpcore version in required range.")
+    return f"{requirement.name}~={applicable_versions[-1]}"
+
+
+def split_comment(line):
+    split_line = line.split("#", maxsplit=1)
+    try:
+        comment = "  # " + split_line[1].strip()
+    except IndexError:
+        comment = ""
+    return split_line[0].strip(), comment
+
+
+def to_upper_bound(req):
+    try:
+        requirement = Requirement(req)
+    except ValueError:
+        return f"# UNPARSABLE: {req}"
+    else:
+        if requirement.name == "pulpcore":
+            # An exception to allow for pulpcore deprecation policy.
+            return fetch_pulpcore_upper_bound(requirement)
+        for spec in requirement.specifier:
+            if spec.operator == "~=":
+                return f"# NO BETTER CONSTRAINT: {req}"
+            if spec.operator == "<=":
+                operator = "=="
+                max_version = spec.version
+                return f"{requirement.name}{operator}{max_version}"
+            if spec.operator == "<":
+                operator = "~="
+                version = Version(spec.version)
+                if version.micro != 0:
+                    max_version = f"{version.major}.{version.minor}.{version.micro - 1}"
+                elif version.minor != 0:
+                    max_version = f"{version.major}.{version.minor - 1}"
+                elif version.major != 0:
+                    max_version = f"{version.major - 1}.0"
+                else:
+                    return f"# NO BETTER CONSTRAINT: {req}"
+                return f"{requirement.name}{operator}{max_version}"
+        return f"# NO UPPER BOUND: {req}"
+
+
+def to_lower_bound(req):
+    try:
+        requirement = Requirement(req)
+    except ValueError:
+        return f"# UNPARSABLE: {req}"
+    else:
+        for spec in requirement.specifier:
+            if spec.operator == ">=":
+                min_version = spec.version
+                if requirement.name == "pulpcore":
+                    # Currently an exception to allow for pulpcore bugfix releases.
+                    # TODO Semver libraries should be allowed too.
+                    operator = "~="
+                    if len(Version(min_version).release) != 3:
+                        raise RuntimeError("Pulpcore lower bound must be in the form '>=x.y.z'.")
+                else:
+                    operator = "=="
+                return f"{requirement.name}{operator}{min_version}"
+        return f"# NO LOWER BOUND: {req}"
+
+
+def main():
+    """Calculate constraints for the lower bound of dependencies where possible."""
+    parser = argparse.ArgumentParser(
+        prog=sys.argv[0],
+        description="Calculate constraints for the lower or upper bound of dependencies where "
+        "possible.",
+    )
+    parser.add_argument("-u", "--upper", action="store_true")
+    parser.add_argument("filename", nargs="*")
+    args = parser.parse_args()
+
+    modifier = to_upper_bound if args.upper else to_lower_bound
+
+    req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")]
+    pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")]
+    if req_files:
+        with fileinput.input(files=req_files) as req_file:
+            for line in req_file:
+                if line.strip().startswith("#"):
+                    # Shortcut comment only lines
+                    print(line.strip())
+                else:
+                    req, comment = split_comment(line)
+                    new_req = modifier(req)
+                    print(new_req + comment)
+    for filename in pyp_files:
+        with open(filename, "rb") as fp:
+            pyproject = tomllib.load(fp)
+            for req in pyproject["project"]["dependencies"]:
+                new_req = modifier(req)
+                print(new_req)
+            optional_dependencies = pyproject["project"].get("optional-dependencies")
+            if optional_dependencies:
+                for opt in optional_dependencies.values():
+                    for req in opt:
+                        new_req = modifier(req)
+                        print(new_req)
+
+
+if __name__ == "__main__":
+    main()