From 543e9db547c73992e423b1a48250c90e6d084a55 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Mon, 7 Jul 2025 10:13:51 +0200 Subject: [PATCH 1/2] Update CI --- .bumpversion.cfg | 21 -- .ci/ansible/Containerfile.j2 | 44 ++- .ci/ansible/settings.py.j2 | 39 +- .ci/ansible/start_container.yaml | 32 +- .ci/assets/ci_constraints.txt | 10 + .ci/assets/release_requirements.txt | 2 +- .ci/scripts/calc_constraints.py | 139 +++++++ .ci/scripts/calc_deps_lowerbounds.py | 34 -- .ci/scripts/changelog.py | 68 ---- .ci/scripts/check_release.py | 175 +++++++++ .ci/scripts/check_requirements.py | 62 +++- .ci/scripts/collect_changes.py | 139 +++++++ .ci/scripts/pr_labels.py | 60 +++ .ci/scripts/tweet.py | 16 - .ci/scripts/update_ci_branches.py | 25 -- .ci/scripts/validate_commit_message.py | 129 ++++--- .flake8 | 5 +- .github/template_gitref | 1 - .github/workflows/build.yml | 113 ++++++ .github/workflows/changelog.yml | 58 --- .github/workflows/ci.yml | 255 ++++--------- .github/workflows/codeql-analysis.yml | 18 +- .github/workflows/create-branch.yml | 77 ++-- .github/workflows/docs.yml | 47 +++ .github/workflows/kanban.yml | 103 ------ .github/workflows/lint.yml | 69 ++++ .github/workflows/nightly.yml | 315 ++-------------- .github/workflows/pr_checks.yml | 73 ++++ .github/workflows/publish.yml | 166 +++++++++ .github/workflows/release.yml | 345 ++---------------- .github/workflows/scripts/before_install.sh | 23 +- .github/workflows/scripts/before_script.sh | 10 +- .../workflows/scripts/build_python_client.sh | 54 +++ .../workflows/scripts/build_ruby_client.sh | 27 ++ .github/workflows/scripts/check_commit.sh | 17 +- .../scripts/create_release_from_tag.sh | 11 - .github/workflows/scripts/docs-publisher.py | 262 ------------- .github/workflows/scripts/install.sh | 77 ++-- .../scripts/install_python_client.sh | 53 --- .../workflows/scripts/install_ruby_client.sh | 43 --- .github/workflows/scripts/post_docs_test.sh | 9 - .../workflows/scripts/publish_client_gem.sh | 30 +- .../workflows/scripts/publish_client_pypi.sh | 29 +- .github/workflows/scripts/publish_docs.sh | 63 ---- .../workflows/scripts/publish_plugin_pypi.sh | 23 +- .../scripts/push_branch_and_tag_to_github.sh | 25 +- .github/workflows/scripts/release.py | 207 ----------- .github/workflows/scripts/release.sh | 27 ++ .github/workflows/scripts/script.sh | 161 ++++---- .../scripts/update_backport_labels.py | 59 +++ .github/workflows/scripts/update_ci.sh | 33 -- .github/workflows/test.yml | 147 ++++++++ .github/workflows/update-labels.yml | 39 ++ .github/workflows/update_ci.yml | 71 ++-- CHANGES/{.TEMPLATE.rst => .TEMPLATE.md} | 28 +- MANIFEST.in | 1 + doc_requirements.txt | 20 +- functest_requirements.txt | 8 +- lint_requirements.txt | 5 +- pyproject.toml | 45 +++ releasing.md | 29 ++ requirements.txt | 2 +- template_config.yml | 38 +- test_requirements.txt | 13 +- 64 files changed, 2036 insertions(+), 2293 deletions(-) delete mode 100644 .bumpversion.cfg create mode 100644 .ci/assets/ci_constraints.txt create mode 100755 .ci/scripts/calc_constraints.py delete mode 100755 .ci/scripts/calc_deps_lowerbounds.py delete mode 100755 .ci/scripts/changelog.py create mode 100755 .ci/scripts/check_release.py create mode 100755 .ci/scripts/collect_changes.py create mode 100755 .ci/scripts/pr_labels.py delete mode 100755 .ci/scripts/tweet.py delete mode 100755 .ci/scripts/update_ci_branches.py mode change 100755 => 100644 .ci/scripts/validate_commit_message.py delete mode 100644 .github/template_gitref create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/changelog.yml create mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/kanban.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pr_checks.yml create mode 100644 .github/workflows/publish.yml create mode 100755 .github/workflows/scripts/build_python_client.sh create mode 100755 .github/workflows/scripts/build_ruby_client.sh delete mode 100755 .github/workflows/scripts/create_release_from_tag.sh delete mode 100755 .github/workflows/scripts/docs-publisher.py delete mode 100755 .github/workflows/scripts/install_python_client.sh delete mode 100755 .github/workflows/scripts/install_ruby_client.sh delete mode 100644 .github/workflows/scripts/post_docs_test.sh delete mode 100755 .github/workflows/scripts/publish_docs.sh delete mode 100755 .github/workflows/scripts/release.py create mode 100755 .github/workflows/scripts/release.sh create mode 100755 .github/workflows/scripts/update_backport_labels.py delete mode 100755 .github/workflows/scripts/update_ci.sh create mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/update-labels.yml rename CHANGES/{.TEMPLATE.rst => .TEMPLATE.md} (64%) create mode 100644 releasing.md diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 79700cdb..00000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[bumpversion] -current_version = 3.10.1.dev -commit = False -tag = False -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+))? -serialize = - {major}.{minor}.{patch}.{release} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = prod -first_value = dev -values = - dev - prod - -[bumpversion:file:./pulp_python/app/__init__.py] - -[bumpversion:file:./setup.py] - -[bumpversion:file:./docs/conf.py] diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 64225b9d..d57b44ed 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,37 +1,47 @@ -FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }} +FROM {{ ci_base | default(pulp_default_container) }} # Add source directories to container {% for item in plugins %} -{% if item.source.startswith("./") or item.ci_requirements | default(false) %} ADD ./{{ item.name }} ./{{ item.name }} -{% endif %} {% endfor %} # Install python packages +# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads # Hacking botocore (https://github.com/boto/botocore/pull/1990) -RUN pip3 install -{%- if stream_test | default(false) -%} -{{ " " }}django-storages[sftp] -{%- endif -%} +# This MUST be the ONLY call to pip install in inside the container. +RUN pip3 install --upgrade pip setuptools wheel && \ + rm -rf /root/.cache/pip && \ + pip3 install {%- if s3_test | default(false) -%} -{{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue -{%- endif -%} -{%- if azure_test | default(false) -%} -{{ " " }}django-storages[azure]>=1.12.2 -{%- endif -%} -{%- if gcp_test | default(false) -%} -{{ " " }}django-storages[google]>=1.13.2 +{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} {%- for item in plugins -%} -{%- if item.name == "pulp-certguard" -%} -{{ " " }}python-dateutil rhsm -{%- endif -%} {{ " " }}{{ item.source }} +{%- if item.upperbounds | default(false) -%} +{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt +{%- endif -%} +{%- if item.lowerbounds | default(false) -%} +{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt +{%- endif -%} {%- if item.ci_requirements | default(false) -%} {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ + rm -rf /root/.cache/pip + +{% if pulp_env is defined and pulp_env %} +{% for key, value in pulp_env.items() %} +ENV {{ key | upper }}={{ value }} +{% endfor %} +{% endif %} + +{% if pulp_scenario_env is defined and pulp_scenario_env %} +{% for key, value in pulp_scenario_env.items() %} +ENV {{ key | upper }}={{ value }} +{% endfor %} +{% endif %} USER pulp:pulp RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 9470aafa..7a73ea2f 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -26,31 +26,40 @@ API_ROOT = {{ api_root | repr }} {% endfor %} {% endif %} -{% if stream_test | default(false) -%} -REDIRECT_TO_OBJECT_STORAGE = False -DEFAULT_FILE_STORAGE = "pulpcore.app.models.storage.PulpSFTPStorage" -MEDIA_ROOT = "" -SFTP_STORAGE_HOST = "ci-sftp" -SFTP_STORAGE_ROOT = "/storage/" -SFTP_STORAGE_PARAMS = { - "username": "foo", - "key_filename": "/keys/id_ed25519", -} -{%- endif %} - {% if s3_test | default(false) %} +MEDIA_ROOT: "" +S3_USE_SIGV4 = True +{% if test_storages_compat_layer is defined and test_storages_compat_layer %} +STORAGES = { + "default": { + "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", + "OPTIONS": { + "access_key": "{{ minio_access_key }}", + "secret_key": "{{ minio_secret_key }}", + "region_name": "eu-central-1", + "addressing_style": "path", + "signature_version": "s3v4", + "bucket_name": "pulp3", + "endpoint_url": "http://minio:9000", + "default_acl": "@none None", + }, + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, +} +{% else %} DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" -MEDIA_ROOT = "" AWS_ACCESS_KEY_ID = "{{ minio_access_key }}" AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}" AWS_S3_REGION_NAME = "eu-central-1" AWS_S3_ADDRESSING_STYLE = "path" -S3_USE_SIGV4 = True AWS_S3_SIGNATURE_VERSION = "s3v4" AWS_STORAGE_BUCKET_NAME = "pulp3" AWS_S3_ENDPOINT_URL = "http://minio:9000" AWS_DEFAULT_ACL = "@none None" {% endif %} +{% endif %} {% if azure_test | default(false) %} DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" @@ -61,7 +70,7 @@ AZURE_CONTAINER = "pulp-test" AZURE_LOCATION = "pulp3" AZURE_OVERWRITE_FILES = True AZURE_URL_EXPIRATION_SECS = 120 -AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;' +AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' {% endif %} {% if gcp_test | default(false) %} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index a05edc0b..47e5221e 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -15,14 +15,6 @@ - ssh - ~/.config/pulp_smash - - name: Generate an OpenSSH keypair - community.crypto.openssh_keypair: - path: ssh/id_ed25519 - type: ed25519 - owner: 700 # pulp in the container - become: true - when: stream_test | default(false) - - name: "Generate Pulp Settings" template: src: settings.py.j2 @@ -91,22 +83,14 @@ command: "docker logs pulp" failed_when: true - - block: - - name: "Check version of component being tested" - assert: - that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] | canonical_semver == (component_version | canonical_semver) - fail_msg: | - Component {{ component_name }} was expected to be installed in version {{ component_version }}. - Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] }}. - rescue: - - name: "Check version of component being tested (legacy)" - assert: - that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] | canonical_semver == (component_version | canonical_semver) - fail_msg: | - Component {{ legacy_component_name }} was expected to be installed in version {{ component_version }}. - Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] }}. + - name: "Check version of component being tested" + assert: + that: + - (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver) + fail_msg: | + Component {{ item.app_label }} was expected to be installed in version {{ component_version }}. + Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}. + loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}" - name: "Set pulp password in .netrc" copy: diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt new file mode 100644 index 00000000..8f41e3f6 --- /dev/null +++ b/.ci/assets/ci_constraints.txt @@ -0,0 +1,10 @@ +# Pulpcore versions without the openapi command do no longer work in the CI +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* + + +tablib!=3.6.0 +# 3.6.0: This release introduced a regression removing the "html" optional dependency. + + +multidict!=6.3.0 +# This release failed the lower bounds test for some case sensitivity in CIMultiDict. diff --git a/.ci/assets/release_requirements.txt b/.ci/assets/release_requirements.txt index c064e947..6635a872 100644 --- a/.ci/assets/release_requirements.txt +++ b/.ci/assets/release_requirements.txt @@ -1,3 +1,3 @@ -bump2version +bump-my-version gitpython towncrier diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py new file mode 100755 index 00000000..93ee2cc7 --- /dev/null +++ b/.ci/scripts/calc_constraints.py @@ -0,0 +1,139 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import fileinput +import urllib.request +import sys +from packaging.requirements import Requirement +from packaging.version import Version +import yaml + +try: + import tomllib +except ImportError: + import tomli as tomllib + + +CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" + + +def fetch_pulpcore_upper_bound(requirement): + with urllib.request.urlopen(CORE_TEMPLATE_URL) as f: + template = yaml.safe_load(f.read()) + supported_versions = template["supported_release_branches"] + supported_versions.append(template["latest_release_branch"]) + applicable_versions = sorted( + requirement.specifier.filter((Version(v) for v in supported_versions)) + ) + if len(applicable_versions) == 0: + raise Exception("No supported pulpcore version in required range.") + return f"{requirement.name}~={applicable_versions[-1]}" + + +def split_comment(line): + split_line = line.split("#", maxsplit=1) + try: + comment = " # " + split_line[1].strip() + except IndexError: + comment = "" + return split_line[0].strip(), comment + + +def to_upper_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + if requirement.name == "pulpcore": + # An exception to allow for pulpcore deprecation policy. + return fetch_pulpcore_upper_bound(requirement) + for spec in requirement.specifier: + if spec.operator == "~=": + return f"# NO BETTER CONSTRAINT: {req}" + if spec.operator == "<=": + operator = "==" + max_version = spec.version + return f"{requirement.name}{operator}{max_version}" + if spec.operator == "<": + operator = "~=" + version = Version(spec.version) + if version.micro != 0: + max_version = f"{version.major}.{version.minor}.{version.micro - 1}" + elif version.minor != 0: + max_version = f"{version.major}.{version.minor - 1}" + elif version.major != 0: + max_version = f"{version.major - 1}.0" + else: + return f"# NO BETTER CONSTRAINT: {req}" + return f"{requirement.name}{operator}{max_version}" + return f"# NO UPPER BOUND: {req}" + + +def to_lower_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + for spec in requirement.specifier: + if spec.operator == ">=": + min_version = spec.version + if requirement.name == "pulpcore": + # Currently an exception to allow for pulpcore bugfix releases. + # TODO Semver libraries should be allowed too. + operator = "~=" + if len(Version(min_version).release) != 3: + raise RuntimeError("Pulpcore lower bound must be in the form '>=x.y.z'.") + else: + operator = "==" + return f"{requirement.name}{operator}{min_version}" + return f"# NO LOWER BOUND: {req}" + + +def main(): + """Calculate constraints for the lower bound of dependencies where possible.""" + parser = argparse.ArgumentParser( + prog=sys.argv[0], + description="Calculate constraints for the lower or upper bound of dependencies where " + "possible.", + ) + parser.add_argument("-u", "--upper", action="store_true") + parser.add_argument("filename", nargs="*") + args = parser.parse_args() + + modifier = to_upper_bound if args.upper else to_lower_bound + + req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")] + pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")] + if req_files: + with fileinput.input(files=req_files) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) + else: + req, comment = split_comment(line) + new_req = modifier(req) + print(new_req + comment) + for filename in pyp_files: + with open(filename, "rb") as fp: + pyproject = tomllib.load(fp) + for req in pyproject["project"]["dependencies"]: + new_req = modifier(req) + print(new_req) + optional_dependencies = pyproject["project"].get("optional-dependencies") + if optional_dependencies: + for opt in optional_dependencies.values(): + for req in opt: + new_req = modifier(req) + print(new_req) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/calc_deps_lowerbounds.py b/.ci/scripts/calc_deps_lowerbounds.py deleted file mode 100755 index 2bb48e25..00000000 --- a/.ci/scripts/calc_deps_lowerbounds.py +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -from packaging.requirements import Requirement - - -def main(): - """Calculate the lower bound of dependencies where possible.""" - with open("requirements.txt") as req_file: - for line in req_file: - try: - requirement = Requirement(line) - except ValueError: - print(line.strip()) - else: - for spec in requirement.specifier: - if spec.operator == ">=": - if requirement.name == "pulpcore": - operator = "~=" - else: - operator = "==" - min_version = str(spec)[2:] - print(f"{requirement.name}{operator}{min_version}") - break - else: - print(line.strip()) - - -if __name__ == "__main__": - main() diff --git a/.ci/scripts/changelog.py b/.ci/scripts/changelog.py deleted file mode 100755 index 2e453a04..00000000 --- a/.ci/scripts/changelog.py +++ /dev/null @@ -1,68 +0,0 @@ -import re -import os -import requests -from packaging.version import Version -from git import Repo - -repo = Repo(os.getcwd()) -heads = repo.git.ls_remote("--heads", "https://github.com/pulp/pulp_python.git").split("\n") -branches = [h.split("/")[-1] for h in heads if re.search(r"^([0-9]+)\.([0-9]+)$", h.split("/")[-1])] -branches.sort(key=lambda ver: Version(ver), reverse=True) - - -def get_changelog(branch): - """ - Get changelog file for a given branch. - - """ - return requests.get( - f"https://raw.githubusercontent.com/pulp/pulp_python/{branch}/CHANGES.rst" - ).text - - -def get_changelog_releases(changelog): - """ - Get all versions in changelog. - - """ - versions = re.findall(r"([0-9]+)\.([0-9]+)\.([0-9]+) \(", changelog) - return {".".join(v) for v in versions} - - -def get_changelog_entry(changelog, version): - """ - Get changelog entry for a given version. - - """ - entries = changelog.split(f"{version} (")[1].split("=====\n") - header = f"{version} ({entries[0]}=====\n" - text = "\n\n\n".join(entries[1].split("\n\n\n")[0:-1]) - return header + text + "\n\n\n" - - -main_changelog = get_changelog("main") -main_entries = get_changelog_releases(main_changelog) -entries_list = list(main_entries) -to_add = {} -for branch in branches: - changelog = get_changelog(branch) - entries = get_changelog_releases(changelog) - for entry in entries.difference(main_entries): - description = get_changelog_entry(changelog, entry) - entries_list.append(entry) - print(description) - to_add[entry] = description - -entries_list.sort(key=lambda ver: Version(ver), reverse=True) -for version in sorted(to_add, key=lambda ver: Version(ver)): - next_version = entries_list[entries_list.index(version) + 1] - new_changelog = main_changelog.split(f"{next_version} (")[0] + to_add[version] - new_changelog = new_changelog + f"{next_version} (" - new_changelog = new_changelog + main_changelog.split(f"{next_version} (")[1] - main_changelog = new_changelog - -with open("CHANGES.rst", "w") as f: - f.write(main_changelog) - -if to_add: - repo.git.commit("-m", "Update Changelog\n\n[noissue]", "CHANGES.rst") diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py new file mode 100755 index 00000000..81bae1fd --- /dev/null +++ b/.ci/scripts/check_release.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python + +import argparse +import re +import os +import tomllib +import yaml +from pathlib import Path +from packaging.version import Version +from git import Repo + +RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" +Y_CHANGELOG_EXTS = [".feature"] +Z_CHANGELOG_EXTS = [".bugfix", ".misc"] + + +def options(): + """Check which branches need a release.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--branches", + default="supported", + help="A comma separated list of branches to check for releases. Can also use keyword: " + "'supported'. Defaults to 'supported', see `supported_release_branches` in " + "`plugin_template.yml`.", + ) + parser.add_argument( + "--no-fetch", + default=False, + action="store_true", + help="Don't fetch remote. Run faster at the expense of maybe being outdated.", + ) + return parser.parse_args() + + +def template_config(): + # Assume this script lies in .ci/scripts + path = Path(__file__).absolute().parent.parent.parent / "template_config.yml" + return yaml.safe_load(path.read_text()) + + +def current_version(repo, commitish): + try: + pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml")) + try: + current_version = pyproject_toml["project"]["version"] + except Exception: + current_version = pyproject_toml["tool"]["bumpversion"]["current_version"] + except Exception: + current_version = repo.git.grep( + "current_version", commitish, "--", ".bumpversion.cfg" + ).split("=")[-1] + return Version(current_version) + + +def check_pyproject_dependencies(repo, from_commit, to_commit): + try: + new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml")) + try: + new_dependencies = set(new_pyproject["project"]["dependencies"]) + except KeyError: + # New branch does not declare dependencies in pyproject.toml. + # Assume no release needed for this reason. + return [] + old_pyproject = tomllib.loads(repo.git.show(f"{from_commit}:pyproject.toml")) + old_dependencies = set(old_pyproject["project"]["dependencies"]) + if old_dependencies != new_dependencies: + return ["dependencies"] + else: + return [] + except Exception as e: + print(f"WARNING: Comparing the dependencies in pyproject.toml failed. ({e})") + # Gathering more details failed. + return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."] + + +def main(options, template_config): + DEFAULT_BRANCH = template_config["plugin_default_branch"] + + repo = Repo() + + upstream_default_branch = next( + (branch for branch in repo.branches if branch.name == DEFAULT_BRANCH) + ).tracking_branch() + remote = upstream_default_branch.remote_name + if not options.no_fetch: + repo.remote(remote).fetch() + + # Warning: This will not work if branch names contain "/" but we don't really care here. + heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")] + available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] + available_branches.sort(key=lambda ver: Version(ver)) + available_branches.append(DEFAULT_BRANCH) + + branches = options.branches + if branches == "supported": + tc = yaml.safe_load(repo.git.show(f"{upstream_default_branch}:template_config.yml")) + branches = set(tc["supported_release_branches"]) + latest_release_branch = tc["latest_release_branch"] + if latest_release_branch is not None: + branches.add(latest_release_branch) + branches.add(DEFAULT_BRANCH) + else: + branches = set(branches.split(",")) + + if diff := branches - set(available_branches): + print(f"Supplied branches contains non-existent branches! {diff}") + exit(1) + + print(f"Checking for releases on branches: {branches}") + + releases = [] + for branch in branches: + if branch != DEFAULT_BRANCH: + # Check if a Z release is needed + reasons = [] + changes = repo.git.ls_tree("-r", "--name-only", f"{remote}/{branch}", "CHANGES/") + z_changelog = False + for change in changes.split("\n"): + # Check each changelog file to make sure everything checks out + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: + print( + f"Warning: A non-backported changelog ({change}) is present in the " + f"{branch} release branch!" + ) + elif ext in Z_CHANGELOG_EXTS: + z_changelog = True + if z_changelog: + reasons.append("Backports") + + last_tag = repo.git.describe("--tags", "--abbrev=0", f"{remote}/{branch}") + req_txt_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "requirements.txt" + ) + if req_txt_diff: + reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.extend(check_pyproject_dependencies(repo, last_tag, f"{remote}/{branch}")) + + if reasons: + curr_version = Version(last_tag) + assert curr_version.base_version.startswith( + branch + ), "Current-version has to belong to the current branch!" + next_version = Version(f"{branch}.{curr_version.micro + 1}") + print( + f"A Z-release is needed for {branch}, " + f"Prev: {last_tag}, " + f"Next: {next_version.base_version}, " + f"Reason: {','.join(reasons)}" + ) + releases.append(next_version) + else: + # Check if a Y release is needed + changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") + for change in changes.split("\n"): + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: + # We don't put Y release bumps in the commit message, check file instead. + # The 'current_version' is always the dev of the next version to release. + next_version = current_version(repo, DEFAULT_BRANCH).base_version + print(f"A new Y-release is needed! New Version: {next_version}") + releases.append(next_version) + break + + if len(releases) == 0: + print("No new releases to perform.") + + +if __name__ == "__main__": + main(options(), template_config()) diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index a3d0ad5c..cf9efbe9 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,11 +5,13 @@ # # For more info visit https://github.com/pulp/plugin_template +import tomllib import warnings -from pkg_resources import Requirement +from packaging.requirements import Requirement CHECK_MATRIX = [ + ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), @@ -20,17 +22,33 @@ ("clitest_requirements.txt", False, True, True), ] -errors = [] -for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: - try: +def iterate_file(filename): + if filename == "pyproject.toml": + with open(filename, "rb") as fd: + pyproject_toml = tomllib.load(fd) + if "project" in pyproject_toml: + for nr, line in enumerate(pyproject_toml["project"]["dependencies"]): + yield nr, line + else: with open(filename, "r") as fd: for nr, line in enumerate(fd.readlines()): line = line.strip() if not line or line.startswith("#"): continue + if "#" in line: + line = line.split("#", maxsplit=1)[0] + yield nr, line.strip() + + +def main(): + errors = [] + + for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: + try: + for nr, line in iterate_file(filename): try: - req = Requirement.parse(line) + req = Requirement(line) except ValueError: if line.startswith("git+"): # The single exception... @@ -42,24 +60,28 @@ else: errors.append(f"{filename}:{nr}: Unreadable requirement {line}") else: - if not line.startswith("opentelemetry"): - if check_prereleases and req.specifier.prereleases: - if req.name != "pulp-python-client": - errors.append( - f"{filename}:{nr}: Prerelease versions found in {line}." - ) - ops = [op for op, ver in req.specs] - spec = str(req.specs) + if check_prereleases and req.specifier.prereleases: + # Do not even think about begging for more exceptions! + if ( + not req.name.startswith("opentelemetry") + and req.name != "pulp-python-client" + ): + errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") + ops = [spec.operator for spec in req.specifier] if "~=" in ops: warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!") elif "<" not in ops and "<=" not in ops and "==" not in ops: if check_upperbound: errors.append(f"{filename}:{nr}: Upper bound missing in {line}.") - except FileNotFoundError: - # skip this test for plugins that don't use this requirements.txt - pass + except FileNotFoundError: + # skip this test for plugins that don't use this requirements.txt + pass + + if errors: + print("Dependency issues found:") + print("\n".join(errors)) + exit(1) + -if errors: - print("Dependency issues found:") - print("\n".join(errors)) - exit(1) +if __name__ == "__main__": + main() diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py new file mode 100755 index 00000000..877ebc8c --- /dev/null +++ b/.ci/scripts/collect_changes.py @@ -0,0 +1,139 @@ +#!/bin/env python3 +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import itertools +import json +import os +import re +import tomllib +import urllib.request +from pathlib import Path + +from git import GitCommandError, Repo +from packaging.version import parse as parse_version + + +PYPI_PROJECT = "pulp_python" + +# Read Towncrier settings +tc_settings = tomllib.loads(Path("pyproject.toml").read_text())["tool"]["towncrier"] + +CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") +START_STRING = tc_settings.get( + "start_string", + ( + "\n" + if CHANGELOG_FILE.endswith(".md") + else ".. towncrier release notes start\n" + ), +) +TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})") + + +# Build a regex to find the header of a changelog section. +# It must have a single capture group to single out the version. +# see help(re.split) for more info. +NAME_REGEX = r".*" +VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*" +VERSION_CAPTURE_REGEX = rf"(?:YANKED )?({VERSION_REGEX})" +DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}" +TITLE_REGEX = ( + "(" + + re.escape( + TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX") + ) + .replace("NAME_REGEX", NAME_REGEX) + .replace("VERSION_REGEX", VERSION_CAPTURE_REGEX, 1) + .replace("VERSION_REGEX", VERSION_REGEX) + .replace("DATE_REGEX", DATE_REGEX) + + ")" +) + + +def get_changelog(repo, branch): + return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n" + + +def _tokenize_changes(splits): + assert len(splits) % 3 == 0 + for i in range(len(splits) // 3): + title = splits[3 * i] + version = parse_version(splits[3 * i + 1]) + yield [version, title + splits[3 * i + 2]] + + +def split_changelog(changelog): + preamble, rest = changelog.split(START_STRING, maxsplit=1) + split_rest = re.split(TITLE_REGEX, rest) + return preamble + START_STRING + split_rest[0], list(_tokenize_changes(split_rest[1:])) + + +def main(): + repo = Repo(os.getcwd()) + remote = repo.remotes[0] + branches = [ref for ref in remote.refs if re.match(r"^([0-9]+)\.([0-9]+)$", ref.remote_head)] + branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True) + branches = [ref.name for ref in branches] + + changed = False + + try: + response = urllib.request.urlopen(f"https://pypi.org/pypi/{PYPI_PROJECT}/json") + pypi_record = json.loads(response.read()) + yanked_versions = { + parse_version(version): release[0]["yanked_reason"] + for version, release in pypi_record["releases"].items() + if release[0]["yanked"] is True + } + except Exception: + # If something failed, just don't mark anything as yanked. + yanked_versions = {} + + with open(CHANGELOG_FILE, "r") as f: + main_changelog = f.read() + preamble, main_changes = split_changelog(main_changelog) + old_length = len(main_changes) + + for branch in branches: + print(f"Looking for './{CHANGELOG_FILE}' at branch {branch}") + try: + changelog = get_changelog(repo, branch) + except GitCommandError: + print("No changelog found on this branch.") + continue + dummy, changes = split_changelog(changelog) + new_changes = sorted(main_changes + changes, key=lambda x: x[0], reverse=True) + # Now remove duplicates (retain the first one) + main_changes = [new_changes[0]] + for left, right in itertools.pairwise(new_changes): + if left[0] != right[0]: + main_changes.append(right) + + if yanked_versions: + for change in main_changes: + if change[0] in yanked_versions and "YANKED" not in change[1].split("\n")[0]: + reason = yanked_versions[change[0]] + version = str(change[0]) + change[1] = change[1].replace(version, "YANKED " + version, count=1) + if reason: + change[1] = change[1].replace("\n", f"\n\nYank reason: {reason}\n", count=1) + changed = True + + new_length = len(main_changes) + if old_length < new_length or changed: + print(f"{new_length - old_length} new versions have been added (or something has changed).") + with open(CHANGELOG_FILE, "w") as fp: + fp.write(preamble) + for change in main_changes: + fp.write(change[1]) + + repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py new file mode 100755 index 00000000..0c478a21 --- /dev/null +++ b/.ci/scripts/pr_labels.py @@ -0,0 +1,60 @@ +#!/bin/env python3 + +# This script is running with elevated privileges from the main branch against pull requests. + +import re +import sys +import tomllib +from pathlib import Path + +from git import Repo + + +def main(): + assert len(sys.argv) == 3 + + with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) + BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT") + ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)") + CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$") + try: + CHANGELOG_EXTS = { + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + } + except KeyError: + CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"} + + repo = Repo(".") + + base_commit = repo.commit(sys.argv[1]) + head_commit = repo.commit(sys.argv[2]) + + pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}")) + + labels = { + "multi-commit": len(pr_commits) > 1, + "cherry-pick": False, + "no-issue": False, + "no-changelog": False, + "wip": False, + } + for commit in pr_commits: + labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None + no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None + labels["no-issue"] |= no_issue + cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None + labels["cherry-pick"] |= cherry_pick + changelog_snippets = [ + k + for k in commit.stats.files + if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS + ] + labels["no-changelog"] |= not changelog_snippets + + print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v))) + print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v))) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/tweet.py b/.ci/scripts/tweet.py deleted file mode 100755 index 5033cf44..00000000 --- a/.ci/scripts/tweet.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import sys -from tweepy import Client - -release_version = sys.argv[1] -if release_version.endswith(".0"): - client = Client( - consumer_key=os.getenv("TWITTER_API_KEY"), - consumer_secret=os.getenv("TWITTER_API_KEY_SECRET"), - access_token=os.getenv("TWITTER_ACCESS_TOKEN"), - access_token_secret=os.getenv("TWITTER_ACCESS_TOKEN_SECRET"), - ) - link = "https://docs.pulpproject.org/pulp_python/changes.html" - msg = f"pulp_python-{release_version} - Check out for more details: {link}" - release_msg = f"Hey! We've just released {msg}" - client.create_tweet(text=release_msg) diff --git a/.ci/scripts/update_ci_branches.py b/.ci/scripts/update_ci_branches.py deleted file mode 100755 index c82e2a07..00000000 --- a/.ci/scripts/update_ci_branches.py +++ /dev/null @@ -1,25 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import os -import sys -import requests - -branches = sys.argv[1:] - -headers = { - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", - "Accept": "application/vnd.github.v3+json", -} - -github_api = "https://api.github.com" -workflow_path = "/actions/workflows/update_ci.yml/dispatches" -url = f"{github_api}/repos/pulp/pulp_python{workflow_path}" - -for branch in branches: - print(f"Updating {branch}") - requests.post(url, headers=headers, json={"ref": branch}) diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py old mode 100755 new mode 100644 index d6f22c71..a4dc9004 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -1,46 +1,31 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template +# This file is managed by the plugin template. +# Do not edit. +import os import re +import subprocess import sys +import tomllib +import yaml from pathlib import Path -import subprocess - -import os -import warnings from github import Github -NO_ISSUE = "[noissue]" -CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] -sha = sys.argv[1] -message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") - - -KEYWORDS = ["fixes", "closes"] - -g = Github(os.environ.get("GITHUB_TOKEN")) -repo = g.get_repo("pulp/pulp_python") - - -def __check_status(issue): +def check_status(issue, repo, cherry_pick): gi = repo.get_issue(int(issue)) if gi.pull_request: sys.exit(f"Error: issue #{issue} is a pull request.") - if gi.closed_at and "cherry picked from commit" not in message: - warnings.warn( - "When backporting, use the -x flag to append a line that says " - "'(cherry picked from commit ...)' to the original commit message." + if gi.closed_at and not cherry_pick: + print("Make sure to use 'git cherry-pick -x' when backporting a change.") + print( + "If a backport of a change requires a significant amount of rewriting, " + "consider creating a new issue." ) sys.exit(f"Error: issue #{issue} is closed.") -def __check_changelog(issue): +def check_changelog(issue, CHANGELOG_EXTS): matches = list(Path("CHANGES").rglob(f"{issue}.*")) if len(matches) < 1: @@ -48,31 +33,65 @@ def __check_changelog(issue): for match in matches: if match.suffix not in CHANGELOG_EXTS: sys.exit(f"Invalid extension for changelog entry '{match}'.") - if match.suffix == ".feature" and "cherry picked from commit" in message: - sys.exit(f"Can not backport '{match}' as it is a feature.") - - -print("Checking commit message for {sha}.".format(sha=sha[0:7])) - -# validate the issue attached to the commit -regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) -pattern = re.compile(regex, re.IGNORECASE) - -issues = pattern.findall(message) - -if issues: - for issue in pattern.findall(message): - __check_status(issue) - __check_changelog(issue) -else: - if NO_ISSUE in message: - print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) - elif "Merge" in message and "cherry picked from commit" in message: - pass - else: - sys.exit( - "Error: no attached issues found for {sha}. If this was intentional, add " - " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE) - ) -print("Commit message for {sha} passed.".format(sha=sha[0:7])) + +def main() -> None: + TEMPLATE_CONFIG = yaml.safe_load(Path("template_config.yml").read_text()) + GITHUB_ORG = TEMPLATE_CONFIG["github_org"] + PLUGIN_NAME = TEMPLATE_CONFIG["plugin_name"] + + with Path("pyproject.toml").open("rb") as _fp: + PYPROJECT_TOML = tomllib.load(_fp) + KEYWORDS = ["fixes", "closes"] + BLOCKING_REGEX = [ + r"^DRAFT", + r"^WIP", + r"^NOMERGE", + r"^DO\s*NOT\s*MERGE", + r"^EXPERIMENT", + r"^FIXUP", + r"^fixup!", # This is created by 'git commit --fixup' + r"Apply suggestions from code review", # This usually comes from GitHub + ] + try: + CHANGELOG_EXTS = [ + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + ] + except KeyError: + CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] + NOISSUE_MARKER = "[noissue]" + + sha = sys.argv[1] + message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") + + if NOISSUE_MARKER in message: + sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") + + blocking_matches = [m for m in (re.match(pattern, message) for pattern in BLOCKING_REGEX) if m] + if blocking_matches: + print("Found these phrases in the commit message:") + for m in blocking_matches: + print(" - " + m.group(0)) + sys.exit("This PR is not ready for consumption.") + + g = Github(os.environ.get("GITHUB_TOKEN")) + repo = g.get_repo(f"{GITHUB_ORG}/{PLUGIN_NAME}") + + print("Checking commit message for {sha}.".format(sha=sha[0:7])) + + # validate the issue attached to the commit + issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords="|".join(KEYWORDS)) + issues = re.findall(issue_regex, message, re.IGNORECASE) + cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" + cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) + + if issues: + for issue in issues: + check_status(issue, repo, cherry_pick) + check_changelog(issue, CHANGELOG_EXTS) + + print("Commit message for {sha} passed.".format(sha=sha[0:7])) + + +if __name__ == "__main__": + main() diff --git a/.flake8 b/.flake8 index 617447a8..9e00b3fa 100644 --- a/.flake8 +++ b/.flake8 @@ -6,13 +6,16 @@ # For more info visit https://github.com/pulp/plugin_template [flake8] exclude = ./docs/*,*/migrations/* -ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402 +per-file-ignores = */__init__.py: F401 + +ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,F824 max-line-length = 100 # Flake8 builtin codes # -------------------- # E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed # W503: This enforces operators before line breaks which is not pep8 or black compatible. +# F824: 'nonlocal' is unused: name is never assigned in scope # Flake8-quotes extension codes # ----------------------------- diff --git a/.github/template_gitref b/.github/template_gitref deleted file mode 100644 index 046f010a..00000000 --- a/.github/template_gitref +++ /dev/null @@ -1 +0,0 @@ -2021.08.26-214-gf2ffaa4 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..67878dc4 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,113 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Build" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + build: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install build packaging twine wheel mkdocs jq + echo ::endgroup:: + - name: "Build package" + run: | + python3 -m build + twine check dist/* + - name: "Install built packages" + run: | + pip install dist/pulp_python-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt + - name: "Generate api specs" + run: | + pulpcore-manager openapi --file "api.json" + pulpcore-manager openapi --bindings --component "python" --file "python-api.json" + - name: "Upload Package whl" + uses: "actions/upload-artifact@v4" + with: + name: "plugin_package" + path: "pulp_python/dist/" + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Upload API specs" + uses: "actions/upload-artifact@v4" + with: + name: "api_spec" + path: | + pulp_python/api.json + pulp_python/python-api.json + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Python bindings packages" + run: | + .github/workflows/scripts/build_python_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v4" + with: + name: "python-client.tar" + path: | + pulp_python/python-python-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Upload python client docs" + uses: "actions/upload-artifact@v4" + with: + name: "python-client-docs.tar" + path: | + pulp_python/python-python-client-docs.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Ruby bindings packages" + run: | + .github/workflows/scripts/build_ruby_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload Ruby client" + uses: "actions/upload-artifact@v4" + with: + name: "ruby-client.tar" + path: | + pulp_python/python-ruby-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true +... diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml deleted file mode 100644 index 976e2ea9..00000000 --- a/.github/workflows/changelog.yml +++ /dev/null @@ -1,58 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - ---- -name: Python changelog update -on: - push: - branches: - - main - paths: - - CHANGES.rst - - CHANGES.md - workflow_dispatch: - -jobs: - - update-changelog: - runs-on: ubuntu-latest - strategy: - fail-fast: false - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v3 - with: - python-version: "3.8" - - - name: Install python dependencies - run: | - echo ::group::PYDEPS - pip install -r doc_requirements.txt - echo ::endgroup:: - - - name: Fake api schema - run: | - mkdir -p docs/_build/html - echo "{}" > docs/_build/html/api.json - mkdir -p docs/_static - echo "{}" > docs/_static/api.json - - name: - run: | - pip install "Jinja2<3.1" - make diagrams html - working-directory: ./docs - env: - PULP_CONTENT_ORIGIN: "http://localhost/" - - - name: Publish changlog to pulpproject.org - run: .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} - env: - PULP_DOCS_KEY: ${{ secrets.PULP_DOCS_KEY }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10123970..0d16d126 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,218 +6,93 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Python CI +name: "Python CI" on: {pull_request: {branches: ['*']}} concurrency: group: ${{ github.ref_name }}-${{ github.workflow }} cancel-in-progress: true -jobs: +defaults: + run: + working-directory: "pulp_python" - ready-to-ship: - runs-on: ubuntu-latest +jobs: + check-commits: + runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v3 + - uses: "actions/checkout@v4" with: fetch-depth: 0 - - uses: actions/setup-python@v3 + path: "pulp_python" + - uses: "actions/setup-python@v5" with: - python-version: "3.8" - - name: Install requirements - run: pip3 install github - - name: Check commit message + python-version: "3.11" + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install requests pygithub pyyaml + echo ::endgroup:: + - name: "Check commit message" if: github.event_name == 'pull_request' env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - run: sh .github/workflows/scripts/check_commit.sh - - name: Verify requirements files - run: python .ci/scripts/check_requirements.py - single_commit: - runs-on: ubuntu-latest - name: Assert single commit - if: github.base_ref == 'main' - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Checkout main - run: git fetch origin main - - name: create local main branch - run: git branch main origin/main - - name: Commit Count Check - run: test `git log --oneline --no-merges HEAD ^main | wc -l ` = 1 - - lint: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - uses: actions/setup-python@v3 - with: - python-version: "3.8" - # lint_requirements contains tools needed for flake8, etc. - - name: Install requirements - run: pip3 install -r lint_requirements.txt - - - - # Lint code. - - name: Run flake8 - run: flake8 - - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + run: | + .github/workflows/scripts/check_commit.sh - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + docs: + uses: "./.github/workflows/docs.yml" - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + lint: + uses: "./.github/workflows/lint.yml" - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + build: + needs: "lint" + uses: "./.github/workflows/build.yml" test: - runs-on: ubuntu-latest - # run only after lint finishes - needs: lint - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - TEST: lowerbounds - outputs: - deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }} - deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }} - deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }} - deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }} + needs: "build" + uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] + deprecations: + runs-on: "ubuntu-latest" + if: github.base_ref == 'main' + needs: "test" steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v3 - with: - python-version: "3.8" - - - name: Install httpie + - name: "Create working directory" run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables + mkdir -p "pulp_python" + working-directory: "." + - name: "Download Deprecations" + uses: actions/download-artifact@v4 + with: + pattern: "deprecations-*" + path: "pulp_python" + merge-multiple: true + - name: "Print deprecations" run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} + cat deprecations-*.txt | sort -u + ! cat deprecations-*.txt | grep '[^[:space:]]' - - name: Setting secrets - if: github.event_name != 'pull_request' - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Extract Deprecations from Logs - id: deprecations - run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" - deprecations: - runs-on: ubuntu-latest - if: always() - needs: test + ready-to-ship: + # This is a dummy dependent task to have a single entry for the branch protection rules. + runs-on: "ubuntu-latest" + needs: + - "check-commits" + - "lint" + - "test" + - "docs" + if: "always()" steps: - - name: Fail on deprecations - run: | - test -z "${{ needs.test.outputs.deprecations-pulp }}" - test -z "${{ needs.test.outputs.deprecations-azure }}" - test -z "${{ needs.test.outputs.deprecations-s3 }}" - test -z "${{ needs.test.outputs.deprecations-lowerbounds }}" - - name: Print deprecations - if: failure() + - name: "Collect needed jobs results" + working-directory: "." run: | - echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d - echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d - echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d - echo "${{ needs.test.outputs.deprecations-lowerbounds }}" | base64 -d - - + echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result' + echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0' + echo "CI says: Looks good!" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 8e074b9e..899147c4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -4,7 +4,7 @@ # './plugin-template --github pulp_python' to update this file. # # For more info visit https://github.com/pulp/plugin_template -name: "CodeQL" +name: "Python CodeQL" on: workflow_dispatch: @@ -30,13 +30,13 @@ jobs: language: [ 'python' ] steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 1fef0e8c..75c3224d 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -9,10 +9,6 @@ name: Create New Release Branch on: workflow_dispatch: - inputs: - name: - description: "Branch name (e.g. 3.14)" - required: true env: RELEASE_WORKFLOW: true @@ -24,56 +20,87 @@ jobs: strategy: fail-fast: false + permissions: + contents: write + steps: - - uses: actions/checkout@v3 + - uses: "actions/checkout@v4" + with: + fetch-depth: 0 + path: "pulp_python" + + - uses: "actions/checkout@v4" with: fetch-depth: 1 + repository: "pulp/plugin_template" + path: "plugin_template" - - uses: actions/setup-python@v3 + - uses: "actions/setup-python@v5" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version + pip install bump-my-version packaging -r plugin_template/requirements.txt echo ::endgroup:: - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + - name: "Setting secrets" + working-directory: "pulp_python" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} + SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: Verify that branch name matches current version string on main branch + - name: Determine new branch name + working-directory: pulp_python run: | - X_Y_VERSION=$(grep version setup.py | sed -rn 's/version="(.*)\.0((a[0-9]+)|(b[0-9]+))?\.dev",/\1/p' | awk '{$1=$1};1') - if [[ "$X_Y_VERSION" != "${{ github.event.inputs.name }}" ]] + # Just to be sure... + git checkout main + NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')" + if [ -z "$NEW_BRANCH" ] then - echo "Branch name doesn't match the current version string $X_Y_VERSION." + echo Could not determine the new branch name. exit 1 fi + echo "NEW_BRANCH=${NEW_BRANCH}" >> "$GITHUB_ENV" - - name: Create ${{ github.event.inputs.name }} release branch + - name: Create release branch + working-directory: pulp_python run: | - git checkout -b ${{ github.event.inputs.name }} - git push origin ${{ github.event.inputs.name }} + git branch "${NEW_BRANCH}" - name: Bump version on main branch + working-directory: pulp_python run: | - git checkout main - bump2version --no-commit minor + bump-my-version bump --no-commit minor - - name: Make a PR with version bump - uses: peter-evans/create-pull-request@v4 + - name: Remove entries from CHANGES directory + working-directory: pulp_python + run: | + find CHANGES -type f -regex ".*\.\(bugfix\|doc\|feature\|misc\|deprecation\|removal\)" -exec git rm {} + + + - name: Update CI branches in template_config + working-directory: plugin_template + run: | + python3 ./plugin-template pulp_python --github --latest-release-branch "${NEW_BRANCH}" + git add -A + + - name: Make a PR with version bump and without CHANGES/* + uses: peter-evans/create-pull-request@v6 with: + path: pulp_python token: ${{ secrets.RELEASE_TOKEN }} committer: pulpbot author: pulpbot branch: minor-version-bump base: main title: Bump minor version - body: '[noissue]' commit-message: | Bump minor version - [noissue] delete-branch: true + + - name: Push release branch + working-directory: pulp_python + run: | + git push origin "${NEW_BRANCH}" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..44f17e93 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,47 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Docs CI" +on: + workflow_call: + +jobs: + changelog: + if: "endsWith(github.base_ref, 'main')" + runs-on: "ubuntu-latest" + defaults: + run: + working-directory: "pulp_python" + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + - uses: "actions/setup-python@v5" + with: + python-version: "3.12" + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install towncrier + echo ::endgroup:: + - name: "Build changelog" + run: | + towncrier build --yes --version 4.0.0.ci + docs: + if: "endsWith(github.base_ref, 'main')" + uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@rewrite-as-mkdocs-plugin' + with: + pulpdocs_ref: 'rewrite-as-mkdocs-plugin' + + no-test: + if: "!endsWith(github.base_ref, 'main')" + runs-on: "ubuntu-latest" + steps: + - run: | + echo "Skip docs testing on non-default branches." diff --git a/.github/workflows/kanban.yml b/.github/workflows/kanban.yml deleted file mode 100644 index 97e75008..00000000 --- a/.github/workflows/kanban.yml +++ /dev/null @@ -1,103 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template -# Manage issues in a project board using https://github.com/leonsteinhaeuser/project-beta-automations - ---- -name: Kanban -on: - pull_request_target: - issues: - types: - - labeled - - reopened - - assigned - - closed - -env: - free_to_take: Free to take - in_progress: In Progress - needs_review: Needs review - done: Done - -jobs: - # only prio-list labeled items should be added to the board - add-to-project-board: - if: github.event_name == 'issues' && contains(github.event.issue.labels.*.name, 'prio-list') && contains(fromJson('["labeled", "reopened"]'), github.event.action) - runs-on: ubuntu-latest - steps: - - name: Add issue to Free-to-take list - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.free_to_take }} # Target status - - move-to-inprogress: - if: github.event_name == 'issues' && github.event.action == 'assigned' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the In Progress column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.in_progress }} # Target status - - find-linked-issues: - if: github.event_name == 'pull_request_target' - runs-on: ubuntu-latest - name: Find issues linked to a PR - outputs: - linked-issues: ${{ steps.linked-issues.outputs.issues }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Get Linked Issues Action - uses: kin/gh-action-get-linked-issues@v1.0 - id: linked-issues - with: - access-token: ${{ secrets.RELEASE_TOKEN }} - - move-to-needs-review: - if: github.event_name == 'pull_request_target' && contains(fromJson(needs.find-linked-issues.outputs.linked-issues).*.issue.state, 'open') - runs-on: ubuntu-latest - name: Move linked issues to Needs Review - needs: find-linked-issues - strategy: - max-parallel: 3 - matrix: - issues: ${{ fromJSON(needs.find-linked-issues.outputs.linked-issues) }} - steps: - - name: Move to Needs Review - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ matrix.issues.issue.node_id }} - operation_mode: status - status_value: ${{ env.needs_review }} # Target status - - move-to-done: - if: github.event_name == 'issues' && github.event.action == 'closed' - runs-on: ubuntu-latest - steps: - - name: Move an issue to the Done column - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 - with: - gh_token: ${{ secrets.RELEASE_TOKEN }} - organization: pulp - project_id: 8 - resource_node_id: ${{ github.event.issue.node_id }} - operation_mode: status - status_value: ${{ env.done }} # Target status diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..9dddcbdd --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,69 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Lint" +on: + workflow_call: + +defaults: + run: + working-directory: "pulp_python" + +jobs: + lint: + runs-on: "ubuntu-latest" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install -r lint_requirements.txt + echo ::endgroup:: + + - name: "Lint workflow files" + run: | + yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + + # Lint code. + - name: "Run flake8" + run: | + flake8 + + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh + + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest + + - name: "Verify requirements files" + run: | + python .ci/scripts/check_requirements.py + + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh + + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 84f38e63..ec3fcb11 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Python Nightly CI/CD +name: "Python Nightly CI" on: schedule: # * is a special character in YAML so you have to quote this string @@ -14,313 +14,58 @@ on: - cron: '00 3 * * *' workflow_dispatch: +defaults: + run: + working-directory: "pulp_python" + concurrency: - group: ${{ github.ref_name }}-${{ github.workflow }} + group: "${{ github.ref_name }}-${{ github.workflow }}" cancel-in-progress: true jobs: - test: - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - - TEST: generate-bindings - - TEST: lowerbounds - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v3 - with: - python-version: "3.8" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - uses: ruby/setup-ruby@v1 - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - with: - ruby-version: "2.6" + build: + uses: "./.github/workflows/build.yml" - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Script - run: .github/workflows/scripts/script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Upload python client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client.tar - path: python-client.tar - - - name: Upload python client docs - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client-docs.tar - path: python-client-docs.tar - - - name: Upload ruby client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: ruby-client.tar - path: ruby-client.tar - - name: Upload built docs - if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v3 - with: - name: docs.tar - path: docs/docs.tar - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + test: + needs: "build" + uses: "./.github/workflows/test.yml" + with: + matrix_env: | + [{"TEST": "pulp"}, {"TEST": "azure"}, {"TEST": "s3"}, {"TEST": "lowerbounds"}] changelog: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: "actions/checkout@v4" with: - fetch-depth: 1 + fetch-depth: 0 + path: "pulp_python" - - uses: actions/setup-python@v3 + - uses: "actions/setup-python@v5" with: - python-version: "3.8" + python-version: "3.13" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install gitpython requests packaging + pip install gitpython packaging toml echo ::endgroup:: - - name: Configure Git with pulpbot name and email + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Changelog history - run: python .ci/scripts/changelog.py + - name: Collect changes from all branches + run: python .ci/scripts/collect_changes.py - name: Create Pull Request - uses: peter-evans/create-pull-request@v4 + uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.RELEASE_TOKEN }} - committer: pulpbot - author: pulpbot - title: 'Update Changelog' - body: '[noissue]' - branch: 'changelog/update' - base: main - commit-message: | - Update Changelog - - [noissue] + title: "Update Changelog" + body: "" + branch: "changelog/update" delete-branch: true - - publish: - runs-on: ubuntu-latest - needs: test - - env: - TEST: publish - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - uses: actions/setup-python@v3 - with: - python-version: "3.8" - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Install python dependencies - run: | - echo ::group::PYDEPS - pip install wheel - echo ::endgroup:: - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install - run: .github/workflows/scripts/install.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - - name: Download built docs - uses: actions/download-artifact@v3 - with: - name: docs.tar - - - name: Download Python client docs - uses: actions/download-artifact@v3 - with: - name: python-client-docs.tar - - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar -C ./docs - .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} - - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + path: "pulp_python" +... diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml new file mode 100644 index 00000000..0e0a7936 --- /dev/null +++ b/.github/workflows/pr_checks.yml @@ -0,0 +1,73 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Python PR static checks" +on: + pull_request_target: + types: + - "opened" + - "synchronize" + - "reopened" + branches: + - "main" + - "[0-9]+.[0-9]+" + +# This workflow runs with elevated permissions. +# Do not even think about running a single bit of code from the PR. +# Static analysis should be fine however. + +concurrency: + group: "${{ github.event.pull_request.number }}-${{ github.workflow }}" + cancel-in-progress: true + +jobs: + apply_labels: + runs-on: "ubuntu-latest" + name: "Label PR" + permissions: + pull-requests: "write" + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 0 + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Determine PR labels" + run: | + pip install GitPython==3.1.42 + git fetch origin ${{ github.event.pull_request.head.sha }} + python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" + - uses: "actions/github-script@v7" + name: "Apply PR Labels" + with: + script: | + const { ADD_LABELS, REMOVE_LABELS } = process.env; + + if (REMOVE_LABELS.length) { + for await (const labelName of REMOVE_LABELS.split(",")) { + try { + await github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch(err) { + } + } + } + if (ADD_LABELS.length) { + await github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ADD_LABELS.split(","), + }); + } +... diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..c9505f82 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,166 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Python Publish Release" +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+" + +defaults: + run: + working-directory: "pulp_python" + +jobs: + build: + uses: "./.github/workflows/build.yml" + publish-package: + runs-on: "ubuntu-latest" + needs: + - "build" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/download-artifact@v4" + with: + name: "plugin_package" + path: "pulp_python/dist/" + + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install twine + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Deploy plugin to pypi" + run: | + .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.ref_name }} + publish-python-bindings: + runs-on: "ubuntu-latest" + needs: + - "build" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - name: "Download Python client" + uses: "actions/download-artifact@v4" + with: + name: "python-client.tar" + path: "pulp_python/" + + - name: "Untar python client packages" + run: | + tar -xvf python-python-client.tar + + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install twine + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Publish client to pypi" + run: | + bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.ref_name }} + publish-ruby-bindings: + runs-on: "ubuntu-latest" + needs: + - "build" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - name: "Download Ruby client" + uses: "actions/download-artifact@v4" + with: + name: "ruby-client.tar" + path: "pulp_python/" + + - name: "Untar Ruby client packages" + run: | + tar -xvf python-ruby-client.tar + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "2.6" + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Publish client to rubygems" + run: | + bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} + + create-gh-release: + runs-on: "ubuntu-latest" + needs: + - "build" + - "publish-package" + - "publish-python-bindings" + - "publish-ruby-bindings" + + permissions: + contents: write + + steps: + - name: "Create release on GitHub" + uses: "actions/github-script@v7" + env: + TAG_NAME: "${{ github.ref_name }}" + with: + script: | + const { TAG_NAME } = process.env; + + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: TAG_NAME, + make_latest: "legacy", + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2bf1a90b..b758bae6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,354 +6,55 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Release Pipeline +name: Python Release Pipeline on: workflow_dispatch: - inputs: - release: - description: "Release tag (e.g. 3.2.1)" - required: true - before_script: - description: | - Bash code to run before script.sh is executed. This should only be used when re-running - a workflow to correct some aspect of the docs. e.g.: git checkout origin/3.14 CHANGES.rst - required: false -env: - RELEASE_WORKFLOW: true +defaults: + run: + working-directory: "pulp_python" jobs: build-artifacts: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: "actions/checkout@v4" with: fetch-depth: 0 + path: "pulp_python" + token: ${{ secrets.RELEASE_TOKEN }} - - uses: actions/setup-python@v3 + - uses: "actions/setup-python@v5" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging~=21.3 bandersnatch bump2version gitpython towncrier==19.9.0 wheel + pip install bump-my-version towncrier echo ::endgroup:: - - name: Configure Git with pulpbot name and email + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Create the release commit, tag it, create a post-release commit, and build plugin package - run: python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} - - - name: 'Tar files' - run: tar -cvf pulp_python.tar . - - - name: 'Upload Artifact' - uses: actions/upload-artifact@v3 - with: - name: pulp_python.tar - path: pulp_python.tar - test: - needs: build-artifacts - - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - env: - - TEST: pulp - - TEST: docs - - TEST: azure - - TEST: s3 - - TEST: generate-bindings - - TEST: lowerbounds - - steps: - - uses: actions/download-artifact@v3 - with: - name: pulp_python.tar - - - uses: actions/setup-python@v3 - with: - python-version: "3.8" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: Untar repository - run: | - tar -xf pulp_python.tar - - # update to the branch's latest ci files rather than the ones from the release tag. this is - # helpful when there was a problem with the ci files during the release which needs to be - # fixed after the release tag has been created - - name: Update ci files - run: | - git checkout "origin/${GITHUB_REF##*/}" -- .ci - git checkout "origin/${GITHUB_REF##*/}" -- .github - - - name: Install httpie - run: | - echo ::group::HTTPIE - pip install httpie - echo ::endgroup:: - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV - - - name: Set environment variables + - name: "Setting secrets" run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Before Install - run: .github/workflows/scripts/before_install.sh - shell: bash + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} + SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: Install + - name: "Tag the release" run: | - export PLUGIN_VERSION=${{ github.event.inputs.release }} - .github/workflows/scripts/install.sh - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - shell: bash - - - name: Before Script - run: .github/workflows/scripts/before_script.sh - shell: bash - env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }} - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install Python client - run: .github/workflows/scripts/install_python_client.sh - shell: bash - - - name: Install Ruby client - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - run: .github/workflows/scripts/install_ruby_client.sh - shell: bash - - - name: Additional before_script - run: ${{ github.event.inputs.before_script }} - shell: bash - - - name: Script - if: ${{ env.TEST != 'generate-bindings' }} - run: .github/workflows/scripts/script.sh - shell: bash + .github/workflows/scripts/release.sh + shell: "bash" env: - PY_COLORS: '1' - ANSIBLE_FORCE_COLOR: '1' - GITHUB_PULL_REQUEST: ${{ github.event.number }} - GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} - GITHUB_BRANCH: ${{ github.head_ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }} - - - name: Upload python client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client.tar - path: python-client.tar - - - name: Upload python client docs - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: python-client-docs.tar - path: python-client-docs.tar - - - name: Upload ruby client packages - if: ${{ env.TEST == 'bindings' || env.TEST == 'generate-bindings' }} - uses: actions/upload-artifact@v3 - with: - name: ruby-client.tar - path: ruby-client.tar - - name: Upload built docs - if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v3 - with: - name: docs.tar - path: docs/docs.tar - - - name: Logs - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" - - - publish: - runs-on: ubuntu-latest - needs: test - - env: - TEST: publish - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - steps: - - uses: actions/download-artifact@v3 - with: - name: pulp_python.tar - - - uses: actions/setup-python@v3 - with: - python-version: "3.8" - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: Configure Git with pulpbot name and email - run: | - git config --global user.name 'pulpbot' - git config --global user.email 'pulp-infra@redhat.com' - - - name: Untar repository - run: | - tar -xf pulp_python.tar - - # update to the branch's latest ci files rather than the ones from the release tag. this is - # helpful when there was a problem with the ci files during the release which needs to be - # fixed after the release tag has been created - - name: Update ci files - run: | - git checkout "origin/${GITHUB_REF##*/}" -- .ci - git checkout "origin/${GITHUB_REF##*/}" -- .github - - - name: Setting secrets - run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: Install python dependencies - run: | - echo ::group::PYDEPS - pip install gitpython requests packaging~=21.3 tweepy - echo ::endgroup:: - - - name: Push branch and tag to GitHub - run: bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - - name: Download built docs - uses: actions/download-artifact@v3 - with: - name: docs.tar - - - name: Download Python client docs - uses: actions/download-artifact@v3 - with: - name: python-client-docs.tar - - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar -C ./docs - .github/workflows/scripts/publish_docs.sh tag ${{ github.event.inputs.release }} - - name: Deploy plugin to pypi - run: bash .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }} - - name: Download Python client - uses: actions/download-artifact@v3 - with: - name: python-client.tar - - - name: Untar python client packages - run: tar -xvf python-client.tar - - - name: Publish client to pypi - run: bash .github/workflows/scripts/publish_client_pypi.sh - - name: Download Ruby client - uses: actions/download-artifact@v3 - with: - name: ruby-client.tar - - - name: Untar Ruby client packages - run: tar -xvf ruby-client.tar - - - name: Publish client to rubygems - run: bash .github/workflows/scripts/publish_client_gem.sh - - - name: Update GitHub - continue-on-error: true - run: | - set -euv - export COMMIT_MSG=$(git log --format=%B --no-merges -1) - export GH_ISSUES=$(echo $COMMIT_MSG | grep -o "GH Issues: .*" | awk '{print $3}') - pip install pygithub - - echo "GH Issues $GH_ISSUES" - python .ci/scripts/update_github.py - - - name: Tweet - continue-on-error: true - run: python .ci/scripts/tweet.py ${{ github.event.inputs.release }} - - - name: Create release on GitHub - run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }} - - - name: Cleanup repository before making changelog PR - run: rm -rf .lock generation pulp_python_client* *-client.tar pulp_python.tar todo web *docs.tar - - - name: Stage changelog for main branch - run: python .github/workflows/scripts/stage-changelog-for-default-branch.py ${{ github.event.inputs.release }} - - - name: Create Pull Request for Changelog - uses: peter-evans/create-pull-request@v4 - with: - token: ${{ secrets.RELEASE_TOKEN }} - committer: pulpbot - author: pulpbot - branch: changelog/${{ github.event.inputs.release }} - base: main - title: 'Cherry pick ${{ github.event.inputs.release }} changelog' - body: '[noissue]' - commit-message: | - ${{ github.event.inputs.release }} changelog - - [noissue] - delete-branch: true + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 474720c1..c31ec734 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -45,27 +45,14 @@ if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi -if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then - echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt." - exit 1 -fi - if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml fi - -cd .. - -git clone --depth=1 https://github.com/pulp/pulp-openapi-generator.git - -# Intall requirements for ansible playbooks -pip install docker netaddr boto3 ansible - for i in {1..3} do - ansible-galaxy collection install "amazon.aws:1.5.0" && s=0 && break || s=$? && sleep 3 + ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 done if [[ $s -gt 0 ]] then @@ -73,11 +60,11 @@ then exit $s fi -cd pulp_python - +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt +fi if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_requirements.txt - mv lowerbounds_requirements.txt requirements.txt + python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi if [ -f $POST_BEFORE_INSTALL ]; then diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 837a3e75..11404f45 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -32,16 +32,14 @@ tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json echo "Containerfile:" tail -v -n +1 .ci/ansible/Containerfile +echo "Constraints Files:" +# The need not even exist. +tail -v -n +1 ../*/*constraints.txt || true + # Needed for some functional tests cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" cmd_prefix bash -c "usermod -a -G wheel pulp" -SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "stream" "generate-bindings" "lowerbounds") -if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then - # Many functional tests require these - cmd_prefix dnf install -yq lsof which -fi - if [[ "${REDIS_DISABLED:-false}" == true ]]; then cmd_prefix bash -c "s6-rc -d change redis" echo "The Redis service was disabled for $TEST" diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh new file mode 100755 index 00000000..e86ffc29 --- /dev/null +++ b/.github/workflows/scripts/build_python_client.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -python-client.tar and -python-client-docs.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +rm -rf "pulp_python-client" + +./gen-client.sh "../pulp_python/python-api.json" "python" python "pulp_python" + +pushd pulp_python-client +python setup.py sdist bdist_wheel --python-tag py3 + +twine check "dist/pulp_python_client-"*"-py3-none-any.whl" +twine check "dist/pulp_python-client-"*".tar.gz" + +tar cvf "../../pulp_python/python-python-client.tar" ./dist + +find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; +find ./docs/* -exec sed -i 's/README//g' {} \; +cp README.md docs/index.md +sed -i 's/docs\///g' docs/index.md +find ./docs/* -exec sed -i 's/\.md//g' {} \; + +cat >> mkdocs.yml << DOCSYAML +--- +site_name: PulpPython Client +site_description: Python bindings +site_author: Pulp Team +site_url: https://docs.pulpproject.org/pulp_python_client/ +repo_name: pulp/pulp_python +repo_url: https://github.com/pulp/pulp_python +theme: readthedocs +DOCSYAML + +# Building the bindings docs +mkdocs build + +# Pack the built site. +tar cvf ../../pulp_python/python-python-client-docs.tar ./site +popd +popd diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh new file mode 100755 index 00000000..6b433edd --- /dev/null +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -ruby-client.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +rm -rf "pulp_python-client" + +./gen-client.sh "../pulp_python/python-api.json" "python" ruby "pulp_python" + +pushd pulp_python-client +gem build pulp_python_client +tar cvf "../../pulp_python/python-ruby-client.tar" "./pulp_python_client-"*".gem" +popd +popd diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index 7780e8be..400e0542 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -8,22 +8,11 @@ # For more info visit https://github.com/pulp/plugin_template # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." set -euv -echo ::group::REQUESTS -pip3 install requests - -pip3 install pygithub - -echo ::endgroup:: - -for sha in $(curl -H "Authorization: token $GITHUB_TOKEN" $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g') +for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha') do - python3 .ci/scripts/validate_commit_message.py $sha - VALUE=$? - if [ "$VALUE" -gt 0 ]; then - exit $VALUE - fi + python3 .ci/scripts/validate_commit_message.py "$SHA" done diff --git a/.github/workflows/scripts/create_release_from_tag.sh b/.github/workflows/scripts/create_release_from_tag.sh deleted file mode 100755 index 328454e0..00000000 --- a/.github/workflows/scripts/create_release_from_tag.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -e - -curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/releases \ --H "Authorization: token $RELEASE_TOKEN" \ --d @- << EOF -{ - "tag_name": "$1", - "name": "$1" -} -EOF diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py deleted file mode 100755 index 61ae6e47..00000000 --- a/.github/workflows/scripts/docs-publisher.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import subprocess -import os -import re -from shutil import rmtree -import tempfile -import requests -import json -from packaging import version - -WORKING_DIR = os.environ["WORKSPACE"] - -VERSION_REGEX = r"(\s*)(version)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" -RELEASE_REGEX = r"(\s*)(release)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" - -USERNAME = "doc_builder_pulp_python" -HOSTNAME = "8.43.85.236" - -SITE_ROOT = "/var/www/docs.pulpproject.org/pulp_python/" - - -def make_directory_with_rsync(remote_paths_list): - """ - Ensure the remote directory path exists. - - :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the - remote. - :type remote_paths_list: a list of strings, with each string representing a directory. - """ - try: - tempdir_path = tempfile.mkdtemp() - cwd = os.getcwd() - os.chdir(tempdir_path) - os.makedirs(os.sep.join(remote_paths_list)) - remote_path_arg = "%s@%s:%s%s" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - remote_paths_list[0], - ) - local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep - rsync_command = ["rsync", "-avzh", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command) - if exit_code != 0: - raise RuntimeError("An error occurred while creating remote directories.") - finally: - rmtree(tempdir_path) - os.chdir(cwd) - - -def ensure_dir(target_dir, clean=True): - """ - Ensure that the directory specified exists and is empty. - - By default this will delete the directory if it already exists - - :param target_dir: The directory to process - :type target_dir: str - :param clean: Whether or not the directory should be removed and recreated - :type clean: bool - """ - if clean: - rmtree(target_dir, ignore_errors=True) - try: - os.makedirs(target_dir) - except OSError: - pass - - -def main(): - """ - Builds documentation using the 'make html' command and rsyncs to docs.pulpproject.org. - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "--build-type", required=True, help="Build type: nightly, tag or changelog." - ) - parser.add_argument("--branch", required=True, help="Branch or tag name.") - opts = parser.parse_args() - - build_type = opts.build_type - - branch = opts.branch - - publish_at_root = False - - # rsync the docs - print("rsync the docs") - docs_directory = os.sep.join([WORKING_DIR, "docs"]) - local_path_arg = os.sep.join([docs_directory, "_build", "html"]) + os.sep - if build_type == "nightly": - # This is a nightly build - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - make_directory_with_rsync(["en", branch, build_type]) - rsync_command = ["rsync", "-avzh", "--delete", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "tag": - if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: - # Only publish docs at the root if this is the latest version - r = requests.get("https://pypi.org/pypi/pulp-python/json") - latest_version = version.parse(json.loads(r.text)["info"]["version"]) - docs_version = version.parse(branch) - # This is to mitigate delays on PyPI which doesn't update metadata in timely manner. - # It doesn't prevent incorrect docs being published at root if 2 releases are done close - # to each other, within PyPI delay. E.g. Release 3.11.0 an then 3.10.1 immediately - # after. - if docs_version >= latest_version: - publish_at_root = True - # Post releases should use the x.y.z part of the version string to form a path - if "post" in branch: - branch = ".".join(branch.split(".")[:-1]) - - # This is a GA build. - # publish to the root of docs.pulpproject.org - if publish_at_root: - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "en", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - # This is a pre-release - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "nightly", - "--exclude", - "testing", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "changelog": - if branch != "main": - raise RuntimeError("Can only publish CHANGELOG from main") - # Publish the CHANGELOG from main branch at the root directory - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - changelog_local_path_arg = os.path.join(local_path_arg, "changes.html") - rsync_command = [ - "rsync", - "-vzh", - "--omit-dir-times", - changelog_local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - raise RuntimeError("Build type must be either 'nightly', 'tag' or 'changelog'.") - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index efe50656..76642845 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,44 +15,51 @@ set -euv source .github/workflows/scripts/utils.sh +PLUGIN_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')" +PLUGIN_SOURCE="./pulp_python/dist/pulp_python-${PLUGIN_VERSION}-py3-none-any.whl" + export PULP_API_ROOT="/pulp/" PIP_REQUIREMENTS=("pulp-cli") -if [[ "$TEST" = "docs" || "$TEST" = "publish" ]] -then - PIP_REQUIREMENTS+=("-r" "doc_requirements.txt") - git clone https://github.com/pulp/pulpcore.git ../pulpcore - PIP_REQUIREMENTS+=("psycopg2-binary" "-r" "../pulpcore/doc_requirements.txt") -fi +# This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} -if [[ "$TEST" != "docs" ]] -then - PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" - git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli -fi +# Check out the pulp-cli branch matching the installed version. +PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" +git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli cd .ci/ansible/ - -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - PLUGIN_NAME=./pulp_python/dist/pulp_python-$PLUGIN_VERSION-py3-none-any.whl -else - PLUGIN_NAME=./pulp_python +if [ "$TEST" = "s3" ]; then + PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[s3]" fi +if [ "$TEST" = "azure" ]; then + PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[azure]" +fi + cat >> vars/main.yaml << VARSYAML image: name: pulp tag: "ci_build" plugins: - name: pulp_python - source: "${PLUGIN_NAME}" + source: "${PLUGIN_SOURCE}" VARSYAML if [[ -f ../../ci_requirements.txt ]]; then cat >> vars/main.yaml << VARSYAML ci_requirements: true VARSYAML fi +if [ "$TEST" = "pulp" ]; then + cat >> vars/main.yaml << VARSYAML + upperbounds: true +VARSYAML +fi +if [ "$TEST" = "lowerbounds" ]; then + cat >> vars/main.yaml << VARSYAML + lowerbounds: true +VARSYAML +fi cat >> vars/main.yaml << VARSYAML services: @@ -65,14 +72,14 @@ services: - ../../../pulp-openapi-generator:/root/pulp-openapi-generator env: PULP_WORKERS: "4" + PULP_HTTPS: "true" VARSYAML cat >> vars/main.yaml << VARSYAML +pulp_env: {} pulp_settings: {"orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} pulp_scheme: https - -pulp_container_tag: https - +pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest VARSYAML if [ "$TEST" = "s3" ]; then @@ -89,25 +96,22 @@ if [ "$TEST" = "s3" ]; then minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ pulp_scenario_settings: null\ +pulp_scenario_env: {}\ +test_storages_compat_layer: false\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" fi if [ "$TEST" = "azure" ]; then - mkdir -p azurite - cd azurite - openssl req -newkey rsa:2048 -x509 -nodes -keyout azkey.pem -new -out azcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:ci-azurite" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN" - sudo cp azcert.pem /usr/local/share/ca-certificates/azcert.crt - sudo dpkg-reconfigure ca-certificates - cd .. sed -i -e '/^services:/a \ - name: ci-azurite\ image: mcr.microsoft.com/azure-storage/azurite\ volumes:\ - ./azurite:/etc/pulp\ - command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml + command: "azurite-blob --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ pulp_scenario_settings: null\ +pulp_scenario_env: {}\ ' vars/main.yaml fi @@ -117,10 +121,8 @@ if [ "${PULP_API_ROOT:-}" ]; then sed -i -e '$a api_root: "'"$PULP_API_ROOT"'"' vars/main.yaml fi -pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" -if [[ "$TEST" != "docs" ]]; then - cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" -fi +pulp config create --base-url https://pulp --api-root "$PULP_API_ROOT" --username "admin" --password "password" +cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" ansible-playbook build_container.yaml ansible-playbook start_container.yaml @@ -140,27 +142,20 @@ sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certi # Hack: adding pulp CA to certifi.where() CERTIFI=$(python -c 'import certifi; print(certifi.where())') cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERTIFI" > /dev/null -if [[ "$TEST" = "azure" ]]; then - cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a "$CERTIFI" > /dev/null -fi # Hack: adding pulp CA to default CA file CERT=$(python -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)') -cat "$CERTIFI" | sudo tee -a "$CERT" > /dev/null +cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a "$CERT" > /dev/null # Updating certs sudo update-ca-certificates echo ::endgroup:: if [[ "$TEST" = "azure" ]]; then - AZCERTIFI=$(/opt/az/bin/python3 -c 'import certifi; print(certifi.where())') - cat /usr/local/share/ca-certificates/azcert.crt >> $AZCERTIFI - cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /usr/local/lib/python3.8/site-packages/certifi/cacert.pem > /dev/null - cat /usr/local/share/ca-certificates/azcert.crt | cmd_stdin_prefix tee -a /etc/pki/tls/cert.pem > /dev/null - AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=https://ci-azurite:10000/devstoreaccount1;' + AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;' az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING fi echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" +cmd_prefix bash -c "pip3 list" echo ::endgroup:: diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh deleted file mode 100755 index 553c78cd..00000000 --- a/.github/workflows/scripts/install_python_client.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -export PULP_URL="${PULP_URL:-https://pulp}" - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -pip install twine wheel - -export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') -export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" -if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then - export VERSION=${REPORTED_VERSION} -else - export EPOCH="$(date +%s)" - export VERSION=${REPORTED_VERSION}${EPOCH} -fi - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python-client/$VERSION/) - -if [ "$response" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Installing from PyPI." - docker exec pulp pip3 install pulp-python-client==$VERSION - mkdir -p dist - tar cvf python-client.tar ./dist - exit -fi - -cd ../pulp-openapi-generator -rm -rf pulp_python-client -./generate.sh pulp_python python $VERSION -cd pulp_python-client -python setup.py sdist bdist_wheel --python-tag py3 -find . -name "*.whl" -exec docker exec pulp pip3 install /root/pulp-openapi-generator/pulp_python-client/{} \; -tar cvf ../../pulp_python/python-client.tar ./dist - -find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; -find ./docs/* -exec sed -i 's/README//g' {} \; -cp README.md docs/index.md -sed -i 's/docs\///g' docs/index.md -find ./docs/* -exec sed -i 's/\.md//g' {} \; -tar cvf ../../pulp_python/python-client-docs.tar ./docs -exit $? diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh deleted file mode 100755 index 1dd84089..00000000 --- a/.github/workflows/scripts/install_ruby_client.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -export PULP_URL="${PULP_URL:-https://pulp}" - -export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version') -export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)" -if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then - export VERSION=${REPORTED_VERSION} -else - export EPOCH="$(date +%s)" - export VERSION=${REPORTED_VERSION}${EPOCH} -fi - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_python_client/versions/$VERSION) - -if [ "$response" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Installing from RubyGems.org." - gem install pulp_python_client -v $VERSION - touch pulp_python_client-$VERSION.gem - tar cvf ruby-client.tar ./pulp_python_client-$VERSION.gem - exit -fi - -cd ../pulp-openapi-generator -rm -rf pulp_python-client -./generate.sh pulp_python ruby $VERSION -cd pulp_python-client -gem build pulp_python_client -gem install --both ./pulp_python_client-$VERSION.gem -tar cvf ../../pulp_python/ruby-client.tar ./pulp_python_client-$VERSION.gem diff --git a/.github/workflows/scripts/post_docs_test.sh b/.github/workflows/scripts/post_docs_test.sh deleted file mode 100644 index 2178e5a8..00000000 --- a/.github/workflows/scripts/post_docs_test.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env sh - -export BASE_ADDR=https://pulp -export CONTENT_ADDR=https://pulp - -pip install -r doc_requirements.txt - -cd docs/_scripts/ -bash ./quickstart.sh diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh index 2cd1d052..e771b5e7 100755 --- a/.github/workflows/scripts/publish_client_gem.sh +++ b/.github/workflows/scripts/publish_client_gem.sh @@ -12,27 +12,17 @@ set -euv # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. +VERSION="$1" -mkdir ~/.gem || true -touch ~/.gem/credentials -echo "--- -:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials -sudo chmod 600 ~/.gem/credentials - -export VERSION=$(ls pulp_python_client* | sed -rn 's/pulp_python_client-(.*)\.gem/\1/p') - -if [[ -z "$VERSION" ]]; then - echo "No client package found." - exit -fi - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulp_python_client/versions/$VERSION) - -if [ "$response" == "200" ]; +if [[ -z "${VERSION}" ]] then - echo "pulp_python client $VERSION has already been released. Skipping." - exit + echo "No version specified." + exit 1 fi -GEM_FILE="$(ls pulp_python_client-*)" -gem push ${GEM_FILE} +mkdir -p ~/.gem +touch ~/.gem/credentials +echo "--- +:rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials +sudo chmod 600 ~/.gem/credentials +gem push "pulp_python_client-${VERSION}.gem" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index 83dbcaf6..144f2597 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -10,28 +10,17 @@ set -euv # make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. +cd "$(dirname "$(realpath -e "$0")")/../../.." -pip install twine +VERSION="$1" -export VERSION=$(ls dist | sed -rn 's/pulp_python-client-(.*)\.tar.gz/\1/p') - -if [[ -z "$VERSION" ]]; then - echo "No client package found." - exit -fi - -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python-client/$VERSION/) - -if [ "$response" == "200" ]; +if [[ -z "${VERSION}" ]] then - echo "pulp_python client $VERSION has already been released. Skipping." - exit + echo "No version specified." + exit 1 fi -twine check dist/pulp_python_client-$VERSION-py3-none-any.whl || exit 1 -twine check dist/pulp_python-client-$VERSION.tar.gz || exit 1 -twine upload dist/pulp_python_client-$VERSION-py3-none-any.whl -u pulp -p $PYPI_PASSWORD -twine upload dist/pulp_python-client-$VERSION.tar.gz -u pulp -p $PYPI_PASSWORD - -exit $? +twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ +"dist/pulp_python_client-${VERSION}-py3-none-any.whl" \ +"dist/pulp_python-client-${VERSION}.tar.gz" \ +; diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh deleted file mode 100755 index b82378f2..00000000 --- a/.github/workflows/scripts/publish_docs.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -mkdir ~/.ssh -touch ~/.ssh/pulp-infra -chmod 600 ~/.ssh/pulp-infra -echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra - -echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> ~/.ssh/known_hosts -chmod 644 ~/.ssh/known_hosts - -pip3 install packaging - -export PYTHONUNBUFFERED=1 -export DJANGO_SETTINGS_MODULE=pulpcore.app.settings -export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -export WORKSPACE=$PWD - -eval "$(ssh-agent -s)" #start the ssh agent -ssh-add ~/.ssh/pulp-infra - -python3 .github/workflows/scripts/docs-publisher.py --build-type $1 --branch $2 - -if [[ "$GITHUB_WORKFLOW" == "Python changelog update" ]]; then - # Do not build bindings docs on changelog update - exit -fi - -pip install mkdocs pymdown-extensions "Jinja2<3.1" - -mkdir -p ../bindings -tar -xvf python-client-docs.tar --directory ../bindings -cd ../bindings -cat >> mkdocs.yml << DOCSYAML ---- -site_name: PulpPython Client -site_description: Python bindings -site_author: Pulp Team -site_url: https://docs.pulpproject.org/pulp_python_client/ -repo_name: pulp/pulp_python -repo_url: https://github.com/pulp/pulp_python -theme: readthedocs -DOCSYAML - -# Building the bindings docs -mkdocs build - -# publish to docs.pulpproject.org/pulp_python_client -rsync -avzh site/ doc_builder_pulp_python@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_python_client/ - -# publish to docs.pulpproject.org/pulp_python_client/en/{release} -rsync -avzh site/ doc_builder_pulp_python@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_python_client/en/"$2" diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index 981f7707..16b0ce05 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -7,23 +7,20 @@ # # For more info visit https://github.com/pulp/plugin_template +set -euv + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -euv +VERSION="$1" -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$1/) -if [ "$response" == "200" ]; +if [[ -z "${VERSION}" ]] then - echo "pulp_python $1 has already been released. Skipping." - exit + echo "No version specified." + exit 1 fi -pip install twine - -twine check dist/pulp_python-$1-py3-none-any.whl || exit 1 -twine check dist/pulp-python-$1.tar.gz || exit 1 -twine upload dist/pulp_python-$1-py3-none-any.whl -u pulp -p $PYPI_PASSWORD -twine upload dist/pulp-python-$1.tar.gz -u pulp -p $PYPI_PASSWORD - -exit $? +twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ +dist/pulp?python-"${VERSION}"-py3-none-any.whl \ +dist/pulp?python-"${VERSION}".tar.gz \ +; diff --git a/.github/workflows/scripts/push_branch_and_tag_to_github.sh b/.github/workflows/scripts/push_branch_and_tag_to_github.sh index 2f13c78d..421eb171 100755 --- a/.github/workflows/scripts/push_branch_and_tag_to_github.sh +++ b/.github/workflows/scripts/push_branch_and_tag_to_github.sh @@ -1,21 +1,16 @@ #!/bin/sh -set -e -BRANCH_NAME=$(echo $GITHUB_REF | sed -rn 's/refs\/heads\/(.*)/\1/p') +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template -ref_string=$(git show-ref --tags | grep refs/tags/$1) +set -eu -SHA=${ref_string:0:40} +BRANCH_NAME="$(echo "$GITHUB_REF" | sed -rn 's/refs\/heads\/(.*)/\1/p')" -remote_repo=https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git +remote_repo="https://pulpbot:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" -git push "${remote_repo}" $BRANCH_NAME - -curl -s -X POST https://api.github.com/repos/$GITHUB_REPOSITORY/git/refs \ --H "Authorization: token $RELEASE_TOKEN" \ --d @- << EOF -{ - "ref": "refs/tags/$1", - "sha": "$SHA" -} -EOF +git push "${remote_repo}" "$BRANCH_NAME" "$1" diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py deleted file mode 100755 index 7990377f..00000000 --- a/.github/workflows/scripts/release.py +++ /dev/null @@ -1,207 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import asyncio -import re -import os -import shutil -import textwrap - -from bandersnatch.mirror import BandersnatchMirror -from bandersnatch.master import Master -from bandersnatch.configuration import BandersnatchConfig - -from git import Repo - -from packaging.requirements import Requirement -from pathlib import Path - - -async def get_package_from_pypi(package_name, plugin_path): - """ - Download a package from PyPI. - - :param name: name of the package to download from PyPI - :return: String path to the package - """ - config = BandersnatchConfig().config - config["mirror"]["master"] = "https://pypi.org" - config["mirror"]["workers"] = "1" - config["mirror"]["directory"] = plugin_path - if not config.has_section("plugins"): - config.add_section("plugins") - config["plugins"]["enabled"] = "blocklist_release\n" - if not config.has_section("allowlist"): - config.add_section("allowlist") - config["plugins"]["enabled"] += "allowlist_release\nallowlist_project\n" - config["allowlist"]["packages"] = "\n".join([package_name]) - os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True) - async with Master("https://pypi.org/") as master: - mirror = BandersnatchMirror(homedir=plugin_path, master=master) - name = Requirement(package_name).name - result = await mirror.synchronize([name]) - package_found = False - - for package in result[name]: - current_path = os.path.join(plugin_path, package) - destination_path = os.path.join(plugin_path, "dist", os.path.basename(package)) - shutil.move(current_path, destination_path) - package_found = True - return package_found - - -def create_release_commits(repo, release_version, plugin_path): - """Build changelog, set version, commit, bump to next dev version, commit.""" - issues_to_close = set() - for filename in Path(f"{plugin_path}/CHANGES").rglob("*"): - if filename.stem.isdigit(): - issue = filename.stem - issues_to_close.add(issue) - - issues = ",".join(issues_to_close) - # First commit: changelog - os.system(f"towncrier --yes --version {release_version}") - git = repo.git - git.add("CHANGES.rst") - git.add("CHANGES/*") - git.commit("-m", f"Add changelog for {release_version}\n\n[noissue]") - - # Second commit: release version - os.system("bump2version release --allow-dirty") - - git.add(f"{plugin_path}/{plugin_name}/*") - git.add(f"{plugin_path}/docs/conf.py") - git.add(f"{plugin_path}/setup.py") - git.add(f"{plugin_path}/requirements.txt") - git.add(f"{plugin_path}/.bumpversion.cfg") - git.commit("-m", f"Release {release_version}\nGH Issues: {issues}\n\n[noissue]") - sha = repo.head.object.hexsha - short_sha = git.rev_parse(sha, short=7) - - os.system("bump2version patch --allow-dirty") - - new_dev_version = None - with open(f"{plugin_path}/setup.py") as fp: - for line in fp.readlines(): - if "version=" in line: - new_dev_version = re.split("\"|'", line)[1] - if not new_dev_version: - raise RuntimeError("Could not detect new dev version ... aborting.") - - git.add(f"{plugin_path}/{plugin_name}/*") - git.add(f"{plugin_path}/docs/conf.py") - git.add(f"{plugin_path}/setup.py") - git.add(f"{plugin_path}/requirements.txt") - git.add(f"{plugin_path}/.bumpversion.cfg") - git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]") - print(f"Release commit == {short_sha}") - print(f"All changes were committed on branch: release_{release_version}") - return sha - - -def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): - """Create a tag if one is needed and build a package if one is not on PyPI.""" - # Remove auth header config - with repo.config_writer() as conf: - conf.remove_section('http "https://github.com/"') - conf.release() - - # Determine if a tag exists and if it matches the specified commit sha - tag = None - for existing_tag in repo.tags: - if existing_tag.name == desired_tag: - if existing_tag.commit.hexsha == commit_sha: - tag = existing_tag - else: - raise RuntimeError( - f"The '{desired_tag}' tag already exists, but the commit sha does not match " - f"'{commit_sha}'." - ) - - # Create a tag if one does not exist - if not tag: - tag = repo.create_tag(desired_tag, ref=commit_sha) - - # Checkout the desired tag and reset the tree - repo.head.reference = tag.commit - repo.head.reset(index=True, working_tree=True) - - # Check if Package is available on PyPI - loop = asyncio.get_event_loop() # noqa - # fmt: off - package_found = asyncio.run( - get_package_from_pypi(f"pulp-python=={tag.name}", plugin_path) - ) # noqa - # fmt: on - if not package_found: - os.system("python3 setup.py sdist bdist_wheel --python-tag py3") - - -helper = textwrap.dedent( - """\ - Start the release process. - - Example: - setup.py on plugin before script: - version="2.0.0.dev" - - $ python .ci/scripts/release.py - - setup.py on plugin after script: - version="2.0.1.dev" - - - """ -) -parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) - -parser.add_argument( - "release_version", - type=str, - help="The version string for the release.", -) - -args = parser.parse_args() - -release_version_arg = args.release_version - -release_path = os.path.dirname(os.path.abspath(__file__)) -plugin_path = release_path.split("/.github")[0] - -plugin_name = "pulp_python" -version = None -with open(f"{plugin_path}/setup.py") as fp: - for line in fp.readlines(): - if "version=" in line: - version = re.split("\"|'", line)[1] - if not version: - raise RuntimeError("Could not detect existing version ... aborting.") -release_version = version.replace(".dev", "") - -print(f"\n\nRepo path: {plugin_path}") -repo = Repo(plugin_path) - -release_commit = None -if release_version != release_version_arg: - # Look for a commit with the requested release version - for commit in repo.iter_commits(): - if f"Release {release_version_arg}\n" in commit.message: - release_commit = commit - release_version = release_version_arg - break - if not release_commit: - raise RuntimeError( - f"The release version {release_version_arg} does not match the .dev version at HEAD. " - "A release commit for such version does not exist." - ) - -if not release_commit: - release_commit_sha = create_release_commits(repo, release_version, plugin_path) -else: - release_commit_sha = release_commit.hexsha -create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh new file mode 100755 index 00000000..a08353cd --- /dev/null +++ b/.github/workflows/scripts/release.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -eu -o pipefail + +BRANCH=$(git branch --show-current) + +if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]] +then + echo ERROR: This is not a release branch! + exit 1 +fi + +# The tail is a necessary workaround to remove the warning from the output. +NEW_VERSION="$(bump-my-version show new_version --increment release | tail -n -1)" +echo "Release ${NEW_VERSION}" + +if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] +then + echo ERROR: Version does not match release branch + exit 1 +fi + +towncrier build --yes --version "${NEW_VERSION}" +bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump-my-version bump patch --commit + +git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 42ae3d1f..00932968 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -12,15 +12,13 @@ set -mveuo pipefail # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -REPO_ROOT="$PWD" source .github/workflows/scripts/utils.sh export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh -export POST_DOCS_TEST=$PWD/.github/workflows/scripts/post_docs_test.sh export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh -# Needed for both starting the service and building the docs. +# Needed for starting the service # Gets set in .github/settings.yml, but doesn't seem to inherited by # this script. export DJANGO_SETTINGS_MODULE=pulpcore.app.settings @@ -28,114 +26,127 @@ export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py export PULP_URL="https://pulp" -if [[ "$TEST" = "docs" ]]; then - if [[ "$GITHUB_WORKFLOW" == "Python CI" ]]; then - pip install towncrier==19.9.0 - towncrier --yes --version 4.0.0.ci - fi - cd docs - make PULP_URL="$PULP_URL" diagrams html - tar -cvf docs.tar ./_build - cd .. - - if [ -f $POST_DOCS_TEST ]; then - source $POST_DOCS_TEST - fi - exit -fi - REPORTED_STATUS="$(pulp status)" -if [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - REPORTED_VERSION="$(echo $REPORTED_STATUS | jq --arg plugin python --arg legacy_plugin pulp_python -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')" - response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulp-python/$REPORTED_VERSION/) - if [ "$response" == "200" ]; - then - echo "pulp_python $REPORTED_VERSION has already been released. Skipping running tests." - exit - fi -fi +echo "${REPORTED_STATUS}" echo "machine pulp login admin password password " | cmd_user_stdin_prefix bash -c "cat >> ~pulp/.netrc" # Some commands like ansible-galaxy specifically require 600 -cmd_user_stdin_prefix bash -c "chmod 600 ~pulp/.netrc" +cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -cd ../pulp-openapi-generator -if [ "$(echo "$REPORTED_STATUS" | jq -r '.versions[0].package')" = "null" ] -then - # We are on an old version of pulpcore without package in the status report - for app_label in $(echo "$REPORTED_STATUS" | jq -r '.versions[].component') +# Generate bindings +################### + +echo "::group::Generate bindings" + +touch bindings_requirements.txt +pushd ../pulp-openapi-generator + # Use app_label to generate api.json and package to produce the proper package name. + + # Workaround: Domains are not supported by the published bindings. + # Sadly: Different pulpcore-versions aren't either... + # * In the 'pulp' scenario we use the published/prebuilt bindings, so we can test it. + # * In other scenarios we generate new bindings from server spec, so we have a more + # reliable client. + if [ "$TEST" = "pulp" ] + then + BUILT_CLIENTS=" python " + else + BUILT_CLIENTS="" + fi + + for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") do - if [ "$app_label" = "core" ] + COMPONENT="$(jq -r '.component' <<<"${ITEM}")" + VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")" + # On older status endpoints, the module was not provided, but the package should be accurate + # there, because we did not merge plugins into pulpcore back then. + MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")" + PACKAGE="${MODULE%%.*}" + cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json" + if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]] then - item=pulpcore + rm -rf "./${PACKAGE}-client" + ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}" + pushd "${PACKAGE}-client" + python setup.py sdist bdist_wheel --python-tag py3 + popd else - item="pulp_${app_label}" + if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ] + then + ls -lR "${PACKAGE}-client/" + echo "Error: Client bindings for ${COMPONENT} not found." + echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing." + exit 1 + fi fi - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done -else - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[].package|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" + echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../pulp_python/bindings_requirements.txt" done -fi +popd + +echo "::endgroup::" -cd $REPO_ROOT +echo "::group::Debug bindings diffs" -cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" -cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt +# Bindings diff for python +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "python-api.json" > "build-api.json" +jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "../pulp-openapi-generator/python-api.json" > "test-api.json" +jsondiff --indent 2 build-api.json test-api.json || true +echo "::endgroup::" + +# Install test requirements +########################### + +# Carry on previous constraints (there might be no such file). +cat *_constraints.txt > bindings_constraints.txt || true +cat .ci/assets/ci_constraints.txt >> bindings_constraints.txt +# Add a safeguard to make sure the proper versions of the clients are installed. +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' >> bindings_constraints.txt +cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') -cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt | tee -a "$CERTIFI" > /dev/null" +cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'" # check for any uncommitted migrations echo "Checking for uncommitted migrations..." cmd_user_prefix bash -c "django-admin makemigrations python --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs pulp_python.tests.unit" - +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --durations=20 --pyargs pulp_python.tests.unit" # Run functional tests if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_python.tests.performance" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs pulp_python.tests.performance" else - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_python.tests.performance.test_$PERFORMANCE_TEST" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs pulp_python.tests.performance.test_${PERFORMANCE_TEST}" fi exit fi -if [ -f $FUNC_TEST_SCRIPT ]; then - source $FUNC_TEST_SCRIPT +if [ -f "$FUNC_TEST_SCRIPT" ]; then + source "$FUNC_TEST_SCRIPT" else - - if [[ "$GITHUB_WORKFLOW" == "Python Nightly CI/CD" ]] || [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" - - - else - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_python.tests.functional -m 'not parallel'" - - - fi - + if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] + then + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" + else + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel'" + fi fi pushd ../pulp-cli pip install -r test_requirements.txt -pytest -v -m pulp_python +pytest -v -m "pulp_python" popd -if [ -f $POST_SCRIPT ]; then - source $POST_SCRIPT +if [ -f "$POST_SCRIPT" ]; then + source "$POST_SCRIPT" fi diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py new file mode 100755 index 00000000..967984a4 --- /dev/null +++ b/.github/workflows/scripts/update_backport_labels.py @@ -0,0 +1,59 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import requests +import yaml +import random +import os + + +def random_color(): + """Generates a random 24-bit number in hex""" + color = random.randrange(0, 2**24) + return format(color, "06x") + + +session = requests.Session() +token = os.getenv("GITHUB_TOKEN") + +headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", +} +session.headers.update(headers) + +# get all labels from the repository's current state +response = session.get("https://api.github.com/repos/pulp/pulp_python/labels", headers=headers) +assert response.status_code == 200 +old_labels = set([x["name"] for x in response.json() if x["name"].startswith("backport-")]) + +# get list of branches from template_config.yml +with open("./template_config.yml", "r") as f: + plugin_template = yaml.safe_load(f) +branches = set(plugin_template["supported_release_branches"]) +latest_release_branch = plugin_template["latest_release_branch"] +if latest_release_branch is not None: + branches.add(latest_release_branch) +new_labels = {"backport-" + x for x in branches} + +# delete old labels that are not in new labels +for label in old_labels.difference(new_labels): + response = session.delete( + f"https://api.github.com/repos/pulp/pulp_python/labels/{label}", headers=headers + ) + assert response.status_code == 204 + +# create new labels that are not in old labels +for label in new_labels.difference(old_labels): + color = random_color() + response = session.post( + "https://api.github.com/repos/pulp/pulp_python/labels", + headers=headers, + json={"name": label, "color": color}, + ) + assert response.status_code == 201 diff --git a/.github/workflows/scripts/update_ci.sh b/.github/workflows/scripts/update_ci.sh deleted file mode 100755 index d591f9e5..00000000 --- a/.github/workflows/scripts/update_ci.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -if [ ! -d ../plugin_template ]; then - echo "Checking out plugin_template" - git clone https://github.com/pulp/plugin_template.git ../plugin_template -fi - - -if [ ! -f "template_config.yml" ]; then - echo "No template_config.yml detected." - exit 1 -fi - -pushd ../plugin_template -pip install -r test_requirements.txt -./plugin-template --github pulp_python -popd - -# Check if only gitref file has changed, so no effect on CI workflows. -if [[ `git diff --name-only` == ".github/template_gitref" ]]; then - echo "CI update has no effect on workflows, skipping PR creation." - git stash - exit 0 -fi - -if [[ `git status --porcelain` ]]; then - git add -A - git commit -m "Update CI files" -m "[noissue]" -else - echo "No updates needed" -fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..a18b9230 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,147 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Test" +on: + workflow_call: + inputs: + matrix_env: + required: true + type: string + +defaults: + run: + working-directory: "pulp_python" + +jobs: + test: + runs-on: "ubuntu-latest" + strategy: + fail-fast: false + matrix: + env: ${{ fromJSON(inputs.matrix_env) }} + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_python" + + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + + - name: "Download plugin package" + uses: "actions/download-artifact@v4" + with: + name: "plugin_package" + path: "pulp_python/dist/" + + - name: "Download API specs" + uses: "actions/download-artifact@v4" + with: + name: "api_spec" + path: "pulp_python/" + + - name: "Download client packages" + uses: "actions/download-artifact@v4" + with: + name: "python-client.tar" + path: "pulp_python" + + - name: "Unpack client packages" + working-directory: "pulp-openapi-generator" + run: | + mkdir -p "pulp_python-client" + pushd "pulp_python-client" + tar xvf "../../pulp_python/python-python-client.tar" + popd + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install towncrier twine wheel httpie docker netaddr boto3 'ansible~=10.3.0' mkdocs jq jsonpatch bump-my-version + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_python/.ci/assets/httpie/" >> $GITHUB_ENV + echo ::endgroup:: + + - name: "Set environment variables" + run: | + echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV + + - name: "Before Install" + run: | + .github/workflows/scripts/before_install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Install" + run: | + .github/workflows/scripts/install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Before Script" + run: | + .github/workflows/scripts/before_script.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}" + + - name: "Script" + run: | + .github/workflows/scripts/script.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Extract Deprecations from Logs" + run: | + docker logs pulp 2>&1 | grep -i pulpcore.deprecation | tee deprecations-${{ matrix.env.TEST }}.txt + + - name: "Upload Deprecations" + uses: actions/upload-artifact@v4 + with: + name: "deprecations-${{ matrix.env.TEST }}" + path: "pulp_python/deprecations-${{ matrix.env.TEST }}.txt" + if-no-files-found: "error" + retention-days: 5 + overwrite: true + + - name: "Logs" + if: always() + run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp bash -c "pip3 list" || true +... diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml new file mode 100644 index 00000000..4565da8c --- /dev/null +++ b/.github/workflows/update-labels.yml @@ -0,0 +1,39 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + + +--- +name: "Python Update Labels" +on: + push: + branches: + - "main" + paths: + - "template_config.yml" + +jobs: + update_backport_labels: + runs-on: "ubuntu-latest" + steps: + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Configure Git with pulpbot name and email" + run: | + git config --global user.name 'pulpbot' + git config --global user.email 'pulp-infra@redhat.com' + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install requests pyyaml + echo ::endgroup:: + - uses: "actions/checkout@v4" + - name: "Update labels" + run: | + python3 .github/workflows/scripts/update_backport_labels.py + env: + GITHUB_TOKEN: "${{ secrets.RELEASE_TOKEN }}" diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 0990ec63..05b93764 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -7,66 +7,71 @@ --- -name: CI Update +name: "Python CI Update" on: schedule: # * is a special character in YAML so you have to quote this string # runs at 2:30 UTC every Sunday - cron: '30 2 * * 0' - workflow_dispatch: - inputs: - all_branches: - description: "Run on all branches" - default: 'no' - required: false jobs: update: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" strategy: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: "actions/checkout@v4" with: - fetch-depth: 1 + fetch-depth: 0 + repository: "pulp/plugin_template" + path: "plugin_template" - - uses: actions/setup-python@v3 + - uses: "actions/setup-python@v5" with: - python-version: "3.8" + python-version: "3.11" - - name: Install python dependencies + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install gitpython requests packaging jinja2 pyyaml + pip install gitpython packaging -r plugin_template/requirements.txt echo ::endgroup:: - - name: Configure Git with pulpbot name and email + - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' + - uses: "actions/checkout@v4" + with: + fetch-depth: 0 + path: "pulp_python" + ref: "main" - - name: Set short_ref - id: vars - run: echo short_ref=${GITHUB_REF#refs/*/} >> $GITHUB_OUTPUT - - - name: Run update + - name: "Run update" + working-directory: "pulp_python" run: | - .github/workflows/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh - - name: Create Pull Request for CI files - uses: peter-evans/create-pull-request@v4 + - name: "Create Pull Request for CI files" + uses: "peter-evans/create-pull-request@v6" + id: "create_pr_main" with: - token: ${{ secrets.RELEASE_TOKEN }} - committer: pulpbot - author: pulpbot - title: 'Update CI files from ${{ steps.vars.outputs.short_ref }} branch' - body: '[noissue]' - branch: 'create-pull-request/${{ steps.vars.outputs.short_ref }}/patch' - commit-message: | - Update CI files - - [noissue] + token: "${{ secrets.RELEASE_TOKEN }}" + path: "pulp_python" + committer: "pulpbot " + author: "pulpbot " + title: "Update CI files for branch main" + branch: "update-ci/main" + base: "main" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_python" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_main.outputs.pull-request-number }}" + if: "steps.create_pr_main.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true +... diff --git a/CHANGES/.TEMPLATE.rst b/CHANGES/.TEMPLATE.md similarity index 64% rename from CHANGES/.TEMPLATE.rst rename to CHANGES/.TEMPLATE.md index ab3826e7..2308193b 100644 --- a/CHANGES/.TEMPLATE.rst +++ b/CHANGES/.TEMPLATE.md @@ -1,37 +1,39 @@ - {# TOWNCRIER TEMPLATE #} {% for section, _ in sections.items() %} -{% set underline = underlines[0] %}{% if section %}{{section}} -{{ underline * section|length }}{% set underline = underlines[1] %} +{%- set section_slug = "-" + section|replace(" ", "-")|replace("_", "-")|lower %} +{%- if section %} +### {{section}} {: #{{versiondata.version}}{{section_slug}} } +{% else %} +{%- set section_slug = "" %} {% endif %} - {% if sections[section] %} {% for category, val in definitions.items() if category in sections[section]%} -{{ definitions[category]['name'] }} -{{ underline * definitions[category]['name']|length }} + +#### {{ definitions[category]['name'] }} {: #{{versiondata.version}}{{section_slug}}-{{category}} } {% if definitions[category]['showcontent'] %} {% for text, values in sections[section][category].items() %} - {{ text }} +{% if values %} {{ values|join(',\n ') }} +{% endif %} {% endfor %} - {% else %} - {{ sections[section][category]['']|join(', ') }} - {% endif %} {% if sections[section][category]|length == 0 %} -No significant changes. +No significant changes. {% else %} {% endif %} - {% endfor %} {% else %} -No significant changes. - +No significant changes. {% endif %} {% endfor %} ----- + +--- + + diff --git a/MANIFEST.in b/MANIFEST.in index aca0d594..94a64886 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,3 +8,4 @@ include functest_requirements.txt include test_requirements.txt include unittest_requirements.txt include pulp_python/app/webserver_snippets/* +exclude releasing.md diff --git a/doc_requirements.txt b/doc_requirements.txt index 925c79a0..6daacdbe 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,14 +1,8 @@ -build -coreapi -django -djangorestframework -django-filter -drf-nested-routers -mistune -plantuml -pyyaml -sphinx -sphinx-rtd-theme -sphinxcontrib-openapi +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_python' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template towncrier -twine +pulp-docs @ git+https://github.com/pulp/pulp-docs@rewrite-as-mkdocs-plugin diff --git a/functest_requirements.txt b/functest_requirements.txt index c43d8b3d..cc539046 100644 --- a/functest_requirements.txt +++ b/functest_requirements.txt @@ -1,7 +1,9 @@ git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash -pytest +pytest<9 +pytest-custom_exit_code +pytest-xdist lxml twine pypi-simple -pulpcore-client -pulp-python-client +trustme~=1.2.0 +pytest-timeout diff --git a/lint_requirements.txt b/lint_requirements.txt index 4e7958f1..a2936209 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,7 +5,8 @@ # # For more info visit https://github.com/pulp/plugin_template -# python packages handy for developers, but not required by pulp +bump-my-version check-manifest flake8 - +packaging +yamllint diff --git a/pyproject.toml b/pyproject.toml index cba5c3f3..10e62a57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,3 +25,48 @@ ignore = [ "lint_requirements.txt", ".flake8", ] + +[tool.bumpversion] +# This section is managed by the plugin template. Do not edit manually. + +current_version = "3.10.1.dev" +commit = false +tag = false +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P0a)?(?P\\d+)(\\.(?P[a-z]+))?" +serialize = [ + "{major}.{minor}.{patch}.{release}", + "{major}.{minor}.{patch}", + "{major}.{minor}.{alpha}{patch}.{release}", + "{major}.{minor}.{alpha}{patch}", +] + +[tool.bumpversion.parts.alpha] +# This section is managed by the plugin template. Do not edit manually. + +# This is sort of a hack. In PEP440 prerelease markers work quite differently. +# But this fits best with the way we have been doing release versions. +optional_value = "final" +values = [ + "0a", + "final", +] +independent = true + +[tool.bumpversion.parts.release] +# This section is managed by the plugin template. Do not edit manually. + +optional_value = "prod" +values = [ + "dev", + "prod", +] + +[[tool.bumpversion.files]] +# This section is managed by the plugin template. Do not edit manually. + +filename = "./pulp_python/app/__init__.py" +search = "version = \"{current_version}\"" +replace = "version = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = "./setup.py" diff --git a/releasing.md b/releasing.md new file mode 100644 index 00000000..db7470b2 --- /dev/null +++ b/releasing.md @@ -0,0 +1,29 @@ +[//]: # "WARNING: DO NOT EDIT!" +[//]: # "" +[//]: # "This file was generated by plugin_template, and is managed by it. Please use" +[//]: # "'./plugin-template --github pulp_python' to update this file." +[//]: # "" +[//]: # "For more info visit https://github.com/pulp/plugin_template" +# Releasing (For Internal Use) + +This document outlines the steps to perform a release. + +### Determine if a Release is Required +- Make sure to have GitPython python package installed +- Run the release checker script: + ``` + python3 .ci/scripts/check_release.py + ``` + +### Release a New Y-version (e.g., 3.23.0) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_python/actions/workflows/create-branch.yml) job from the main branch via the GitHub Actions. +- Look for the "Bump minor version" pull request and merge it. +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_python/actions/workflows/release.yml) job by specifying the new release branch (X.**Y**) via the GitHub Actions. + +### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_python/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. + +## Final Steps +- Ensure the new version appears on PyPI (it should appear after [Publish Release](https://github.com/pulp/pulp_python/actions/workflows/publish.yml) workflow succeeds). +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request (A new PR should be available on the next day). +- [optional] Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). diff --git a/requirements.txt b/requirements.txt index 99699677..f1271e41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pulpcore>=3.25.0,<3.40 +pulpcore>=3.28.0,<3.40 pkginfo>=1.8.2,<1.9.7 bandersnatch>=6.1,<6.2 pypi-simple>=0.9.0,<1.0.0 diff --git a/template_config.yml b/template_config.yml index 3748bd8c..0de37e3f 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,18 +1,17 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-212-g18b9346 +# generated with plugin_template -additional_repos: [] api_root: /pulp/ black: false check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: true +ci_base_image: ghcr.io/pulp/pulp-ci-centos9 ci_env: {} ci_trigger: '{pull_request: {branches: [''*'']}}' -ci_update_branches: [] cli_package: pulp-cli cli_repo: https://github.com/pulp/pulp-cli.git core_import_allowed: [] @@ -20,30 +19,26 @@ deploy_client_to_pypi: true deploy_client_to_rubygems: true deploy_to_pypi: true disabled_redis_runners: [] -doc_requirements_from_pulpcore: true docker_fixtures: false -docs_test: true flake8: true flake8_ignore: [] github_org: pulp -issue_tracker: github -kanban: true +latest_release_branch: null lint_requirements: true -noissue_marker: '[noissue]' +os_required_packages: [] parallel_test_workers: 8 plugin_app_label: python -plugin_camel: PulpPython -plugin_camel_short: Python -plugin_caps: PULP_PYTHON -plugin_caps_short: PYTHON -plugin_dash: pulp-python -plugin_dash_short: python plugin_default_branch: main plugin_name: pulp_python -plugin_snake: pulp_python +plugins: +- app_label: python + name: pulp_python post_job_template: null pre_job_template: null -publish_docs_to_pulpprojectdotorg: true +pulp_env: {} +pulp_env_azure: {} +pulp_env_gcp: {} +pulp_env_s3: {} pulp_scheme: https pulp_settings: orphan_protection_time: 0 @@ -51,19 +46,14 @@ pulp_settings: pulp_settings_azure: null pulp_settings_gcp: null pulp_settings_s3: null -pulp_settings_stream: null -pulpprojectdotorg_key_id: null pydocstyle: true -pypi_username: pulp -python_version: '3.8' release_email: pulp-infra@redhat.com release_user: pulpbot -run_pulpcore_tests_for_plugins: false -single_commit_check: true stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 stalebot_limit_to_pulls: true +supported_release_branches: [] sync_ci: true test_azure: true test_cli: true @@ -73,7 +63,5 @@ test_lowerbounds: true test_performance: false test_reroute: true test_s3: true -test_stream: false -update_github: true +test_storages_compat_layer: false use_issue_template: true - diff --git a/test_requirements.txt b/test_requirements.txt index d5c09b9c..2544df3c 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,10 +1,3 @@ -coveralls -coverage -flake8 -flake8-docstrings -flake8-tuple -flake8-quotes -mock -git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash -pydocstyle<4 -pytest +# All test requirements +-r functest_requirements.txt +-r unittest_requirements.txt From 546774a60dc2742ee94c8370d9e9737828a3d886 Mon Sep 17 00:00:00 2001 From: Jitka Obselkova <41325380+jobselko@users.noreply.github.com> Date: Tue, 1 Jul 2025 17:09:38 +0200 Subject: [PATCH 2/2] Merge pull request #877 from jobselko/fix_nightly Fix tests to account for new fixtures (cherry picked from commit ee1aed2b7412b58a8803fe54a1e241ae3f6471af) (cherry picked from commit 7433f61c8fb0f38c3bec3ce2894cacbddda4f00e) --- CHANGES/876.bugfix | 1 + .../functional/api/test_download_content.py | 5 ++++- .../tests/functional/api/test_full_mirror.py | 21 ++++++++++++------- pulp_python/tests/functional/constants.py | 10 +++++---- 4 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 CHANGES/876.bugfix diff --git a/CHANGES/876.bugfix b/CHANGES/876.bugfix new file mode 100644 index 00000000..621a1a84 --- /dev/null +++ b/CHANGES/876.bugfix @@ -0,0 +1 @@ +Fixed tests to account for new fixtures. diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index 2f20cca0..2f2b2eb6 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -14,6 +14,7 @@ PYTHON_MD_PROJECT_SPECIFIER, PYTHON_MD_FIXTURE_SUMMARY, PYTHON_LG_FIXTURE_SUMMARY, + PYTHON_LG_FIXTURE_PRERELEASES_SUMMARY, PYTHON_LG_PROJECT_SPECIFIER, ) from pulp_python.tests.functional.utils import ( @@ -126,7 +127,9 @@ def test_full_fixtures_to_pulp_sync(self): """ remote = self._create_remote(includes=[], prereleases=True) repo = self._create_repo_and_sync_with_remote(remote) - self.assertEqual(get_content_summary(repo.to_dict()), PYTHON_LG_FIXTURE_SUMMARY) + self.assertEqual( + get_content_summary(repo.to_dict()), PYTHON_LG_FIXTURE_PRERELEASES_SUMMARY + ) def test_full_pulp_to_pulp_sync(self): """ diff --git a/pulp_python/tests/functional/api/test_full_mirror.py b/pulp_python/tests/functional/api/test_full_mirror.py index ddfa60d6..bbcbe9e3 100644 --- a/pulp_python/tests/functional/api/test_full_mirror.py +++ b/pulp_python/tests/functional/api/test_full_mirror.py @@ -108,21 +108,26 @@ def test_pull_through_install(self): remote = self._create_remote(url=PYPI_URL) distro = self._create_distribution(remote=remote.pulp_href) - if self.cli_client.run(("pip", "list")).stdout.find("numpy") == -1: - content = self.content_api.list(name="numpy") - self.assertEqual(content.count, 0, msg="numpy content already present in test") + PACKAGE = "sampleproject" + if self.cli_client.run(("pip", "list")).stdout.find(PACKAGE) == -1: + content = self.content_api.list(name=PACKAGE) + self.assertEqual( + content.count, 0, msg=f"{PACKAGE} content already present in test" + ) host = urlsplit(PULP_CONTENT_BASE_URL).hostname url = urljoin(self.PYPI_HOST, f"{distro.base_path}/simple/") out = self.cli_client.run( - ("pip", "install", "--trusted-host", host, "-i", url, "numpy") + ("pip", "install", "--trusted-host", host, "-i", url, PACKAGE) ) self.addCleanup(delete_orphans) - self.assertTrue(self.cli_client.run(("pip", "list")).stdout.find("numpy") != -1, out) - self.addCleanup(self.cli_client.run, ("pip", "uninstall", "numpy", "-y")) - content = self.content_api.list(name="numpy") + self.assertTrue( + self.cli_client.run(("pip", "list")).stdout.find(PACKAGE) != -1, out + ) + self.addCleanup(self.cli_client.run, ("pip", "uninstall", PACKAGE, "-y")) + content = self.content_api.list(name=PACKAGE) self.assertEqual(content.count, 1) else: - self.skipTest("Uninstall numpy before running this test") + self.skipTest(f"Uninstall {PACKAGE} before running this test") def test_pull_through_simple(self): """Tests that the simple page is properly modified when requesting a pull-through.""" diff --git a/pulp_python/tests/functional/constants.py b/pulp_python/tests/functional/constants.py index 83083439..698464ce 100644 --- a/pulp_python/tests/functional/constants.py +++ b/pulp_python/tests/functional/constants.py @@ -145,14 +145,16 @@ "celery", # matches 13 "Django", # matches 31 "scipy", # matches 23 + "setuptools", # matches 2 "shelf-reader", # matches 2 ] -PYTHON_LG_PACKAGE_COUNT = 76 +PYTHON_LG_PACKAGE_COUNT = 78 PYTHON_LG_FIXTURE_SUMMARY = {PYTHON_CONTENT_NAME: PYTHON_LG_PACKAGE_COUNT} +PYTHON_LG_FIXTURE_PRERELEASES_SUMMARY = {PYTHON_CONTENT_NAME: 92} PYTHON_LG_FIXTURE_COUNTS = { - "latest_3": 35, - "sdist": 23, - "bdist_wheel": 53, + "latest_3": 37, + "sdist": 24, + "bdist_wheel": 54, } DJANGO_LATEST_3 = 4 # latest version has 2 dists, each other has 1