diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 93ee2cc7..66c494e9 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -53,6 +53,9 @@ def to_upper_bound(req): if requirement.name == "pulpcore": # An exception to allow for pulpcore deprecation policy. return fetch_pulpcore_upper_bound(requirement) + # skip requirement with environment scopes. E.g 'foo==1.0.0;python_version>=3.9' + if requirement.marker: + return f"# ENVIRONMENT IS UNTRACKABLE: {req}" for spec in requirement.specifier: if spec.operator == "~=": return f"# NO BETTER CONSTRAINT: {req}" diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index dfc4a41e..81bae1fd 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -10,8 +10,8 @@ from git import Repo RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" -Y_CHANGELOG_EXTS = [".feature", ".removal", ".deprecation"] -Z_CHANGELOG_EXTS = [".bugfix", ".doc", ".misc"] +Y_CHANGELOG_EXTS = [".feature"] +Z_CHANGELOG_EXTS = [".bugfix", ".misc"] def options(): diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 8cae45a9..877ebc8c 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -7,16 +7,21 @@ # For more info visit https://github.com/pulp/plugin_template import itertools +import json import os import re import tomllib +import urllib.request +from pathlib import Path from git import GitCommandError, Repo from packaging.version import parse as parse_version + +PYPI_PROJECT = "pulp_python" + # Read Towncrier settings -with open("pyproject.toml", "rb") as fp: - tc_settings = tomllib.load(fp)["tool"]["towncrier"] +tc_settings = tomllib.loads(Path("pyproject.toml").read_text())["tool"]["towncrier"] CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( @@ -35,7 +40,7 @@ # see help(re.split) for more info. NAME_REGEX = r".*" VERSION_REGEX = r"[0-9]+\.[0-9]+\.[0-9][0-9ab]*" -VERSION_CAPTURE_REGEX = rf"({VERSION_REGEX})" +VERSION_CAPTURE_REGEX = rf"(?:YANKED )?({VERSION_REGEX})" DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}" TITLE_REGEX = ( "(" @@ -75,6 +80,20 @@ def main(): branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True) branches = [ref.name for ref in branches] + changed = False + + try: + response = urllib.request.urlopen(f"https://pypi.org/pypi/{PYPI_PROJECT}/json") + pypi_record = json.loads(response.read()) + yanked_versions = { + parse_version(version): release[0]["yanked_reason"] + for version, release in pypi_record["releases"].items() + if release[0]["yanked"] is True + } + except Exception: + # If something failed, just don't mark anything as yanked. + yanked_versions = {} + with open(CHANGELOG_FILE, "r") as f: main_changelog = f.read() preamble, main_changes = split_changelog(main_changelog) @@ -95,9 +114,19 @@ def main(): if left[0] != right[0]: main_changes.append(right) + if yanked_versions: + for change in main_changes: + if change[0] in yanked_versions and "YANKED" not in change[1].split("\n")[0]: + reason = yanked_versions[change[0]] + version = str(change[0]) + change[1] = change[1].replace(version, "YANKED " + version, count=1) + if reason: + change[1] = change[1].replace("\n", f"\n\nYank reason: {reason}\n", count=1) + changed = True + new_length = len(main_changes) - if old_length < new_length: - print(f"{new_length - old_length} new versions have been added.") + if old_length < new_length or changed: + print(f"{new_length - old_length} new versions have been added (or something has changed).") with open(CHANGELOG_FILE, "w") as fp: fp.write(preamble) for change in main_changes: diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py old mode 100755 new mode 100644 index bb9a7d8b..a4dc9004 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -1,66 +1,31 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_python' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template +# This file is managed by the plugin template. +# Do not edit. import os import re import subprocess import sys import tomllib +import yaml from pathlib import Path from github import Github -with open("pyproject.toml", "rb") as fp: - PYPROJECT_TOML = tomllib.load(fp) -KEYWORDS = ["fixes", "closes"] -BLOCKING_REGEX = [ - r"^DRAFT", - r"^WIP", - r"^NOMERGE", - r"^DO\s*NOT\s*MERGE", - r"^EXPERIMENT", - r"^FIXUP", - r"^fixup!", # This is created by 'git commit --fixup' - r"Apply suggestions from code review", # This usually comes from GitHub -] -try: - CHANGELOG_EXTS = [ - f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] - ] -except KeyError: - CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] -NOISSUE_MARKER = "[noissue]" - -sha = sys.argv[1] -message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") - -if NOISSUE_MARKER in message: - sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") - -blocking_matches = [m for m in (re.match(pattern, message) for pattern in BLOCKING_REGEX) if m] -if blocking_matches: - print("Found these phrases in the commit message:") - for m in blocking_matches: - print(" - " + m.group(0)) - sys.exit("This PR is not ready for consumption.") -g = Github(os.environ.get("GITHUB_TOKEN")) -repo = g.get_repo("pulp/pulp_python") - - -def check_status(issue): +def check_status(issue, repo, cherry_pick): gi = repo.get_issue(int(issue)) if gi.pull_request: sys.exit(f"Error: issue #{issue} is a pull request.") - if gi.closed_at: + if gi.closed_at and not cherry_pick: + print("Make sure to use 'git cherry-pick -x' when backporting a change.") + print( + "If a backport of a change requires a significant amount of rewriting, " + "consider creating a new issue." + ) sys.exit(f"Error: issue #{issue} is closed.") -def check_changelog(issue): +def check_changelog(issue, CHANGELOG_EXTS): matches = list(Path("CHANGES").rglob(f"{issue}.*")) if len(matches) < 1: @@ -70,18 +35,63 @@ def check_changelog(issue): sys.exit(f"Invalid extension for changelog entry '{match}'.") -print("Checking commit message for {sha}.".format(sha=sha[0:7])) +def main() -> None: + TEMPLATE_CONFIG = yaml.safe_load(Path("template_config.yml").read_text()) + GITHUB_ORG = TEMPLATE_CONFIG["github_org"] + PLUGIN_NAME = TEMPLATE_CONFIG["plugin_name"] + + with Path("pyproject.toml").open("rb") as _fp: + PYPROJECT_TOML = tomllib.load(_fp) + KEYWORDS = ["fixes", "closes"] + BLOCKING_REGEX = [ + r"^DRAFT", + r"^WIP", + r"^NOMERGE", + r"^DO\s*NOT\s*MERGE", + r"^EXPERIMENT", + r"^FIXUP", + r"^fixup!", # This is created by 'git commit --fixup' + r"Apply suggestions from code review", # This usually comes from GitHub + ] + try: + CHANGELOG_EXTS = [ + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + ] + except KeyError: + CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] + NOISSUE_MARKER = "[noissue]" + + sha = sys.argv[1] + message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") + + if NOISSUE_MARKER in message: + sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") + + blocking_matches = [m for m in (re.match(pattern, message) for pattern in BLOCKING_REGEX) if m] + if blocking_matches: + print("Found these phrases in the commit message:") + for m in blocking_matches: + print(" - " + m.group(0)) + sys.exit("This PR is not ready for consumption.") + + g = Github(os.environ.get("GITHUB_TOKEN")) + repo = g.get_repo(f"{GITHUB_ORG}/{PLUGIN_NAME}") + + print("Checking commit message for {sha}.".format(sha=sha[0:7])) + + # validate the issue attached to the commit + issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords="|".join(KEYWORDS)) + issues = re.findall(issue_regex, message, re.IGNORECASE) + cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" + cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) + + if issues: + for issue in issues: + check_status(issue, repo, cherry_pick) + check_changelog(issue, CHANGELOG_EXTS) -# validate the issue attached to the commit -issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) -issues = re.findall(issue_regex, message, re.IGNORECASE) -cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" -cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) + print("Commit message for {sha} passed.".format(sha=sha[0:7])) -if issues: - for issue in issues: - if not cherry_pick: - check_status(issue) - check_changelog(issue) -print("Commit message for {sha} passed.".format(sha=sha[0:7])) +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d82f9fee..0d16d126 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install requests pygithub + pip install requests pygithub pyyaml echo ::endgroup:: - name: "Check commit message" if: github.event_name == 'pull_request' diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 9ea30712..75c3224d 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -20,6 +20,9 @@ jobs: strategy: fail-fast: false + permissions: + contents: write + steps: - uses: "actions/checkout@v4" with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d5a55a50..44f17e93 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,12 +6,12 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: "Docs" +name: "Docs CI" on: workflow_call: jobs: - test: + changelog: if: "endsWith(github.base_ref, 'main')" runs-on: "ubuntu-latest" defaults: @@ -24,31 +24,24 @@ jobs: path: "pulp_python" - uses: "actions/setup-python@v5" with: - python-version: "3.11" - - name: "Setup cache key" - run: | - git ls-remote https://github.com/pulp/pulp-docs main | tee pulp-docs-main-sha - - uses: "actions/cache@v4" - with: - path: "~/.cache/pip" - key: ${{ runner.os }}-pip-${{ hashFiles('pulp-docs-main-sha') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: "3.12" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install -r doc_requirements.txt + pip install towncrier echo ::endgroup:: - name: "Build changelog" run: | towncrier build --yes --version 4.0.0.ci - - name: "Build docs" - run: | - pulp-docs build + docs: + if: "endsWith(github.base_ref, 'main')" + uses: 'pulp/pulp-docs/.github/workflows/docs-ci.yml@rewrite-as-mkdocs-plugin' + with: + pulpdocs_ref: 'rewrite-as-mkdocs-plugin' no-test: if: "!endsWith(github.base_ref, 'main')" runs-on: "ubuntu-latest" steps: - run: | - echo "Skip docs testing on non-main branches." + echo "Skip docs testing on non-default branches." diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f155d81c..ec3fcb11 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -43,7 +43,7 @@ jobs: - uses: "actions/setup-python@v5" with: - python-version: "3.11" + python-version: "3.13" - name: "Install python dependencies" run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 274490d9..c9505f82 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -146,6 +146,9 @@ jobs: - "publish-python-bindings" - "publish-ruby-bindings" + permissions: + contents: write + steps: - name: "Create release on GitHub" uses: "actions/github-script@v7" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 0faec043..00932968 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -119,7 +119,7 @@ echo "Checking for uncommitted migrations..." cmd_user_prefix bash -c "django-admin makemigrations python --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --pyargs pulp_python.tests.unit" +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --durations=20 --pyargs pulp_python.tests.unit" # Run functional tests if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then @@ -135,11 +135,11 @@ if [ -f "$FUNC_TEST_SCRIPT" ]; then else if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] then - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel' --nightly" else - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_python.tests.functional -m 'not parallel'" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --durations=20 --pyargs pulp_python.tests.functional -m 'not parallel'" fi fi pushd ../pulp-cli diff --git a/doc_requirements.txt b/doc_requirements.txt index 6456b7d8..6daacdbe 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -5,4 +5,4 @@ # # For more info visit https://github.com/pulp/plugin_template towncrier -pulp-docs @ git+https://github.com/pulp/pulp-docs@main +pulp-docs @ git+https://github.com/pulp/pulp-docs@rewrite-as-mkdocs-plugin