diff --git a/README.md b/README.md index 69ae17b3..a368c996 100644 --- a/README.md +++ b/README.md @@ -171,8 +171,6 @@ The following settings are stored in `template_config.yml`. ci_trigger Value for the `on` clause on workflow/ci.yml (push, pull_request, etc...) ci_env Environment variables to set for the CI build. - pre_job_template holds name and a path for a template to be included to run before jobs. - post_job_template holds name and a path for a template to be included to run after jobs. lint_requirements Boolean (defaults True) to enable upper bound check on requirements.txt ``` diff --git a/plugin-template b/plugin-template index 41d275a1..5b91154e 100755 --- a/plugin-template +++ b/plugin-template @@ -1,15 +1,26 @@ #!/usr/bin/env python3 - +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "jamldump>=1.2.0,<1.3.0", +# "jinja2>=3.1.6,<3.2.0", +# "pyyaml>=6.0.3,<6.1.0", +# "requests~=2.32.3", +# "requests-cache>=1.3.0,<1.4.0", +# "tomlkit>=0.14.0,<0.14.1", +# ] +# /// + +import typing as t import argparse import os -import pprint import shlex import shutil import subprocess import sys -import textwrap from pathlib import Path +import jamldump import yaml from jinja2 import Environment, FileSystemLoader @@ -45,8 +56,6 @@ DEFAULT_SETTINGS = { "plugin_default_branch": "main", "plugin_name": None, "plugins": None, - "post_job_template": None, - "pre_job_template": None, "pulp_env": {}, "pulp_env_azure": {}, "pulp_env_gcp": {}, @@ -185,7 +194,7 @@ DEPRECATED_FILES = { } -def main(): +def main() -> int: parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, description="Create or update a plugin using the current template.", @@ -193,161 +202,83 @@ def main(): parser.add_argument( "plugin_name", type=str, - help=textwrap.dedent( - """\ - Create or update this plugin. The name should start with pulp- or pulp_. - - """ - ), + nargs="?", + help="Create or update this plugin. The name should start with pulp- or pulp_.", ) parser.add_argument( "--plugin-app-label", type=str, - help=textwrap.dedent( - """\ - the Django app label for the plugin - usually the part after pulp_ or pulp-. - - """ - ), + help="The Django app label for the plugin - defaults to the part after pulp_ or pulp-.", ) parser.add_argument( "--generate-config", action="store_true", - help=textwrap.dedent( - """\ - Create or update a plugin template config for a plugin and exit. - - """ - ), + help="Create or update a plugin template config for a plugin and exit.", ) parser.add_argument( "--bootstrap", action="store_true", - help=textwrap.dedent( - """\ - Create a new plugin and template boilerplate code. - - """ - ), + help="Create a new plugin and template boilerplate code.", ) parser.add_argument( "--test", action="store_true", - help=textwrap.dedent( - """\ - Generate or update functional and unit tests. - - """ - ), + help="Generate or update functional and unit tests.", ) parser.add_argument( "--github", action="store_true", - help=textwrap.dedent( - """\ - Generate or update github CI/CD configuration files. - - """ - ), + help="Generate or update github CI/CD configuration files.", ) parser.add_argument( "--all", action="store_true", - help=textwrap.dedent( - """\ - Create a new plugin and template all non-excluded files. - - """ - ), + help="Create a new plugin and template all non-excluded files.", ) parser.add_argument( "--verbose", action="store_true", - help=textwrap.dedent( - """\ - Include more output. - - """ - ), + help="Include more output.", ) parser.add_argument( "--latest-release-branch", metavar="VERSION_BRANCH", action="store", - help=textwrap.dedent( - """\ - Mark specified version as the latest_release_branch before templating. - - """ - ), + help="Mark specified version as the latest_release_branch before templating.", ) parser.add_argument( "--plugin-root-dir", action="store", - help=textwrap.dedent( - """\ - Root directory of the plugin if it's not '../{plugin_name}' - """ - ), + help="Root directory of the plugin if it's not '../{plugin_name}'.", ) args = parser.parse_args() - here = os.path.dirname(os.path.abspath(__file__)) - plugin_root_dir = args.plugin_root_dir or os.path.join(os.path.dirname(here), args.plugin_name) - plugin_config_path = os.path.join(plugin_root_dir, "template_config.yml") + here = Path(__file__).parent.resolve() + if args.plugin_root_dir is not None: + plugin_root_dir = Path(args.plugin_root_dir).resolve() + elif args.plugin_name is None: + # Assume this is called from an existing plugin directory. + plugin_root_dir = Path(".").resolve() + else: + plugin_root_dir = (here.parent / args.plugin_name).resolve() + plugin_config_path = plugin_root_dir / "template_config.yml" write_new_config = False - try: - with open(plugin_config_path) as config_file: - try: - config_in_file = yaml.safe_load(config_file) - if config_in_file: - config = config_in_file - # Add any missing value from the list of defaults - for key, value in DEFAULT_SETTINGS.items(): - if key not in config: - config[key] = value - write_new_config = True - # remove deprecated options - for key in set(config.keys()) - set(DEFAULT_SETTINGS.keys()): - config.pop(key) - write_new_config = True - # TODO: validate config - # validate versions are strings - assert all( - ( - isinstance(version, str) - for version in config["supported_release_branches"] - ) - ) - print( - "\nLoaded plugin template config from " - "{path}/template_config.yml.\n".format(path=plugin_root_dir) - ) - - if args.verbose: - print("\nUsing the following config:\n") - pp = pprint.PrettyPrinter(indent=4) - pp.pprint(config) - print("\n") - except yaml.YAMLError as exc: - print(exc) - return 2 - except FileNotFoundError: - print( - "\nCould not find a plugin template config at {path}/template_config.yml.\n".format( - path=plugin_root_dir - ) - ) + if plugin_config_path.exists(): + config, write_new_config = load_config(plugin_config_path, args.verbose) + if args.plugin_name is not None and args.plugin_name != config["plugin_name"]: + raise RuntimeError("Aborting: 'plugin_name' in the config does not match.") + if ( + args.plugin_app_label is not None + and args.plugin_app_label != config["plugin_app_label"] + ): + raise RuntimeError("Aborting: 'plugin_app_label' in the config does not match.") + + else: + print(f"Could not find a plugin template config at {plugin_config_path}.") if args.all or args.generate_config: - if not args.plugin_app_label: - print( - "\n--plugin-app-label needs to be specified when generating a template " - "config. See ./plugin-template -h for usage.\n" - ) - return 2 - else: - config = generate_config(args.plugin_name, args.plugin_app_label) + print(f"Creating a new plugin at {plugin_root_dir}") + config = generate_new_config(args.plugin_name, args.plugin_app_label) else: return 2 @@ -355,55 +286,69 @@ def main(): config["latest_release_branch"] = str(args.latest_release_branch) write_new_config = True - # Config key is used by the template_config.yml.j2 template to dump - # the config. (note: uses .copy() to avoid a self reference) - config["config"] = config.copy() - sections = [ section for section in ["generate_config", "bootstrap", "github", "test"] - if getattr(args, section) or args.all + if getattr(args, section) or args.all or (section == "generate_config" and write_new_config) ] for section in sections: write_template_section(config, section, plugin_root_dir, verbose=args.verbose) - if write_new_config and not (args.generate_config or args.all): - write_template_section(config, "generate_config", plugin_root_dir, verbose=args.verbose) - file_path = os.path.join(plugin_root_dir, "template_config.yml") - print("\nAn updated plugin template config written to {path}.\n".format(path=file_path)) + # This is a dummy call left here as an example for future migrations to come. + migrate_dummy(plugin_root_dir, config, sections) - if "github" in sections: - migrate_dummy(plugin_root_dir) + remove_deprecated_files(plugin_root_dir, sections) - if plugin_root_dir: - print("\nDeprecation check:") - check_for_deprecated_files(plugin_root_dir, sections) + reformat_files(plugin_root_dir, config) - if config["black"]: - try: - subprocess.run(["black", "--quiet", "."], cwd=plugin_root_dir) - except FileNotFoundError: - pass + return 0 + + +def load_config(plugin_config_path: Path, verbose: bool) -> tuple[dict[str, t.Any], bool]: + write_new_config = False + with open(plugin_config_path) as config_file: + config = yaml.safe_load(config_file) + # Add any missing value from the list of defaults + for key, value in DEFAULT_SETTINGS.items(): + if key not in config: + config[key] = value + write_new_config = True + # remove deprecated options + for key in set(config.keys()) - set(DEFAULT_SETTINGS.keys()): + config.pop(key) + write_new_config = True + # TODO: validate config + # validate versions are strings + assert all((isinstance(version, str) for version in config["supported_release_branches"])) + print(f"Loaded plugin template config from {plugin_config_path}.") + + if verbose: + print("Using the following config:") + print(jamldump.to_jaml(config)) + + return config, write_new_config -def migrate_dummy(plugin_root_dir): +def migrate_dummy(plugin_root_dir: Path, config: dict[str, t.Any], sections: list[str]) -> None: pass -def write_template_section(config, name, plugin_root_dir, verbose=False): +def write_template_section( + config: dict[str, t.Any], section: str, plugin_root_dir: Path, verbose: bool = False +) -> None: """ Template or copy all files for the section. """ - plugin_root_path = Path(plugin_root_dir) - section_template_dir = f"templates/{name}" + templates_dir = Path(__file__).parent.resolve() / "templates" + section_templates_dir = templates_dir / section env = Environment( loader=FileSystemLoader( [ - section_template_dir, # The specified section folder - "templates", # The default templates folder - "../", # The parent dir to allow including pre/post templates from + section_templates_dir, # Where all templates are actually templated. + templates_dir / "include", # Included to find the macros. ] ), + extensions=["jamldump.jinja2.Jaml"], line_statement_prefix="%%", line_comment_prefix="%#", ) @@ -420,19 +365,20 @@ def write_template_section(config, name, plugin_root_dir, verbose=False): PULPDOCS_BRANCH = "main" template_vars = { - "section": name, - "setup_py": (plugin_root_path / "setup.py").exists(), + "section": section, + "setup_py": (plugin_root_dir / "setup.py").exists(), "ci_update_branches": utils.ci_update_branches(config), "python_version": "3.11", "ci_update_hour": sum((ord(c) for c in config["plugin_app_label"])) % 24, - "current_version": utils.current_version(plugin_root_path), + "current_version": utils.current_version(plugin_root_dir), "pulpdocs_branch": PULPDOCS_BRANCH, "is_pulpdocs_member": config["plugin_name"] in utils.get_pulpdocs_members(PULPDOCS_BRANCH), "black_version": utils.black_version(), + "config": config, **config, } - relative_path_set = generate_relative_path_set(section_template_dir) + relative_path_set = generate_relative_path_set(section_templates_dir) for relative_path in relative_path_set: if not config["stalebot"] and "stale" in relative_path: continue @@ -454,14 +400,14 @@ def write_template_section(config, name, plugin_root_dir, verbose=False): if destination.startswith("pyproject.toml."): utils.merge_toml( template, - plugin_root_path, + plugin_root_dir, destination, template_vars, ) else: utils.template_to_file( template, - plugin_root_path, + plugin_root_dir, destination, template_vars, ) @@ -472,18 +418,17 @@ def write_template_section(config, name, plugin_root_dir, verbose=False): if destination_relative_path.endswith(".copy"): destination_relative_path = destination_relative_path[: -len(".copy")] shutil.copy( - os.path.join(section_template_dir, relative_path), - os.path.join(plugin_root_dir, destination_relative_path), + section_templates_dir / relative_path, + plugin_root_dir / destination_relative_path, ) files_copied += 1 if verbose: print(f"Copied file: {relative_path}") - print(f"Section: {name} \n templated: {files_templated}\n copied: {files_copied}") - return 0 + print(f"Section: {section} \n templated: {files_templated}\n copied: {files_copied}") -def generate_relative_path_set(root_dir): +def generate_relative_path_set(root_dir: Path) -> set[str]: """ Create a set of relative paths within the specified directory. """ @@ -496,7 +441,7 @@ def generate_relative_path_set(root_dir): return applicable_paths -def generate_config(plugin_name, plugin_app_label): +def generate_new_config(plugin_name: str, plugin_app_label: str | None) -> dict[str, t.Any]: """ Generates a default config for a plugin @@ -508,6 +453,8 @@ def generate_config(plugin_name, plugin_app_label): Returns: config (dict): a dictionary containing a default template configuration for the plugin """ + if plugin_app_label is None: + plugin_app_label = plugin_name.removeprefix("pulp").replace("-", "_").strip("_") config = DEFAULT_SETTINGS.copy() config["plugin_name"] = plugin_name config["plugin_app_label"] = plugin_app_label @@ -515,8 +462,9 @@ def generate_config(plugin_name, plugin_app_label): return config -def check_for_deprecated_files(plugin_root_dir, sections): +def remove_deprecated_files(plugin_root_dir: Path, sections: list[str]) -> None: """Check for files that have been deprecated (ie moved or removed).""" + print("Deprecation check:") files_found = False for section in sections: for fp in DEPRECATED_FILES.get(section, []): @@ -524,14 +472,22 @@ def check_for_deprecated_files(plugin_root_dir, sections): if path.exists(): if path.is_dir(): shutil.rmtree(path) - print(f"Removed deprecated directory: '{path}'.") + print(f" Removed deprecated directory: '{path}'.") else: path.unlink() - print(f"Removed deprecated file: '{path}'.") + print(f" Removed deprecated file: '{path}'.") files_found = True if not files_found: - print("No deprecated files found.") + print(" No deprecated files found.") + + +def reformat_files(plugin_root_dir: Path, config: dict[str, t.Any]) -> None: + if config["black"]: + try: + subprocess.run(["black", "--quiet", "."], cwd=plugin_root_dir) + except FileNotFoundError: + pass if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index fb4fc085..85104f8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ # This needs to match the version in templates/github/lint_requirement.txt.j2 black==24.3.0 +jamldump>=1.2.0,<1.3.0 jinja2 pyyaml requests~=2.32.3 diff --git a/scripts/get_template_config_value.py b/scripts/get_template_config_value.py index 82046990..3636e634 100755 --- a/scripts/get_template_config_value.py +++ b/scripts/get_template_config_value.py @@ -1,20 +1,30 @@ +# /// script +# dependencies = [ +# "pyyaml>=6.0.3,<6.1.0", +# ] +# +# /// + +import sys + import argparse import yaml -# Parse the command-line argument -parser = argparse.ArgumentParser() -parser.add_argument("value_name", help="Name of the value to extract") -args = parser.parse_args() +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("value_name", help="Name of the value to extract") + args = parser.parse_args() -# Read the YAML file -with open("template_config.yml") as file: - data = yaml.safe_load(file) + with open("template_config.yml") as file: + data = yaml.safe_load(file) -# Extract the value based on the provided name -value = data.get(args.value_name) + try: + value = data.get(args.value_name) + except KeyError: + sys.exit(1) -# Output the value -if value is not None: - print(value) -else: - print("") + # Output the value + if value is not None: + print(value) + else: + print("") diff --git a/scripts/update_ci.sh b/scripts/update_ci.sh index 637917ca..79a77811 100755 --- a/scripts/update_ci.sh +++ b/scripts/update_ci.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eu +set -eu -o pipefail if [ ! -f "template_config.yml" ] then @@ -18,19 +18,19 @@ then "template_config.yml" fi -plugin_name="$(python ../plugin_template/scripts/get_template_config_value.py plugin_name)" -ci_update_docs="$(python ../plugin_template/scripts/get_template_config_value.py ci_update_docs)" -use_black="$(python ../plugin_template/scripts/get_template_config_value.py black)" +PLUGIN_NAME="$(python ../plugin_template/scripts/get_template_config_value.py plugin_name)" +CI_UPDATE_DOCS="$(python ../plugin_template/scripts/get_template_config_value.py ci_update_docs)" +USE_BLACK="$(python ../plugin_template/scripts/get_template_config_value.py black)" -if [[ "${ci_update_docs}" == "True" ]]; then - docs=("--docs") +if [[ "${CI_UPDATE_DOCS}" == "True" ]]; then + DOCS=("--docs") else - docs=() + DOCS=() fi pushd ../plugin_template -pip install -r requirements.txt -./plugin-template --github "${docs[@]}" "${plugin_name}" + pip install -r requirements.txt + ./plugin-template --github "${DOCS[@]}" "${PLUGIN_NAME}" popd if [[ $(git status --porcelain) ]]; then @@ -40,7 +40,7 @@ else echo "No updates needed" fi -if [[ "${use_black}" == "True" ]] +if [[ "${USE_BLACK}" == "True" ]] then pip install -r lint_requirements.txt black . @@ -55,7 +55,7 @@ then fi # Check that pulpcore lowerbounds is set to a supported branch -if [[ "$plugin_name" != "pulpcore" ]]; then +if [[ "${PLUGIN_NAME}" != "pulpcore" ]]; then python ../plugin_template/scripts/update_core_lowerbound.py if [[ $(git status --porcelain) ]]; then git add -A diff --git a/scripts/update_core_lowerbound.py b/scripts/update_core_lowerbound.py index c02aeb27..bc09c089 100755 --- a/scripts/update_core_lowerbound.py +++ b/scripts/update_core_lowerbound.py @@ -1,4 +1,13 @@ #!/bin/env python +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "packaging>=26.0,<26.1", +# "pyyaml>=6.0.3,<6.1.0", +# "requests>=2.32.5,<2.33.0", +# "tomlkit>=0.14.0,<0.14.1", +# ] +# /// import requests import tomlkit diff --git a/templates/generate_config/template_config.yml.j2 b/templates/generate_config/template_config.yml.j2 index 2e88a569..3254a096 100644 --- a/templates/generate_config/template_config.yml.j2 +++ b/templates/generate_config/template_config.yml.j2 @@ -2,5 +2,7 @@ # were not present before running plugin-template have been added with their default values. # generated with plugin_template +# +# After editing this file please always reapply the plugin template before committing any changes. -{{ config | to_yaml }} +{{ config | jaml }} diff --git a/templates/github/.ci/scripts/check_release.py b/templates/github/.ci/scripts/check_release.py index 81bae1fd..86e250e4 100755 --- a/templates/github/.ci/scripts/check_release.py +++ b/templates/github/.ci/scripts/check_release.py @@ -1,11 +1,20 @@ #!/usr/bin/env python +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# "pyyaml>=6.0.3,<6.1.0", +# ] +# /// import argparse import re import os import tomllib -import yaml from pathlib import Path + +import yaml from packaging.version import Version from git import Repo diff --git a/templates/github/.github/workflows/ci.yml.j2 b/templates/github/.github/workflows/ci.yml.j2 index ec12c716..4fffe125 100644 --- a/templates/github/.github/workflows/ci.yml.j2 +++ b/templates/github/.github/workflows/ci.yml.j2 @@ -19,9 +19,6 @@ defaults: working-directory: "{{ plugin_name }}" jobs: - {%- if pre_job_template %} - {% include pre_job_template.path | indent(2) %} - {%- endif %} {%- if check_commit_message or lint_requirements %} check-commits: runs-on: "ubuntu-latest" @@ -86,9 +83,6 @@ jobs: lint: needs: - "check-changes" - {%- if pre_job_template %} - - "{{ pre_job_template.name }}" - {%- endif %} if: needs.check-changes.outputs.run_tests == '1' uses: "./.github/workflows/lint.yml" @@ -170,7 +164,3 @@ jobs: check_jobs "$FILTERS" echo "CI says: Looks good!" - -{%- if post_job_template %} - {% include post_job_template.path | indent (2) %} -{%- endif %} diff --git a/templates/header.j2 b/templates/include/header.j2 similarity index 100% rename from templates/header.j2 rename to templates/include/header.j2 diff --git a/templates/header.md.j2 b/templates/include/header.md.j2 similarity index 100% rename from templates/header.md.j2 rename to templates/include/header.md.j2 diff --git a/templates/macros.j2 b/templates/include/macros.j2 similarity index 100% rename from templates/macros.j2 rename to templates/include/macros.j2 diff --git a/utils.py b/utils.py index 64b60b65..57bf0b19 100644 --- a/utils.py +++ b/utils.py @@ -1,13 +1,13 @@ from datetime import timedelta from pathlib import Path import re -import requests_cache import stat import textwrap -import tomlkit import tomllib -import yaml +import requests_cache +import tomlkit +import yaml # Jinja tests and filters @@ -110,7 +110,9 @@ def get_pulpdocs_members(pulpdocs_branch="main") -> list[str]: Raises if can't get the authoritative file. """ - session = requests_cache.CachedSession(".requests_cache", expire_after=timedelta(days=1)) + session = requests_cache.CachedSession( + Path(__file__).parent / ".requests_cache", expire_after=timedelta(days=1) + ) response = session.get( f"https://raw.githubusercontent.com/pulp/pulp-docs/{pulpdocs_branch}/mkdocs.yml" )