From dc9ef01001835cda79513c91010c2334c616e769 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 7 Aug 2024 16:14:03 +0200 Subject: [PATCH 01/13] use sandboxed env --- conda_forge_tick/update_recipe/v2/__init__.py | 18 +++ .../update_recipe/v2/build_number.py | 55 ++++++++ .../update_recipe/v2/conditional_list.py | 69 ++++++++++ conda_forge_tick/update_recipe/v2/context.py | 16 +++ .../update_recipe/v2/jinja/__init__.py | 0 .../update_recipe/v2/jinja/filters.py | 28 ++++ .../update_recipe/v2/jinja/jinja.py | 116 ++++++++++++++++ .../update_recipe/v2/jinja/objects.py | 35 +++++ .../update_recipe/v2/jinja/utils.py | 11 ++ conda_forge_tick/update_recipe/v2/source.py | 47 +++++++ conda_forge_tick/update_recipe/v2/version.py | 124 ++++++++++++++++++ conda_forge_tick/update_recipe/v2/yaml.py | 22 ++++ 12 files changed, 541 insertions(+) create mode 100644 conda_forge_tick/update_recipe/v2/__init__.py create mode 100644 conda_forge_tick/update_recipe/v2/build_number.py create mode 100644 conda_forge_tick/update_recipe/v2/conditional_list.py create mode 100644 conda_forge_tick/update_recipe/v2/context.py create mode 100644 conda_forge_tick/update_recipe/v2/jinja/__init__.py create mode 100644 conda_forge_tick/update_recipe/v2/jinja/filters.py create mode 100644 conda_forge_tick/update_recipe/v2/jinja/jinja.py create mode 100644 conda_forge_tick/update_recipe/v2/jinja/objects.py create mode 100644 conda_forge_tick/update_recipe/v2/jinja/utils.py create mode 100644 conda_forge_tick/update_recipe/v2/source.py create mode 100644 conda_forge_tick/update_recipe/v2/version.py create mode 100644 conda_forge_tick/update_recipe/v2/yaml.py diff --git a/conda_forge_tick/update_recipe/v2/__init__.py b/conda_forge_tick/update_recipe/v2/__init__.py new file mode 100644 index 000000000..4a21dc82f --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/__init__.py @@ -0,0 +1,18 @@ +import io +from ruamel.yaml import YAML + +yaml = YAML() +yaml.preserve_quotes = True +yaml.width = 4096 +yaml.indent(mapping=2, sequence=4, offset=2) + +def load_yaml(file: Path) -> dict: + """Load a YAML file.""" + with file.open("r") as f: + return yaml.load(f) + +def dump_yaml_to_str(data: dict) -> str: + """Dump a dictionary to a YAML string.""" + with io.StringIO() as f: + yaml.dump(data, f) + return f.getvalue() \ No newline at end of file diff --git a/conda_forge_tick/update_recipe/v2/build_number.py b/conda_forge_tick/update_recipe/v2/build_number.py new file mode 100644 index 000000000..8776eca29 --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/build_number.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Literal +from conda_forge_tick.update_recipe.v2.yaml import _load_yaml, _dump_yaml_to_str +if TYPE_CHECKING: + from pathlib import Path + +logger = logging.getLogger(__name__) + +HashType = Literal["md5", "sha256"] + + +def _update_build_number_in_context(recipe: dict[str, Any], new_build_number: int) -> bool: + for key in recipe.get("context", {}): + if key.startswith("build_") or key == "build": + recipe["context"][key] = new_build_number + return True + return False + + +def _update_build_number_in_recipe(recipe: dict[str, Any], new_build_number: int) -> bool: + is_modified = False + if "build" in recipe and "number" in recipe["build"]: + recipe["build"]["number"] = new_build_number + is_modified = True + + if "outputs" in recipe: + for output in recipe["outputs"]: + if "build" in output and "number" in output["build"]: + output["build"]["number"] = new_build_number + is_modified = True + + return is_modified + + +def update_build_number(file: Path, new_build_number: int = 0) -> str: + """ + Update the build number in the recipe file. + + Arguments: + ---------- + * `file` - The path to the recipe file. + * `new_build_number` - The new build number to use. (default: 0) + + Returns: + -------- + The updated recipe as a string. + """ + data = _load_yaml(file) + build_number_modified = _update_build_number_in_context(data, new_build_number) + if not build_number_modified: + _update_build_number_in_recipe(data, new_build_number) + + return _dump_yaml_to_str(data) diff --git a/conda_forge_tick/update_recipe/v2/conditional_list.py b/conda_forge_tick/update_recipe/v2/conditional_list.py new file mode 100644 index 000000000..68c1205f9 --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/conditional_list.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Generic, List, TypeVar, Union, cast + +if TYPE_CHECKING: + from collections.abc import Callable, Generator + +T = TypeVar("T") +K = TypeVar("K") + + +class IfStatement(Generic[T]): + if_: Any + then: T | list[T] + else_: T | list[T] | None + + +ConditionalList = Union[T, IfStatement[T], List[Union[T, IfStatement[T]]]] + + +def visit_conditional_list( # noqa: C901 + value: T | IfStatement[T] | list[T | IfStatement[T]], + evaluator: Callable[[Any], bool] | None = None, +) -> Generator[T, None, None]: + """ + A function that yields individual branches of a conditional list. + + Arguments + --------- + * `value` - The value to evaluate + * `evaluator` - An optional evaluator to evaluate the `if` expression. + + Returns + ------- + A generator that yields the individual branches. + """ + + def yield_from_list(value: list[K] | K) -> Generator[K, None, None]: + if isinstance(value, list): + yield from value + else: + yield value + + if not isinstance(value, list): + value = [value] + + for element in value: + if isinstance(element, dict): + if (expr := element.get("if", None)) is not None: + then = element.get("then") + otherwise = element.get("else") + # Evaluate the if expression if the evaluator is provided + if evaluator: + if evaluator(expr): + yield from yield_from_list(then) + elif otherwise: + yield from yield_from_list(otherwise) + # Otherwise, just yield the branches + else: + yield from yield_from_list(then) + if otherwise: + yield from yield_from_list(otherwise) + else: + # In this case its not an if statement + yield cast(T, element) + # If the element is not a dictionary, just yield it + else: + # (tim) I get a pyright error here, but I don't know how to fix it + yield cast(T, element) diff --git a/conda_forge_tick/update_recipe/v2/context.py b/conda_forge_tick/update_recipe/v2/context.py new file mode 100644 index 000000000..cd8bc1e2b --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/context.py @@ -0,0 +1,16 @@ +import jinja2 + +def load_recipe_context(context: dict[str, str], jinja_env: jinja2.Environment) -> dict[str, str]: + """ + Load all string values from the context dictionary as Jinja2 templates. + Use linux-64 as default target_platform, build_platform, and mpi. + """ + # Process each key-value pair in the dictionary + for key, value in context.items(): + # If the value is a string, render it as a template + if isinstance(value, str): + template = jinja_env.from_string(value) + rendered_value = template.render(context) + context[key] = rendered_value + + return context diff --git a/conda_forge_tick/update_recipe/v2/jinja/__init__.py b/conda_forge_tick/update_recipe/v2/jinja/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/conda_forge_tick/update_recipe/v2/jinja/filters.py b/conda_forge_tick/update_recipe/v2/jinja/filters.py new file mode 100644 index 000000000..b28e4555d --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/jinja/filters.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from rattler_build_conda_compat.jinja.utils import _MissingUndefined + + +def _version_to_build_string(some_string: str | _MissingUndefined) -> str: + """ + Converts some version by removing the . character and returning only the first two elements of the version. + If piped value is undefined, it returns the undefined value as is. + """ + if isinstance(some_string, _MissingUndefined): + return f"{some_string._undefined_name}_version_to_build_string" # noqa: SLF001 + # We first split the string by whitespace and take the first part + split = some_string.split()[0] if some_string.split() else some_string + # We then split the string by . and take the first two parts + parts = split.split(".") + major = parts[0] if len(parts) > 0 else "" + minor = parts[1] if len(parts) > 1 else "" + return f"{major}{minor}" + + +def _bool(value: str) -> bool: + return bool(value) + + +def _split(s: str, sep: str = " ") -> list[str]: + """Filter that split a string by a separator""" + return s.split(sep) diff --git a/conda_forge_tick/update_recipe/v2/jinja/jinja.py b/conda_forge_tick/update_recipe/v2/jinja/jinja.py new file mode 100644 index 000000000..89229bf7f --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/jinja/jinja.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from typing import Any, TypedDict + +import jinja2 +import yaml + +from rattler_build_conda_compat.jinja.filters import _bool, _split, _version_to_build_string +from rattler_build_conda_compat.jinja.objects import ( + _stub_compatible_pin, + _stub_is_linux, + _stub_is_unix, + _stub_is_win, + _stub_match, + _stub_subpackage_pin, + _StubEnv, +) +from rattler_build_conda_compat.jinja.utils import _MissingUndefined +from rattler_build_conda_compat.loader import load_yaml + + +class RecipeWithContext(TypedDict, total=False): + context: dict[str, str] + + +def jinja_env() -> jinja2.Environment: + """ + Create a `rattler-build` specific Jinja2 environment with modified syntax. + Target platform, build platform, and mpi are set to linux-64 by default. + """ + env = jinja2.sandbox.SandboxedEnvironment( + variable_start_string="${{", + variable_end_string="}}", + trim_blocks=True, + lstrip_blocks=True, + autoescape=jinja2.select_autoescape(default_for_string=False), + undefined=_MissingUndefined, + ) + + env_obj = _StubEnv() + + # inject rattler-build recipe functions in jinja environment + env.globals.update( + { + "compiler": lambda x: x + "_compiler_stub", + "stdlib": lambda x: x + "_stdlib_stub", + "pin_subpackage": _stub_subpackage_pin, + "pin_compatible": _stub_compatible_pin, + "cdt": lambda *args, **kwargs: "cdt_stub", # noqa: ARG005 + "env": env_obj, + "match": _stub_match, + "is_unix": _stub_is_unix, + "is_win": _stub_is_win, + "is_linux": _stub_is_linux, + "unix": True, + "linux": True, + "target_platform": "linux-64", + "build_platform": "linux-64", + "mpi": "mpi", + } + ) + + # inject rattler-build recipe filters in jinja environment + env.filters.update( + { + "version_to_buildstring": _version_to_build_string, + "split": _split, + "bool": _bool, + } + ) + return env + + +def load_recipe_context(context: dict[str, str], jinja_env: jinja2.Environment) -> dict[str, str]: + """ + Load all string values from the context dictionary as Jinja2 templates. + Use linux-64 as default target_platform, build_platform, and mpi. + """ + # Process each key-value pair in the dictionary + for key, value in context.items(): + # If the value is a string, render it as a template + if isinstance(value, str): + template = jinja_env.from_string(value) + rendered_value = template.render(context) + context[key] = rendered_value + + return context + + +def render_recipe_with_context(recipe_content: RecipeWithContext) -> dict[str, Any]: + """ + Render the recipe using known values from context section. + Unknown values are not evaluated and are kept as it is. + Target platform, build platform, and mpi are set to linux-64 by default. + + Examples: + --- + ```python + >>> from pathlib import Path + >>> from rattler_build_conda_compat.loader import load_yaml + >>> recipe_content = load_yaml((Path().resolve() / "tests" / "data" / "eval_recipe_using_context.yaml").read_text()) + >>> evaluated_context = render_recipe_with_context(recipe_content) + >>> assert "my_value-${{ not_present_value }}" == evaluated_context["build"]["string"] + >>> + ``` + """ + env = jinja_env() + context = recipe_content.get("context", {}) + # render out the context section and retrieve dictionary + context_variables = load_recipe_context(context, env) + + # render the rest of the document with the values from the context + # and keep undefined expressions _as is_. + template = env.from_string(yaml.dump(recipe_content)) + rendered_content = template.render(context_variables) + return load_yaml(rendered_content) diff --git a/conda_forge_tick/update_recipe/v2/jinja/objects.py b/conda_forge_tick/update_recipe/v2/jinja/objects.py new file mode 100644 index 000000000..32e64bab6 --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/jinja/objects.py @@ -0,0 +1,35 @@ +from __future__ import annotations + + +class _StubEnv: + """A class to represent the env object used in rattler-build recipe.""" + + def get(self, env_var: str, default: str | None = None) -> str: # noqa: ARG002 + return f"""env_"{env_var}" """ + + def exists(self, env_var: str) -> str: + return f"""env_exists_"{env_var}" """ + + +def _stub_compatible_pin(*args, **kwargs) -> str: # noqa: ARG001, ANN003, ANN002 + return f"compatible_pin {args[0]}" + + +def _stub_subpackage_pin(*args, **kwargs) -> str: # noqa: ARG001, ANN003, ANN002 + return f"subpackage_pin {args[0]}" + + +def _stub_match(*args, **kwargs) -> str: # noqa: ARG001, ANN003, ANN002 + return f"match {args[0]}" + + +def _stub_is_unix(platform: str) -> str: + return f"is_unix {platform}" + + +def _stub_is_win(platform: str) -> str: + return f"is_win {platform}" + + +def _stub_is_linux(platform: str) -> str: + return f"is_linux {platform}" diff --git a/conda_forge_tick/update_recipe/v2/jinja/utils.py b/conda_forge_tick/update_recipe/v2/jinja/utils.py new file mode 100644 index 000000000..dc688e937 --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/jinja/utils.py @@ -0,0 +1,11 @@ +from jinja2 import DebugUndefined + + +class _MissingUndefined(DebugUndefined): + def __str__(self) -> str: + """ + By default, `DebugUndefined` return values in the form `{{ value }}`. + `rattler-build` has a different syntax, so we need to override this method, + and return the value in the form `${{ value }}`. + """ + return f"${super().__str__()}" diff --git a/conda_forge_tick/update_recipe/v2/source.py b/conda_forge_tick/update_recipe/v2/source.py new file mode 100644 index 000000000..8339bfe89 --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/source.py @@ -0,0 +1,47 @@ +import typing +from typing import Union, List, TypedDict, Any, Iterator, NotRequired, Mapping +from conda_forge_tick.update_recipe.v2.conditional_list import visit_conditional_list, ConditionalList + +OptionalUrlList = Union[str, List[str], None] + +class Source(TypedDict): + url: NotRequired[str | list[str]] + sha256: NotRequired[str] + md5: NotRequired[str] + + +def get_all_sources(recipe: Mapping[Any, Any]) -> Iterator[Source]: + """ + Get all sources from the recipe. This can be from a list of sources, + a single source, or conditional and its branches. + + Arguments + --------- + * `recipe` - The recipe to inspect. This should be a yaml object. + + Returns + ------- + A list of source objects. + """ + sources = recipe.get("source", None) + sources = typing.cast(ConditionalList[Source], sources) + + # Try getting all url top-level sources + if sources is not None: + source_list = visit_conditional_list(sources, None) + for source in source_list: + yield source + + outputs = recipe.get("outputs", None) + if outputs is None: + return + + outputs = visit_conditional_list(outputs, None) + for output in outputs: + sources = output.get("source", None) + sources = typing.cast(ConditionalList[Source], sources) + if sources is None: + continue + source_list = visit_conditional_list(sources, None) + for source in source_list: + yield source diff --git a/conda_forge_tick/update_recipe/v2/version.py b/conda_forge_tick/update_recipe/v2/version.py new file mode 100644 index 000000000..8eccdacd5 --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/version.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import copy +import hashlib +import logging +import re +from typing import TYPE_CHECKING, Literal + +from conda_forge_tick.update_recipe.v2.context import load_recipe_context +from conda_forge_tick.update_recipe.v2.jinja import jinja_env +from conda_forge_tick.update_recipe.v2.source import Source, get_all_sources +from conda_forge_tick.update_recipe.v2.yaml import _load_yaml, _dump_yaml_to_str + +import requests + +if TYPE_CHECKING: + from pathlib import Path + +logger = logging.getLogger(__name__) + +HashType = Literal["md5", "sha256"] + +class CouldNotUpdateVersionError(Exception): + NO_CONTEXT = "Could not find context in recipe" + NO_VERSION = "Could not find version in recipe context" + + def __init__(self, message: str = "Could not update version") -> None: + self.message = message + super().__init__(self.message) + + +class Hash: + def __init__(self, hash_type: HashType, hash_value: str) -> None: + self.hash_type = hash_type + self.hash_value = hash_value + + def __str__(self) -> str: + return f"{self.hash_type}: {self.hash_value}" + + +def _has_jinja_version(url: str) -> bool: + """Check if the URL has a jinja `${{ version }}` in it.""" + pattern = r"\${{\s*version" + return re.search(pattern, url) is not None + + +def update_hash(source: Source, url: str, hash_: Hash | None) -> None: + """ + Update the sha256 hash in the source dictionary. + + Arguments: + ---------- + * `source` - The source dictionary to update. + * `url` - The URL to download and hash (if no hash is provided). + * `hash_` - The hash to use. If not provided, the file will be downloaded and `sha256` hashed. + """ + hash_type: HashType = hash_.hash_type if hash_ is not None else "sha256" + # delete all old hashes that we are not updating + all_hash_types: set[HashType] = {"md5", "sha256"} + for key in all_hash_types - {hash_type}: + if key in source: + del source[key] + + if hash_ is not None: + source[hash_.hash_type] = hash_.hash_value + else: + # download and hash the file + hasher = hashlib.sha256() + print(f"Retrieving and hashing {url}") + with requests.get(url, stream=True, timeout=100) as r: + for chunk in r.iter_content(chunk_size=4096): + hasher.update(chunk) + source["sha256"] = hasher.hexdigest() + + +def update_version(file: Path, new_version: str, hash_: Hash | None) -> str: + """ + Update the version in the recipe file. + + Arguments: + ---------- + * `file` - The path to the recipe file. + * `new_version` - The new version to use. + * `hash_type` - The hash type to use. If not provided, the file will be downloaded and `sha256` hashed. + + Returns: + -------- + The updated recipe as a string. + """ + + data = _load_yaml(file) + + if "context" not in data: + raise CouldNotUpdateVersionError(CouldNotUpdateVersionError.NO_CONTEXT) + if "version" not in data["context"]: + raise CouldNotUpdateVersionError(CouldNotUpdateVersionError.NO_VERSION) + + data["context"]["version"] = new_version + + # set up the jinja context + env = jinja_env() + context = copy.deepcopy(data.get("context", {})) + context_variables = load_recipe_context(context, env) + # for r-recipes we add the default `cran_mirror` variable + context_variables["cran_mirror"] = "https://cran.r-project.org" + + for source in get_all_sources(data): + # render the whole URL and find the hash + if "url" not in source: + continue + + url = source["url"] + if isinstance(url, list): + url = url[0] + + if not _has_jinja_version(url): + continue + + template = env.from_string(url) + rendered_url = template.render(context_variables) + + update_hash(source, rendered_url, hash_) + + return _dump_yaml_to_str(data) diff --git a/conda_forge_tick/update_recipe/v2/yaml.py b/conda_forge_tick/update_recipe/v2/yaml.py new file mode 100644 index 000000000..4efd901a7 --- /dev/null +++ b/conda_forge_tick/update_recipe/v2/yaml.py @@ -0,0 +1,22 @@ +import io +from ruamel.yaml import YAML +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +yaml = YAML() +yaml.preserve_quotes = True +yaml.width = 4096 +yaml.indent(mapping=2, sequence=4, offset=2) + +def _load_yaml(file: Path) -> dict: + """Load a YAML file.""" + with file.open("r") as f: + return yaml.load(f) + +def _dump_yaml_to_str(data: dict) -> str: + """Dump a dictionary to a YAML string.""" + with io.StringIO() as f: + yaml.dump(data, f) + return f.getvalue() \ No newline at end of file From e5751ef381917aba7c4b3a30f377ab81520829fc Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 7 Aug 2024 16:15:49 +0200 Subject: [PATCH 02/13] remove init contents --- conda_forge_tick/update_recipe/v2/__init__.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/conda_forge_tick/update_recipe/v2/__init__.py b/conda_forge_tick/update_recipe/v2/__init__.py index 4a21dc82f..e69de29bb 100644 --- a/conda_forge_tick/update_recipe/v2/__init__.py +++ b/conda_forge_tick/update_recipe/v2/__init__.py @@ -1,18 +0,0 @@ -import io -from ruamel.yaml import YAML - -yaml = YAML() -yaml.preserve_quotes = True -yaml.width = 4096 -yaml.indent(mapping=2, sequence=4, offset=2) - -def load_yaml(file: Path) -> dict: - """Load a YAML file.""" - with file.open("r") as f: - return yaml.load(f) - -def dump_yaml_to_str(data: dict) -> str: - """Dump a dictionary to a YAML string.""" - with io.StringIO() as f: - yaml.dump(data, f) - return f.getvalue() \ No newline at end of file From 4c83a3b2f494ec5272caf70f17315f7218e7e979 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 7 Aug 2024 16:18:50 +0200 Subject: [PATCH 03/13] add tests --- .../build_number/test_1/expected.yaml | 10 +++++ .../recipe_v2/build_number/test_1/recipe.yaml | 10 +++++ .../build_number/test_2/expected.yaml | 11 ++++++ .../recipe_v2/build_number/test_2/recipe.yaml | 11 ++++++ tests/recipe_v2/version/test_1/expected.yaml | 12 ++++++ tests/recipe_v2/version/test_1/recipe.yaml | 12 ++++++ tests/recipe_v2/version/test_2/expected.yaml | 15 ++++++++ tests/recipe_v2/version/test_2/recipe.yaml | 15 ++++++++ tests/recipe_v2/version/test_3/expected.yaml | 11 ++++++ tests/recipe_v2/version/test_3/recipe.yaml | 11 ++++++ tests/recipe_v2/version/test_4/expected.yaml | 14 +++++++ tests/recipe_v2/version/test_4/recipe.yaml | 14 +++++++ tests/test_recipe_editing_v2.py | 38 +++++++++++++++++++ 13 files changed, 184 insertions(+) create mode 100644 tests/recipe_v2/build_number/test_1/expected.yaml create mode 100644 tests/recipe_v2/build_number/test_1/recipe.yaml create mode 100644 tests/recipe_v2/build_number/test_2/expected.yaml create mode 100644 tests/recipe_v2/build_number/test_2/recipe.yaml create mode 100644 tests/recipe_v2/version/test_1/expected.yaml create mode 100644 tests/recipe_v2/version/test_1/recipe.yaml create mode 100644 tests/recipe_v2/version/test_2/expected.yaml create mode 100644 tests/recipe_v2/version/test_2/recipe.yaml create mode 100644 tests/recipe_v2/version/test_3/expected.yaml create mode 100644 tests/recipe_v2/version/test_3/recipe.yaml create mode 100644 tests/recipe_v2/version/test_4/expected.yaml create mode 100644 tests/recipe_v2/version/test_4/recipe.yaml create mode 100644 tests/test_recipe_editing_v2.py diff --git a/tests/recipe_v2/build_number/test_1/expected.yaml b/tests/recipe_v2/build_number/test_1/expected.yaml new file mode 100644 index 000000000..f22fd1978 --- /dev/null +++ b/tests/recipe_v2/build_number/test_1/expected.yaml @@ -0,0 +1,10 @@ +# set the build number to something +context: + build: 0 + +package: + name: recipe_1 + version: "0.1.0" + +build: + number: ${{ build }} diff --git a/tests/recipe_v2/build_number/test_1/recipe.yaml b/tests/recipe_v2/build_number/test_1/recipe.yaml new file mode 100644 index 000000000..5ea4da2af --- /dev/null +++ b/tests/recipe_v2/build_number/test_1/recipe.yaml @@ -0,0 +1,10 @@ +# set the build number to something +context: + build: 123 + +package: + name: recipe_1 + version: "0.1.0" + +build: + number: ${{ build }} diff --git a/tests/recipe_v2/build_number/test_2/expected.yaml b/tests/recipe_v2/build_number/test_2/expected.yaml new file mode 100644 index 000000000..3095c27dd --- /dev/null +++ b/tests/recipe_v2/build_number/test_2/expected.yaml @@ -0,0 +1,11 @@ +# set the build number to something +package: + name: recipe_1 + version: "0.1.0" + +# set the build number to something directly in the recipe text +build: + number: 0 + +source: + - url: foo diff --git a/tests/recipe_v2/build_number/test_2/recipe.yaml b/tests/recipe_v2/build_number/test_2/recipe.yaml new file mode 100644 index 000000000..d0906a9a5 --- /dev/null +++ b/tests/recipe_v2/build_number/test_2/recipe.yaml @@ -0,0 +1,11 @@ +# set the build number to something +package: + name: recipe_1 + version: "0.1.0" + +# set the build number to something directly in the recipe text +build: + number: 321 + +source: +- url: foo diff --git a/tests/recipe_v2/version/test_1/expected.yaml b/tests/recipe_v2/version/test_1/expected.yaml new file mode 100644 index 000000000..02ad23f80 --- /dev/null +++ b/tests/recipe_v2/version/test_1/expected.yaml @@ -0,0 +1,12 @@ +context: + name: xtensor + version: "0.25.0" + + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + url: https://github.com/xtensor-stack/xtensor/archive/${{ version }}.tar.gz + sha256: 32d5d9fd23998c57e746c375a544edf544b74f0a18ad6bc3c38cbba968d5e6c7 diff --git a/tests/recipe_v2/version/test_1/recipe.yaml b/tests/recipe_v2/version/test_1/recipe.yaml new file mode 100644 index 000000000..ca0cf055a --- /dev/null +++ b/tests/recipe_v2/version/test_1/recipe.yaml @@ -0,0 +1,12 @@ +context: + name: xtensor + version: "0.23.5" + + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + url: https://github.com/xtensor-stack/xtensor/archive/${{ version }}.tar.gz + sha256: 0811011e448628f0dfa6ebb5e3f76dc7bf6a15ee65ea9c5a277b12ea976d35bc diff --git a/tests/recipe_v2/version/test_2/expected.yaml b/tests/recipe_v2/version/test_2/expected.yaml new file mode 100644 index 000000000..e9a70d0f9 --- /dev/null +++ b/tests/recipe_v2/version/test_2/expected.yaml @@ -0,0 +1,15 @@ +context: + name: xtensor + version: "0.25.0" + + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + # please update the version here. + - if: target_platform == linux-64 + then: + url: https://github.com/xtensor-stack/xtensor/archive/${{ version }}.tar.gz + sha256: 32d5d9fd23998c57e746c375a544edf544b74f0a18ad6bc3c38cbba968d5e6c7 diff --git a/tests/recipe_v2/version/test_2/recipe.yaml b/tests/recipe_v2/version/test_2/recipe.yaml new file mode 100644 index 000000000..6de44d524 --- /dev/null +++ b/tests/recipe_v2/version/test_2/recipe.yaml @@ -0,0 +1,15 @@ +context: + name: xtensor + version: "0.23.5" + + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + # please update the version here. + - if: target_platform == linux-64 + then: + url: https://github.com/xtensor-stack/xtensor/archive/${{ version }}.tar.gz + sha256: 0811011e448628f0dfa6ebb5e3f76dc7bf6a15ee65ea9c5a277b12ea976d35bc diff --git a/tests/recipe_v2/version/test_3/expected.yaml b/tests/recipe_v2/version/test_3/expected.yaml new file mode 100644 index 000000000..d6820f4bf --- /dev/null +++ b/tests/recipe_v2/version/test_3/expected.yaml @@ -0,0 +1,11 @@ +context: + name: pytest-aio + version: 1.9.0 + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + url: https://pypi.io/packages/source/${{ name[0] }}/${{ name }}/${{ name.replace('-', '_') }}-${{ version }}.tar.gz + sha256: aa72e6ca4672b7f5a08ce44e7c6254dca988d3d578bf0c9485a47c3bff393ac1 diff --git a/tests/recipe_v2/version/test_3/recipe.yaml b/tests/recipe_v2/version/test_3/recipe.yaml new file mode 100644 index 000000000..cad30b78f --- /dev/null +++ b/tests/recipe_v2/version/test_3/recipe.yaml @@ -0,0 +1,11 @@ +context: + name: pytest-aio + version: 1.8.1 + +package: + name: ${{ name|lower }} + version: ${{ version }} + +source: + url: https://pypi.io/packages/source/${{ name[0] }}/${{ name }}/${{ name.replace('-', '_') }}-${{ version }}.tar.gz + sha256: 97dcbc1c5ac991705f32bb2cf72f9ba94a8889fd0295d29ed4d7252b3e158684 diff --git a/tests/recipe_v2/version/test_4/expected.yaml b/tests/recipe_v2/version/test_4/expected.yaml new file mode 100644 index 000000000..2cb6a9cd4 --- /dev/null +++ b/tests/recipe_v2/version/test_4/expected.yaml @@ -0,0 +1,14 @@ +context: + version: "1.1-30" + posix: ${{ 'm2' if win else '' }} + native: ${{ 'm2w64' if win else '' }} + +package: + name: r-systemfit + version: ${{ version|replace("-", "_") }} + +source: + url: + - ${{ cran_mirror }}/src/contrib/systemfit_${{ version }}.tar.gz + - ${{ cran_mirror }}/src/contrib/Archive/systemfit/systemfit_${{ version }}.tar.gz + sha256: 5994fbb81f1678325862414f58328cdc2c46d47efa1f23218e9416a4da431ce2 diff --git a/tests/recipe_v2/version/test_4/recipe.yaml b/tests/recipe_v2/version/test_4/recipe.yaml new file mode 100644 index 000000000..fb35a82f5 --- /dev/null +++ b/tests/recipe_v2/version/test_4/recipe.yaml @@ -0,0 +1,14 @@ +context: + version: "1.1-26" + posix: ${{ 'm2' if win else '' }} + native: ${{ 'm2w64' if win else '' }} + +package: + name: r-systemfit + version: ${{ version|replace("-", "_") }} + +source: + url: + - ${{ cran_mirror }}/src/contrib/systemfit_${{ version }}.tar.gz + - ${{ cran_mirror }}/src/contrib/Archive/systemfit/systemfit_${{ version }}.tar.gz + sha256: a99a59787dc5556afe9a1a153f2a3a8047aa7d357aab450101e20ab1f329f758 diff --git a/tests/test_recipe_editing_v2.py b/tests/test_recipe_editing_v2.py new file mode 100644 index 000000000..469d5911d --- /dev/null +++ b/tests/test_recipe_editing_v2.py @@ -0,0 +1,38 @@ +import pytest +from pathlib import Path + +from rattler_build_conda_compat.modify_recipe import update_build_number, update_version + + +@pytest.fixture +def data_dir() -> Path: + return Path(__file__).parent / "recipe_v2" + +def test_build_number_mod(data_dir: Path) -> None: + tests = data_dir / "build_number" + result = update_build_number(tests / "test_1/recipe.yaml", 0) + expected = tests / "test_1/expected.yaml" + assert result == expected.read_text() + + result = update_build_number(tests / "test_2/recipe.yaml", 0) + expected = tests / "test_2/expected.yaml" + assert result == expected.read_text() + + +def test_version_mod(data_dir: Path) -> None: + tests = data_dir / "version" + test_recipes = [tests / "test_1/recipe.yaml", tests / "test_2/recipe.yaml"] + for recipe in test_recipes: + result = update_version(recipe, "0.25.0", None) + expected = recipe.parent / "expected.yaml" + assert result == expected.read_text() + + test_python = tests / "test_3/recipe.yaml" + result = update_version(test_python, "1.9.0", None) + expected = test_python.parent / "expected.yaml" + assert result == expected.read_text() + + test_cran = tests / "test_4/recipe.yaml" + result = update_version(test_cran, "1.1-30", None) + expected = test_cran.parent / "expected.yaml" + assert result == expected.read_text() From ee9497f9a81f6f1336d48bc24c7d25737e9537cc Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 7 Aug 2024 16:49:43 +0200 Subject: [PATCH 04/13] try to fix tests --- conda_forge_tick/update_recipe/v2/jinja/filters.py | 2 +- conda_forge_tick/update_recipe/v2/jinja/jinja.py | 10 +++++----- tests/test_recipe_editing_v2.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conda_forge_tick/update_recipe/v2/jinja/filters.py b/conda_forge_tick/update_recipe/v2/jinja/filters.py index b28e4555d..dcf3af12f 100644 --- a/conda_forge_tick/update_recipe/v2/jinja/filters.py +++ b/conda_forge_tick/update_recipe/v2/jinja/filters.py @@ -1,6 +1,6 @@ from __future__ import annotations -from rattler_build_conda_compat.jinja.utils import _MissingUndefined +from conda_forge_tick.jinja.utils import _MissingUndefined def _version_to_build_string(some_string: str | _MissingUndefined) -> str: diff --git a/conda_forge_tick/update_recipe/v2/jinja/jinja.py b/conda_forge_tick/update_recipe/v2/jinja/jinja.py index 89229bf7f..7034acc8a 100644 --- a/conda_forge_tick/update_recipe/v2/jinja/jinja.py +++ b/conda_forge_tick/update_recipe/v2/jinja/jinja.py @@ -5,8 +5,8 @@ import jinja2 import yaml -from rattler_build_conda_compat.jinja.filters import _bool, _split, _version_to_build_string -from rattler_build_conda_compat.jinja.objects import ( +from conda_forge_tick.jinja.filters import _bool, _split, _version_to_build_string +from conda_forge_tick.jinja.objects import ( _stub_compatible_pin, _stub_is_linux, _stub_is_unix, @@ -15,8 +15,8 @@ _stub_subpackage_pin, _StubEnv, ) -from rattler_build_conda_compat.jinja.utils import _MissingUndefined -from rattler_build_conda_compat.loader import load_yaml +from conda_forge_tick.jinja.utils import _MissingUndefined +from conda_forge_tick.loader import load_yaml class RecipeWithContext(TypedDict, total=False): @@ -97,7 +97,7 @@ def render_recipe_with_context(recipe_content: RecipeWithContext) -> dict[str, A --- ```python >>> from pathlib import Path - >>> from rattler_build_conda_compat.loader import load_yaml + >>> from conda_forge_tick.loader import load_yaml >>> recipe_content = load_yaml((Path().resolve() / "tests" / "data" / "eval_recipe_using_context.yaml").read_text()) >>> evaluated_context = render_recipe_with_context(recipe_content) >>> assert "my_value-${{ not_present_value }}" == evaluated_context["build"]["string"] diff --git a/tests/test_recipe_editing_v2.py b/tests/test_recipe_editing_v2.py index 469d5911d..11a02ec0d 100644 --- a/tests/test_recipe_editing_v2.py +++ b/tests/test_recipe_editing_v2.py @@ -1,7 +1,7 @@ import pytest from pathlib import Path -from rattler_build_conda_compat.modify_recipe import update_build_number, update_version +from conda_forge_tick.recipe_editing_v2 import update_build_number, update_version @pytest.fixture From eed8c2e27fdffb8b554c4e5ec0f0dc584761bef7 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 7 Aug 2024 17:53:46 +0200 Subject: [PATCH 05/13] run pre-commit --- conda_forge_tick/update_recipe/v2/__init__.py | 4 + .../update_recipe/v2/build_number.py | 12 ++- conda_forge_tick/update_recipe/v2/context.py | 5 +- .../update_recipe/v2/jinja/__init__.py | 3 + .../update_recipe/v2/jinja/filters.py | 2 +- .../update_recipe/v2/jinja/jinja.py | 79 ++++++++++--------- conda_forge_tick/update_recipe/v2/source.py | 15 ++-- conda_forge_tick/update_recipe/v2/version.py | 7 +- conda_forge_tick/update_recipe/v2/yaml.py | 10 +-- tests/test_recipe_editing_v2.py | 6 +- 10 files changed, 86 insertions(+), 57 deletions(-) diff --git a/conda_forge_tick/update_recipe/v2/__init__.py b/conda_forge_tick/update_recipe/v2/__init__.py index e69de29bb..f097d437c 100644 --- a/conda_forge_tick/update_recipe/v2/__init__.py +++ b/conda_forge_tick/update_recipe/v2/__init__.py @@ -0,0 +1,4 @@ +from .build_number import update_build_number +from .version import update_version + +__all__ = ["update_build_number", "update_version"] diff --git a/conda_forge_tick/update_recipe/v2/build_number.py b/conda_forge_tick/update_recipe/v2/build_number.py index 8776eca29..11f39a7b6 100644 --- a/conda_forge_tick/update_recipe/v2/build_number.py +++ b/conda_forge_tick/update_recipe/v2/build_number.py @@ -2,7 +2,9 @@ import logging from typing import TYPE_CHECKING, Any, Literal -from conda_forge_tick.update_recipe.v2.yaml import _load_yaml, _dump_yaml_to_str + +from conda_forge_tick.update_recipe.v2.yaml import _dump_yaml_to_str, _load_yaml + if TYPE_CHECKING: from pathlib import Path @@ -11,7 +13,9 @@ HashType = Literal["md5", "sha256"] -def _update_build_number_in_context(recipe: dict[str, Any], new_build_number: int) -> bool: +def _update_build_number_in_context( + recipe: dict[str, Any], new_build_number: int +) -> bool: for key in recipe.get("context", {}): if key.startswith("build_") or key == "build": recipe["context"][key] = new_build_number @@ -19,7 +23,9 @@ def _update_build_number_in_context(recipe: dict[str, Any], new_build_number: in return False -def _update_build_number_in_recipe(recipe: dict[str, Any], new_build_number: int) -> bool: +def _update_build_number_in_recipe( + recipe: dict[str, Any], new_build_number: int +) -> bool: is_modified = False if "build" in recipe and "number" in recipe["build"]: recipe["build"]["number"] = new_build_number diff --git a/conda_forge_tick/update_recipe/v2/context.py b/conda_forge_tick/update_recipe/v2/context.py index cd8bc1e2b..cb86d8066 100644 --- a/conda_forge_tick/update_recipe/v2/context.py +++ b/conda_forge_tick/update_recipe/v2/context.py @@ -1,6 +1,9 @@ import jinja2 -def load_recipe_context(context: dict[str, str], jinja_env: jinja2.Environment) -> dict[str, str]: + +def load_recipe_context( + context: dict[str, str], jinja_env: jinja2.Environment +) -> dict[str, str]: """ Load all string values from the context dictionary as Jinja2 templates. Use linux-64 as default target_platform, build_platform, and mpi. diff --git a/conda_forge_tick/update_recipe/v2/jinja/__init__.py b/conda_forge_tick/update_recipe/v2/jinja/__init__.py index e69de29bb..7d3fad5a0 100644 --- a/conda_forge_tick/update_recipe/v2/jinja/__init__.py +++ b/conda_forge_tick/update_recipe/v2/jinja/__init__.py @@ -0,0 +1,3 @@ +from .jinja import jinja_env + +__all__ = ["jinja_env"] diff --git a/conda_forge_tick/update_recipe/v2/jinja/filters.py b/conda_forge_tick/update_recipe/v2/jinja/filters.py index dcf3af12f..2580adf4e 100644 --- a/conda_forge_tick/update_recipe/v2/jinja/filters.py +++ b/conda_forge_tick/update_recipe/v2/jinja/filters.py @@ -1,6 +1,6 @@ from __future__ import annotations -from conda_forge_tick.jinja.utils import _MissingUndefined +from conda_forge_tick.update_recipe.v2.jinja.utils import _MissingUndefined def _version_to_build_string(some_string: str | _MissingUndefined) -> str: diff --git a/conda_forge_tick/update_recipe/v2/jinja/jinja.py b/conda_forge_tick/update_recipe/v2/jinja/jinja.py index 7034acc8a..b25e1189d 100644 --- a/conda_forge_tick/update_recipe/v2/jinja/jinja.py +++ b/conda_forge_tick/update_recipe/v2/jinja/jinja.py @@ -1,12 +1,16 @@ from __future__ import annotations -from typing import Any, TypedDict +from typing import TypedDict import jinja2 -import yaml +from jinja2.sandbox import SandboxedEnvironment -from conda_forge_tick.jinja.filters import _bool, _split, _version_to_build_string -from conda_forge_tick.jinja.objects import ( +from conda_forge_tick.update_recipe.v2.jinja.filters import ( + _bool, + _split, + _version_to_build_string, +) +from conda_forge_tick.update_recipe.v2.jinja.objects import ( _stub_compatible_pin, _stub_is_linux, _stub_is_unix, @@ -15,20 +19,21 @@ _stub_subpackage_pin, _StubEnv, ) -from conda_forge_tick.jinja.utils import _MissingUndefined -from conda_forge_tick.loader import load_yaml +from conda_forge_tick.update_recipe.v2.jinja.utils import _MissingUndefined + +# from conda_forge_tick.update_recipe.v2.loader import load_yaml class RecipeWithContext(TypedDict, total=False): context: dict[str, str] -def jinja_env() -> jinja2.Environment: +def jinja_env() -> SandboxedEnvironment: """ Create a `rattler-build` specific Jinja2 environment with modified syntax. Target platform, build platform, and mpi are set to linux-64 by default. """ - env = jinja2.sandbox.SandboxedEnvironment( + env = SandboxedEnvironment( variable_start_string="${{", variable_end_string="}}", trim_blocks=True, @@ -71,7 +76,9 @@ def jinja_env() -> jinja2.Environment: return env -def load_recipe_context(context: dict[str, str], jinja_env: jinja2.Environment) -> dict[str, str]: +def load_recipe_context( + context: dict[str, str], jinja_env: jinja2.Environment +) -> dict[str, str]: """ Load all string values from the context dictionary as Jinja2 templates. Use linux-64 as default target_platform, build_platform, and mpi. @@ -87,30 +94,30 @@ def load_recipe_context(context: dict[str, str], jinja_env: jinja2.Environment) return context -def render_recipe_with_context(recipe_content: RecipeWithContext) -> dict[str, Any]: - """ - Render the recipe using known values from context section. - Unknown values are not evaluated and are kept as it is. - Target platform, build platform, and mpi are set to linux-64 by default. - - Examples: - --- - ```python - >>> from pathlib import Path - >>> from conda_forge_tick.loader import load_yaml - >>> recipe_content = load_yaml((Path().resolve() / "tests" / "data" / "eval_recipe_using_context.yaml").read_text()) - >>> evaluated_context = render_recipe_with_context(recipe_content) - >>> assert "my_value-${{ not_present_value }}" == evaluated_context["build"]["string"] - >>> - ``` - """ - env = jinja_env() - context = recipe_content.get("context", {}) - # render out the context section and retrieve dictionary - context_variables = load_recipe_context(context, env) - - # render the rest of the document with the values from the context - # and keep undefined expressions _as is_. - template = env.from_string(yaml.dump(recipe_content)) - rendered_content = template.render(context_variables) - return load_yaml(rendered_content) +# def render_recipe_with_context(recipe_content: RecipeWithContext) -> dict[str, Any]: +# """ +# Render the recipe using known values from context section. +# Unknown values are not evaluated and are kept as it is. +# Target platform, build platform, and mpi are set to linux-64 by default. + +# Examples: +# --- +# ```python +# >>> from pathlib import Path +# >>> from conda_forge_tick.loader import load_yaml +# >>> recipe_content = load_yaml((Path().resolve() / "tests" / "data" / "eval_recipe_using_context.yaml").read_text()) +# >>> evaluated_context = render_recipe_with_context(recipe_content) +# >>> assert "my_value-${{ not_present_value }}" == evaluated_context["build"]["string"] +# >>> +# ``` +# """ +# env = jinja_env() +# context = recipe_content.get("context", {}) +# # render out the context section and retrieve dictionary +# context_variables = load_recipe_context(context, env) + +# # render the rest of the document with the values from the context +# # and keep undefined expressions _as is_. +# template = env.from_string(yaml.dump(recipe_content)) +# rendered_content = template.render(context_variables) +# return load_yaml(rendered_content) diff --git a/conda_forge_tick/update_recipe/v2/source.py b/conda_forge_tick/update_recipe/v2/source.py index 8339bfe89..7e6b5d66a 100644 --- a/conda_forge_tick/update_recipe/v2/source.py +++ b/conda_forge_tick/update_recipe/v2/source.py @@ -1,9 +1,14 @@ import typing -from typing import Union, List, TypedDict, Any, Iterator, NotRequired, Mapping -from conda_forge_tick.update_recipe.v2.conditional_list import visit_conditional_list, ConditionalList +from typing import Any, Iterator, List, Mapping, NotRequired, TypedDict, Union + +from conda_forge_tick.update_recipe.v2.conditional_list import ( + ConditionalList, + visit_conditional_list, +) OptionalUrlList = Union[str, List[str], None] + class Source(TypedDict): url: NotRequired[str | list[str]] sha256: NotRequired[str] @@ -29,8 +34,7 @@ def get_all_sources(recipe: Mapping[Any, Any]) -> Iterator[Source]: # Try getting all url top-level sources if sources is not None: source_list = visit_conditional_list(sources, None) - for source in source_list: - yield source + yield from source_list outputs = recipe.get("outputs", None) if outputs is None: @@ -43,5 +47,4 @@ def get_all_sources(recipe: Mapping[Any, Any]) -> Iterator[Source]: if sources is None: continue source_list = visit_conditional_list(sources, None) - for source in source_list: - yield source + yield from source_list diff --git a/conda_forge_tick/update_recipe/v2/version.py b/conda_forge_tick/update_recipe/v2/version.py index 8eccdacd5..6d89187a3 100644 --- a/conda_forge_tick/update_recipe/v2/version.py +++ b/conda_forge_tick/update_recipe/v2/version.py @@ -6,12 +6,12 @@ import re from typing import TYPE_CHECKING, Literal +import requests + from conda_forge_tick.update_recipe.v2.context import load_recipe_context from conda_forge_tick.update_recipe.v2.jinja import jinja_env from conda_forge_tick.update_recipe.v2.source import Source, get_all_sources -from conda_forge_tick.update_recipe.v2.yaml import _load_yaml, _dump_yaml_to_str - -import requests +from conda_forge_tick.update_recipe.v2.yaml import _dump_yaml_to_str, _load_yaml if TYPE_CHECKING: from pathlib import Path @@ -20,6 +20,7 @@ HashType = Literal["md5", "sha256"] + class CouldNotUpdateVersionError(Exception): NO_CONTEXT = "Could not find context in recipe" NO_VERSION = "Could not find version in recipe context" diff --git a/conda_forge_tick/update_recipe/v2/yaml.py b/conda_forge_tick/update_recipe/v2/yaml.py index 4efd901a7..3a748eec6 100644 --- a/conda_forge_tick/update_recipe/v2/yaml.py +++ b/conda_forge_tick/update_recipe/v2/yaml.py @@ -1,22 +1,22 @@ import io -from ruamel.yaml import YAML -from typing import TYPE_CHECKING +from pathlib import Path -if TYPE_CHECKING: - from pathlib import Path +from ruamel.yaml import YAML yaml = YAML() yaml.preserve_quotes = True yaml.width = 4096 yaml.indent(mapping=2, sequence=4, offset=2) + def _load_yaml(file: Path) -> dict: """Load a YAML file.""" with file.open("r") as f: return yaml.load(f) + def _dump_yaml_to_str(data: dict) -> str: """Dump a dictionary to a YAML string.""" with io.StringIO() as f: yaml.dump(data, f) - return f.getvalue() \ No newline at end of file + return f.getvalue() diff --git a/tests/test_recipe_editing_v2.py b/tests/test_recipe_editing_v2.py index 11a02ec0d..c971e1b29 100644 --- a/tests/test_recipe_editing_v2.py +++ b/tests/test_recipe_editing_v2.py @@ -1,13 +1,15 @@ -import pytest from pathlib import Path -from conda_forge_tick.recipe_editing_v2 import update_build_number, update_version +import pytest + +from conda_forge_tick.update_recipe.v2 import update_build_number, update_version @pytest.fixture def data_dir() -> Path: return Path(__file__).parent / "recipe_v2" + def test_build_number_mod(data_dir: Path) -> None: tests = data_dir / "build_number" result = update_build_number(tests / "test_1/recipe.yaml", 0) From b8186763733fa90a3e78762070b327ebe82bd163 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Wed, 7 Aug 2024 17:56:51 +0200 Subject: [PATCH 06/13] run the actual pre-commit --- .../update_recipe/v2/conditional_list.py | 4 +-- .../update_recipe/v2/jinja/jinja.py | 29 ------------------- 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/conda_forge_tick/update_recipe/v2/conditional_list.py b/conda_forge_tick/update_recipe/v2/conditional_list.py index 68c1205f9..5b0a08cdc 100644 --- a/conda_forge_tick/update_recipe/v2/conditional_list.py +++ b/conda_forge_tick/update_recipe/v2/conditional_list.py @@ -21,7 +21,7 @@ class IfStatement(Generic[T]): def visit_conditional_list( # noqa: C901 value: T | IfStatement[T] | list[T | IfStatement[T]], evaluator: Callable[[Any], bool] | None = None, -) -> Generator[T, None, None]: +) -> Generator[T]: """ A function that yields individual branches of a conditional list. @@ -35,7 +35,7 @@ def visit_conditional_list( # noqa: C901 A generator that yields the individual branches. """ - def yield_from_list(value: list[K] | K) -> Generator[K, None, None]: + def yield_from_list(value: list[K] | K) -> Generator[K]: if isinstance(value, list): yield from value else: diff --git a/conda_forge_tick/update_recipe/v2/jinja/jinja.py b/conda_forge_tick/update_recipe/v2/jinja/jinja.py index b25e1189d..fa3cb89ab 100644 --- a/conda_forge_tick/update_recipe/v2/jinja/jinja.py +++ b/conda_forge_tick/update_recipe/v2/jinja/jinja.py @@ -92,32 +92,3 @@ def load_recipe_context( context[key] = rendered_value return context - - -# def render_recipe_with_context(recipe_content: RecipeWithContext) -> dict[str, Any]: -# """ -# Render the recipe using known values from context section. -# Unknown values are not evaluated and are kept as it is. -# Target platform, build platform, and mpi are set to linux-64 by default. - -# Examples: -# --- -# ```python -# >>> from pathlib import Path -# >>> from conda_forge_tick.loader import load_yaml -# >>> recipe_content = load_yaml((Path().resolve() / "tests" / "data" / "eval_recipe_using_context.yaml").read_text()) -# >>> evaluated_context = render_recipe_with_context(recipe_content) -# >>> assert "my_value-${{ not_present_value }}" == evaluated_context["build"]["string"] -# >>> -# ``` -# """ -# env = jinja_env() -# context = recipe_content.get("context", {}) -# # render out the context section and retrieve dictionary -# context_variables = load_recipe_context(context, env) - -# # render the rest of the document with the values from the context -# # and keep undefined expressions _as is_. -# template = env.from_string(yaml.dump(recipe_content)) -# rendered_content = template.render(context_variables) -# return load_yaml(rendered_content) From d109abdd03b82680cf5d423bc41392a3799f05fa Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 8 Aug 2024 07:33:16 +0200 Subject: [PATCH 07/13] use existing hashing code --- conda_forge_tick/hashing.py | 17 +++++++-- conda_forge_tick/update_recipe/v2/version.py | 40 ++++++++++++-------- conda_forge_tick/update_recipe/version.py | 10 +++-- tests/test_recipe_editing_v2.py | 6 +-- 4 files changed, 48 insertions(+), 25 deletions(-) diff --git a/conda_forge_tick/hashing.py b/conda_forge_tick/hashing.py index dd6649536..a8ab1c9a6 100644 --- a/conda_forge_tick/hashing.py +++ b/conda_forge_tick/hashing.py @@ -2,12 +2,18 @@ import hashlib import math import time -from multiprocessing import Pipe, Process +from multiprocessing import Pipe, Process, connection import requests -def _hash_url(url, hash_type, progress=False, conn=None, timeout=None): +def _hash_url( + url: str, + hash_type: str, + progress: bool = False, + conn: connection.Connection | None = None, + timeout: int | None = None, +) -> str | None: _hash = None try: ha = getattr(hashlib, hash_type)() @@ -68,7 +74,12 @@ def _hash_url(url, hash_type, progress=False, conn=None, timeout=None): @functools.lru_cache(maxsize=1024) -def hash_url(url, timeout=None, progress=False, hash_type="sha256"): +def hash_url( + url: str, + timeout: int | None = None, + progress: bool = False, + hash_type: str = "sha256", +) -> str | None: """Hash a url with a timeout. Parameters diff --git a/conda_forge_tick/update_recipe/v2/version.py b/conda_forge_tick/update_recipe/v2/version.py index 6d89187a3..a4c938983 100644 --- a/conda_forge_tick/update_recipe/v2/version.py +++ b/conda_forge_tick/update_recipe/v2/version.py @@ -1,17 +1,15 @@ from __future__ import annotations import copy -import hashlib import logging import re from typing import TYPE_CHECKING, Literal -import requests - from conda_forge_tick.update_recipe.v2.context import load_recipe_context from conda_forge_tick.update_recipe.v2.jinja import jinja_env from conda_forge_tick.update_recipe.v2.source import Source, get_all_sources from conda_forge_tick.update_recipe.v2.yaml import _dump_yaml_to_str, _load_yaml +from conda_forge_tick.update_recipe.version import _try_url_and_hash_it if TYPE_CHECKING: from pathlib import Path @@ -30,6 +28,13 @@ def __init__(self, message: str = "Could not update version") -> None: super().__init__(self.message) +class HashError(Exception): + def __init__(self, url: str) -> None: + self.url = url + self.message = f"Could not hash {url}" + super().__init__(self.message) + + class Hash: def __init__(self, hash_type: HashType, hash_value: str) -> None: self.hash_type = hash_type @@ -66,15 +71,17 @@ def update_hash(source: Source, url: str, hash_: Hash | None) -> None: source[hash_.hash_type] = hash_.hash_value else: # download and hash the file - hasher = hashlib.sha256() - print(f"Retrieving and hashing {url}") - with requests.get(url, stream=True, timeout=100) as r: - for chunk in r.iter_content(chunk_size=4096): - hasher.update(chunk) - source["sha256"] = hasher.hexdigest() + logger.debug(f"Retrieving and hashing {url}") + new_hash = _try_url_and_hash_it(url, "sha256") + if new_hash is None: + logger.error(f"Could not hash {url}") + raise HashError(url) + source["sha256"] = new_hash -def update_version(file: Path, new_version: str, hash_: Hash | None) -> str: +def update_version( + file: Path, new_version: str, hash_: Hash | None +) -> (str | None, set[str]): """ Update the version in the recipe file. @@ -92,9 +99,10 @@ def update_version(file: Path, new_version: str, hash_: Hash | None) -> str: data = _load_yaml(file) if "context" not in data: - raise CouldNotUpdateVersionError(CouldNotUpdateVersionError.NO_CONTEXT) + # raise CouldNotUpdateVersionError(CouldNotUpdateVersionError.NO_CONTEXT) + return None, {CouldNotUpdateVersionError.NO_CONTEXT} if "version" not in data["context"]: - raise CouldNotUpdateVersionError(CouldNotUpdateVersionError.NO_VERSION) + return None, {CouldNotUpdateVersionError.NO_VERSION} data["context"]["version"] = new_version @@ -119,7 +127,9 @@ def update_version(file: Path, new_version: str, hash_: Hash | None) -> str: template = env.from_string(url) rendered_url = template.render(context_variables) + try: + update_hash(source, rendered_url, hash_) + except HashError: + return None, set(f"Could not retrieve hash for '{rendered_url}'") - update_hash(source, rendered_url, hash_) - - return _dump_yaml_to_str(data) + return _dump_yaml_to_str(data), {} diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index b3d709521..c9855db32 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -5,7 +5,7 @@ import pprint import re import traceback -from typing import Any, MutableMapping +from typing import Any, MutableMapping, Optional, Set import jinja2 import jinja2.sandbox @@ -99,7 +99,7 @@ def _compile_all_selectors(cmeta: Any, src: str): return set(selectors) -def _try_url_and_hash_it(url: str, hash_type: str): +def _try_url_and_hash_it(url: str, hash_type: str) -> Optional[str]: logger.debug("downloading url: %s", url) try: @@ -379,7 +379,9 @@ def _try_to_update_version(cmeta: Any, src: str, hash_type: str): return updated_version, errors -def update_version(raw_meta_yaml, version, hash_type="sha256"): +def update_version( + raw_meta_yaml: str, version: str, hash_type: str = "sha256" +) -> (Optional[str], Set[str]): """Update the version in a recipe. Parameters @@ -395,7 +397,7 @@ def update_version(raw_meta_yaml, version, hash_type="sha256"): ------- updated_meta_yaml : str or None The updated meta.yaml. Will be None if there is an error. - errors : str of str + errors : set of str A set of strings giving any errors found when updating the version. The set will be empty if there were no errors. """ diff --git a/tests/test_recipe_editing_v2.py b/tests/test_recipe_editing_v2.py index c971e1b29..8adacdd92 100644 --- a/tests/test_recipe_editing_v2.py +++ b/tests/test_recipe_editing_v2.py @@ -25,16 +25,16 @@ def test_version_mod(data_dir: Path) -> None: tests = data_dir / "version" test_recipes = [tests / "test_1/recipe.yaml", tests / "test_2/recipe.yaml"] for recipe in test_recipes: - result = update_version(recipe, "0.25.0", None) + result = update_version(recipe, "0.25.0", None)[0] expected = recipe.parent / "expected.yaml" assert result == expected.read_text() test_python = tests / "test_3/recipe.yaml" - result = update_version(test_python, "1.9.0", None) + result = update_version(test_python, "1.9.0", None)[0] expected = test_python.parent / "expected.yaml" assert result == expected.read_text() test_cran = tests / "test_4/recipe.yaml" - result = update_version(test_cran, "1.1-30", None) + result = update_version(test_cran, "1.1-30", None)[0] expected = test_cran.parent / "expected.yaml" assert result == expected.read_text() From b7f61366a355ab41e0728e7cc325ed36596c3d67 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 8 Aug 2024 11:23:27 +0200 Subject: [PATCH 08/13] wire up functions --- conda_forge_tick/migrators/core.py | 25 +++++++++-------- conda_forge_tick/migrators/version.py | 40 ++++++++++++++++----------- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index 48f0c014b..de18203c4 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -5,6 +5,7 @@ import logging import re import typing +from pathlib import Path from typing import Any, List, Sequence, Set import dateutil.parser @@ -14,7 +15,7 @@ from conda_forge_tick.lazy_json_backends import LazyJson from conda_forge_tick.make_graph import make_outputs_lut_from_graph from conda_forge_tick.path_lengths import cyclic_topological_sort -from conda_forge_tick.update_recipe import update_build_number +from conda_forge_tick.update_recipe import update_build_number, v2 from conda_forge_tick.utils import ( frozen_to_json_friendly, get_bot_run_url, @@ -560,7 +561,7 @@ def order( } return cyclic_topological_sort(graph, top_level) - def set_build_number(self, filename: str) -> None: + def set_build_number(self, filename: str | Path) -> None: """Bump the build number of the specified recipe. Parameters @@ -568,17 +569,19 @@ def set_build_number(self, filename: str) -> None: filename : str Path the the meta.yaml """ - with open(filename) as f: - raw = f.read() + filename = Path(filename) + if filename.name == "recipe.yaml": + filename.write_text(v2.update_build_number(filename, self.new_build_number)) + else: + raw = filename.read_text() - new_myaml = update_build_number( - raw, - self.new_build_number, - build_patterns=self.build_patterns, - ) + new_myaml = update_build_number( + raw, + self.new_build_number, + build_patterns=self.build_patterns, + ) - with open(filename, "w") as f: - f.write(new_myaml) + filename.write_text(new_myaml) def new_build_number(self, old_number: int) -> int: """Determine the new build number to use. diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index 9cf8b8186..48ff44c6a 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -5,6 +5,7 @@ import random import typing import warnings +from pathlib import Path from typing import Any, List, Sequence import conda.exceptions @@ -14,9 +15,8 @@ from conda_forge_tick.contexts import FeedstockContext from conda_forge_tick.migrators.core import Migrator from conda_forge_tick.models.pr_info import MigratorName -from conda_forge_tick.os_utils import pushd from conda_forge_tick.update_deps import get_dep_updates_and_hints -from conda_forge_tick.update_recipe import update_version +from conda_forge_tick.update_recipe import update_version, v2 from conda_forge_tick.utils import get_keys_default, sanitize_string if typing.TYPE_CHECKING: @@ -195,23 +195,31 @@ def migrate( ) -> "MigrationUidTypedDict": version = attrs["new_version"] - with open(os.path.join(recipe_dir, "meta.yaml")) as fp: - raw_meta_yaml = fp.read() + recipe_dir = Path(recipe_dir) + meta_yaml = recipe_dir / "meta.yaml" + recipe_yaml = recipe_dir / "recipe.yaml" + if meta_yaml.exists(): + raw_meta_yaml = meta_yaml.read_text() - updated_meta_yaml, errors = update_version( - raw_meta_yaml, - version, - hash_type=hash_type, - ) + updated_meta_yaml, errors = update_version( + raw_meta_yaml, + version, + hash_type=hash_type, + ) - if len(errors) == 0 and updated_meta_yaml is not None: - with pushd(recipe_dir): - with open("meta.yaml", "w") as fp: - fp.write(updated_meta_yaml) - self.set_build_number("meta.yaml") + if len(errors) == 0 and updated_meta_yaml is not None: + meta_yaml.write_text(updated_meta_yaml) + self.set_build_number(meta_yaml) - return super().migrate(recipe_dir, attrs) - else: + return super().migrate(recipe_dir, attrs) + + elif recipe_yaml.exists(): + updated_recipe, errors = v2.update_version(recipe_yaml, version) + if len(errors) == 0 and updated_recipe is not None: + recipe_yaml.write_text(updated_recipe) + self.set_build_number(recipe_yaml) + + if len(errors) != 0: raise VersionMigrationError( _fmt_error_message( errors, From 4b3d827ebc595b885977af09915337bdc40c95e7 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 8 Aug 2024 16:38:44 +0200 Subject: [PATCH 09/13] small fixes --- conda_forge_tick/update_recipe/v2/version.py | 7 +++++-- conda_forge_tick/update_recipe/v2/yaml.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/conda_forge_tick/update_recipe/v2/version.py b/conda_forge_tick/update_recipe/v2/version.py index a4c938983..d30d1e87b 100644 --- a/conda_forge_tick/update_recipe/v2/version.py +++ b/conda_forge_tick/update_recipe/v2/version.py @@ -99,7 +99,6 @@ def update_version( data = _load_yaml(file) if "context" not in data: - # raise CouldNotUpdateVersionError(CouldNotUpdateVersionError.NO_CONTEXT) return None, {CouldNotUpdateVersionError.NO_CONTEXT} if "version" not in data["context"]: return None, {CouldNotUpdateVersionError.NO_VERSION} @@ -113,6 +112,7 @@ def update_version( # for r-recipes we add the default `cran_mirror` variable context_variables["cran_mirror"] = "https://cran.r-project.org" + errors: set[str] = set() for source in get_all_sources(data): # render the whole URL and find the hash if "url" not in source: @@ -130,6 +130,9 @@ def update_version( try: update_hash(source, rendered_url, hash_) except HashError: - return None, set(f"Could not retrieve hash for '{rendered_url}'") + errors.add(f"Could not hash {url}") + + if len(errors) != 0: + return None, errors return _dump_yaml_to_str(data), {} diff --git a/conda_forge_tick/update_recipe/v2/yaml.py b/conda_forge_tick/update_recipe/v2/yaml.py index 3a748eec6..fd5990636 100644 --- a/conda_forge_tick/update_recipe/v2/yaml.py +++ b/conda_forge_tick/update_recipe/v2/yaml.py @@ -5,7 +5,7 @@ yaml = YAML() yaml.preserve_quotes = True -yaml.width = 4096 +yaml.width = 320 yaml.indent(mapping=2, sequence=4, offset=2) From d0681fbc1c63ce264be3795e7129782a4dea2743 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 8 Aug 2024 16:39:00 +0200 Subject: [PATCH 10/13] .. --- conda_forge_tick/update_recipe/v2/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/update_recipe/v2/version.py b/conda_forge_tick/update_recipe/v2/version.py index d30d1e87b..24e46e536 100644 --- a/conda_forge_tick/update_recipe/v2/version.py +++ b/conda_forge_tick/update_recipe/v2/version.py @@ -132,7 +132,7 @@ def update_version( except HashError: errors.add(f"Could not hash {url}") - if len(errors) != 0: + if errors: return None, errors return _dump_yaml_to_str(data), {} From 1f0fba7d556beb06b11f1317d66a6d1aae68b6f1 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 8 Aug 2024 16:48:42 +0200 Subject: [PATCH 11/13] fix return in docstring --- .../update_recipe/v2/build_number.py | 2 +- conda_forge_tick/update_recipe/v2/version.py | 29 ++++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/conda_forge_tick/update_recipe/v2/build_number.py b/conda_forge_tick/update_recipe/v2/build_number.py index 11f39a7b6..d7e07d376 100644 --- a/conda_forge_tick/update_recipe/v2/build_number.py +++ b/conda_forge_tick/update_recipe/v2/build_number.py @@ -51,7 +51,7 @@ def update_build_number(file: Path, new_build_number: int = 0) -> str: Returns: -------- - The updated recipe as a string. + * The updated recipe as a string. """ data = _load_yaml(file) build_number_modified = _update_build_number_in_context(data, new_build_number) diff --git a/conda_forge_tick/update_recipe/v2/version.py b/conda_forge_tick/update_recipe/v2/version.py index 24e46e536..76eb70e64 100644 --- a/conda_forge_tick/update_recipe/v2/version.py +++ b/conda_forge_tick/update_recipe/v2/version.py @@ -2,7 +2,6 @@ import copy import logging -import re from typing import TYPE_CHECKING, Literal from conda_forge_tick.update_recipe.v2.context import load_recipe_context @@ -44,12 +43,6 @@ def __str__(self) -> str: return f"{self.hash_type}: {self.hash_value}" -def _has_jinja_version(url: str) -> bool: - """Check if the URL has a jinja `${{ version }}` in it.""" - pattern = r"\${{\s*version" - return re.search(pattern, url) is not None - - def update_hash(source: Source, url: str, hash_: Hash | None) -> None: """ Update the sha256 hash in the source dictionary. @@ -93,7 +86,8 @@ def update_version( Returns: -------- - The updated recipe as a string. + * The updated recipe string (or None if there was an error). + * A set of errors that occurred during the update. """ data = _load_yaml(file) @@ -103,14 +97,20 @@ def update_version( if "version" not in data["context"]: return None, {CouldNotUpdateVersionError.NO_VERSION} + old_context = copy.deepcopy(data["context"]) data["context"]["version"] = new_version + CRAN_MIRROR = "https://cran.r-project.org" + # set up the jinja context env = jinja_env() context = copy.deepcopy(data.get("context", {})) - context_variables = load_recipe_context(context, env) + old_context_variables = load_recipe_context(old_context, env) + old_context_variables["cran_mirror"] = CRAN_MIRROR + + new_context_variables = load_recipe_context(context, env) # for r-recipes we add the default `cran_mirror` variable - context_variables["cran_mirror"] = "https://cran.r-project.org" + new_context_variables["cran_mirror"] = CRAN_MIRROR errors: set[str] = set() for source in get_all_sources(data): @@ -122,11 +122,14 @@ def update_version( if isinstance(url, list): url = url[0] - if not _has_jinja_version(url): + template = env.from_string(url) + old_rendered_url = template.render(old_context_variables) + rendered_url = template.render(new_context_variables) + + # nothing to do + if old_rendered_url == rendered_url: continue - template = env.from_string(url) - rendered_url = template.render(context_variables) try: update_hash(source, rendered_url, hash_) except HashError: From 08824216a63de269e58cd149b4c0b2f00f5631c9 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 8 Aug 2024 16:50:11 +0200 Subject: [PATCH 12/13] use `none` as default for split --- conda_forge_tick/update_recipe/v2/jinja/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/update_recipe/v2/jinja/filters.py b/conda_forge_tick/update_recipe/v2/jinja/filters.py index 2580adf4e..132771619 100644 --- a/conda_forge_tick/update_recipe/v2/jinja/filters.py +++ b/conda_forge_tick/update_recipe/v2/jinja/filters.py @@ -23,6 +23,6 @@ def _bool(value: str) -> bool: return bool(value) -def _split(s: str, sep: str = " ") -> list[str]: +def _split(s: str, sep: str | None = None) -> list[str]: """Filter that split a string by a separator""" return s.split(sep) From 7fee6de647678dc3af3282079d611761e8a11c4f Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 8 Aug 2024 17:06:22 +0200 Subject: [PATCH 13/13] properly call `super().migrate(...)` --- conda_forge_tick/migrators/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index 48ff44c6a..afd500803 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -211,8 +211,6 @@ def migrate( meta_yaml.write_text(updated_meta_yaml) self.set_build_number(meta_yaml) - return super().migrate(recipe_dir, attrs) - elif recipe_yaml.exists(): updated_recipe, errors = v2.update_version(recipe_yaml, version) if len(errors) == 0 and updated_recipe is not None: @@ -226,6 +224,8 @@ def migrate( version, ) ) + else: + return super().migrate(recipe_dir, attrs) def pr_body(self, feedstock_ctx: FeedstockContext, add_label_text=False) -> str: if feedstock_ctx.feedstock_name in self.effective_graph.nodes: