diff --git a/conda_forge_tick/feedstock_parser.py b/conda_forge_tick/feedstock_parser.py index 2ff0662e3..8edaa3a51 100644 --- a/conda_forge_tick/feedstock_parser.py +++ b/conda_forge_tick/feedstock_parser.py @@ -150,7 +150,7 @@ def _extract_requirements(meta_yaml, outputs_to_keep=None): requirements_dict[section].update( list(as_iterable(req.get(section, []) or [])), ) - test: "TestTypedDict" = block.get("test", {}) + test: "TestTypedDict" = {} if block.get("test") is None else block.get("test") requirements_dict["test"].update(test.get("requirements", []) or []) requirements_dict["test"].update(test.get("requires", []) or []) run_exports = (block.get("build", {}) or {}).get("run_exports", {}) @@ -350,6 +350,11 @@ def populate_feedstock_attributes( parse_meta_yaml(meta_yaml, platform=plat, arch=arch) for plat, arch in plat_archs ] + elif isinstance(recipe_yaml, str): + variant_yamls = [ + parse_recipe_yaml(recipe_yaml, platform=plat, arch=arch) + for plat, arch in plat_archs + ] except Exception as e: import traceback @@ -378,11 +383,11 @@ def populate_feedstock_attributes( if k.endswith("_meta_yaml") or k.endswith("_requirements"): sub_graph.pop(k) - for k, v in zip(plat_archs, variant_yamls): - plat_arch_name = "_".join(k) - sub_graph[f"{plat_arch_name}_meta_yaml"] = v + for plat_arch, variant_yaml in zip(plat_archs, variant_yamls): + plat_arch_name = "_".join(plat_arch) + sub_graph[f"{plat_arch_name}_meta_yaml"] = variant_yaml _, sub_graph[f"{plat_arch_name}_requirements"], _ = _extract_requirements( - v, + variant_yaml, outputs_to_keep=BOOTSTRAP_MAPPINGS.get(name, None), ) diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index 48f0c014b..3a1be1967 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -5,6 +5,7 @@ import logging import re import typing +from pathlib import Path from typing import Any, List, Sequence, Set import dateutil.parser @@ -14,7 +15,8 @@ from conda_forge_tick.lazy_json_backends import LazyJson from conda_forge_tick.make_graph import make_outputs_lut_from_graph from conda_forge_tick.path_lengths import cyclic_topological_sort -from conda_forge_tick.update_recipe import update_build_number +from conda_forge_tick.update_recipe import update_build_number_meta_yaml +from conda_forge_tick.update_recipe.build_number import update_build_number_recipe_yaml from conda_forge_tick.utils import ( frozen_to_json_friendly, get_bot_run_url, @@ -439,7 +441,7 @@ def run_post_piggyback_migrations( def migrate( self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any ) -> "MigrationUidTypedDict": - """Perform the migration, updating the ``meta.yaml`` + """Perform the migration, updating the recipe Parameters ---------- @@ -560,25 +562,34 @@ def order( } return cyclic_topological_sort(graph, top_level) - def set_build_number(self, filename: str) -> None: + def set_build_number(self, filename: str | Path) -> None: """Bump the build number of the specified recipe. Parameters ---------- - filename : str - Path the the meta.yaml + filename : str | Path + Path the the recipe file """ - with open(filename) as f: - raw = f.read() - - new_myaml = update_build_number( - raw, - self.new_build_number, - build_patterns=self.build_patterns, - ) + filename = Path(filename) + raw = filename.read_text() + + if filename.name == "meta.yaml": + new_yaml = update_build_number_meta_yaml( + raw, + self.new_build_number, + build_patterns=self.build_patterns, + ) + elif filename.name == "recipe.yaml": + new_yaml = update_build_number_recipe_yaml( + raw, + self.new_build_number, + ) + else: + raise ValueError( + f"`{filename=}` needs to be a `meta.yaml` or `recipe.yaml`." + ) - with open(filename, "w") as f: - f.write(new_myaml) + filename.write_text(new_yaml) def new_build_number(self, old_number: int) -> int: """Determine the new build number to use. diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index 9cf8b8186..2ff27c928 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -5,6 +5,7 @@ import random import typing import warnings +from pathlib import Path from typing import Any, List, Sequence import conda.exceptions @@ -14,9 +15,11 @@ from conda_forge_tick.contexts import FeedstockContext from conda_forge_tick.migrators.core import Migrator from conda_forge_tick.models.pr_info import MigratorName -from conda_forge_tick.os_utils import pushd from conda_forge_tick.update_deps import get_dep_updates_and_hints -from conda_forge_tick.update_recipe import update_version +from conda_forge_tick.update_recipe import ( + update_recipe_yaml_version, + update_version, +) from conda_forge_tick.utils import get_keys_default, sanitize_string if typing.TYPE_CHECKING: @@ -195,21 +198,29 @@ def migrate( ) -> "MigrationUidTypedDict": version = attrs["new_version"] - with open(os.path.join(recipe_dir, "meta.yaml")) as fp: - raw_meta_yaml = fp.read() + meta_yaml_path = Path(recipe_dir, "meta.yaml") + recipe_yaml_path = Path(recipe_dir, "recipe.yaml") - updated_meta_yaml, errors = update_version( - raw_meta_yaml, - version, - hash_type=hash_type, - ) - - if len(errors) == 0 and updated_meta_yaml is not None: - with pushd(recipe_dir): - with open("meta.yaml", "w") as fp: - fp.write(updated_meta_yaml) - self.set_build_number("meta.yaml") + if meta_yaml_path.exists(): + output_path = meta_yaml_path + raw_meta_yaml = meta_yaml_path.read_text() + updated_recipe, errors = update_version( + raw_meta_yaml, + version, + hash_type=hash_type, + ) + elif recipe_yaml_path.exists(): + output_path = recipe_yaml_path + raw_recipe_yaml = recipe_yaml_path.read_text() + updated_recipe, errors = update_recipe_yaml_version( + raw_recipe_yaml, + version, + hash_type=hash_type, + ) + if len(errors) == 0 and updated_recipe is not None: + output_path.write_text(updated_recipe) + self.set_build_number(output_path) return super().migrate(recipe_dir, attrs) else: raise VersionMigrationError( diff --git a/conda_forge_tick/update_recipe/__init__.py b/conda_forge_tick/update_recipe/__init__.py index e2deb50c1..68889f3f5 100644 --- a/conda_forge_tick/update_recipe/__init__.py +++ b/conda_forge_tick/update_recipe/__init__.py @@ -1,2 +1,12 @@ -from .build_number import DEFAULT_BUILD_PATTERNS, update_build_number # noqa -from .version import update_version # noqa +from .build_number import ( + DEFAULT_BUILD_PATTERNS as DEFAULT_BUILD_PATTERNS, +) +from .build_number import ( + update_build_number_meta_yaml as update_build_number_meta_yaml, +) +from .build_number import ( + update_build_number_recipe_yaml as update_build_number_recipe_yaml, +) + +# noqa +from .version import update_recipe_yaml_version, update_version # noqa diff --git a/conda_forge_tick/update_recipe/build_number.py b/conda_forge_tick/update_recipe/build_number.py index 09d9bc807..79a0e079e 100644 --- a/conda_forge_tick/update_recipe/build_number.py +++ b/conda_forge_tick/update_recipe/build_number.py @@ -1,4 +1,7 @@ import re +from typing import Callable + +import yaml DEFAULT_BUILD_PATTERNS = ( (re.compile(r"(\s*?)number:\s*([0-9]+)"), "number: {}"), @@ -13,7 +16,11 @@ ) -def update_build_number(raw_meta_yaml, new_build_number, build_patterns=None): +def update_build_number_meta_yaml( + raw_meta_yaml: str, + new_build_number: Callable[[str], str] | str, + build_patterns=None, +): """Update the build number for a recipe. Parameters @@ -51,3 +58,28 @@ def update_build_number(raw_meta_yaml, new_build_number, build_patterns=None): raw_meta_yaml = "\n".join(lines) + "\n" return raw_meta_yaml + + +def update_build_number_recipe_yaml( + raw_recipe_yaml: str, new_build_number: Callable[[str], str] | str +): + def replace_build_number(recipe, first_key, second_key): + if first := recipe.get(first_key): + if second_key in first and isinstance(first[second_key], int): + if callable(new_build_number): + first[second_key] = new_build_number(first[second_key]) + else: + first[second_key] = new_build_number + + recipe = yaml.safe_load(raw_recipe_yaml) + + cases = [ + ("build", "number"), + ("context", "build_number"), + ("context", "build"), + ] + + for case in cases: + replace_build_number(recipe, *case) + + return yaml.dump(recipe, sort_keys=False) diff --git a/conda_forge_tick/update_recipe/version.py b/conda_forge_tick/update_recipe/version.py index b3d709521..c5652d718 100644 --- a/conda_forge_tick/update_recipe/version.py +++ b/conda_forge_tick/update_recipe/version.py @@ -379,7 +379,9 @@ def _try_to_update_version(cmeta: Any, src: str, hash_type: str): return updated_version, errors -def update_version(raw_meta_yaml, version, hash_type="sha256"): +def update_version( + raw_meta_yaml: str, version: str, hash_type: str = "sha256" +) -> tuple[str | None, set[str]]: """Update the version in a recipe. Parameters @@ -395,7 +397,7 @@ def update_version(raw_meta_yaml, version, hash_type="sha256"): ------- updated_meta_yaml : str or None The updated meta.yaml. Will be None if there is an error. - errors : str of str + errors : set of str A set of strings giving any errors found when updating the version. The set will be empty if there were no errors. """ @@ -527,3 +529,28 @@ def update_version(raw_meta_yaml, version, hash_type="sha256"): else: logger.critical("Recipe did not change in version migration!") return None, errors + + +def update_recipe_yaml_version( + raw_recipe_yaml: str, version: str, hash_type: str = "sha256" +) -> tuple[str | None, set[str]]: + """Update the version in a recipe. + + Parameters + ---------- + raw_recipe_yaml : str + The recipe meta.yaml as a string. + version : str + The version of the recipe. + hash_type : str, optional + The kind of hash used on the source. Default is sha256. + + Returns + ------- + updated_recipe_yaml : str or None + The updated meta.yaml. Will be None if there is an error. + errors : set of str + A set of strings giving any errors found when updating the + version. The set will be empty if there were no errors. + """ + raise NotImplementedError() diff --git a/tests/test_build_number.py b/tests/test_build_number.py index f6109a688..ee940df48 100644 --- a/tests/test_build_number.py +++ b/tests/test_build_number.py @@ -1,6 +1,9 @@ import pytest -from conda_forge_tick.update_recipe import update_build_number +from conda_forge_tick.update_recipe import ( + update_build_number_meta_yaml, + update_build_number_recipe_yaml, +) @pytest.mark.parametrize( @@ -12,8 +15,8 @@ ("{% set build = 2 %}", "{% set build = 0 %}\n"), ], ) -def test_update_build_number(meta_yaml, new_meta_yaml): - out_meta_yaml = update_build_number(meta_yaml, 0) +def test_update_build_number_meta_yaml(meta_yaml, new_meta_yaml): + out_meta_yaml = update_build_number_meta_yaml(meta_yaml, 0) assert out_meta_yaml == new_meta_yaml @@ -26,6 +29,71 @@ def test_update_build_number(meta_yaml, new_meta_yaml): ("{% set build = 2 %}", "{% set build = 3 %}\n"), ], ) -def test_update_build_number_function(meta_yaml, new_meta_yaml): - out_meta_yaml = update_build_number(meta_yaml, lambda x: x + 1) +def test_update_build_number_meta_yaml_function(meta_yaml, new_meta_yaml): + out_meta_yaml = update_build_number_meta_yaml(meta_yaml, lambda x: x + 1) assert out_meta_yaml == new_meta_yaml + + +RECIPE_YAML_IN_CONTEXT_1 = """\ +context: + build_number: 100 +build: + number: ${{ build_number }} +""" + +RECIPE_YAML_EXP_CONTEXT_1 = """\ +context: + build_number: 101 +build: + number: ${{ build_number }} +""" + +RECIPE_YAML_IN_CONTEXT_2 = """\ +context: + build: 100 +build: + number: ${{ build }} +""" + +RECIPE_YAML_EXP_CONTEXT_2 = """\ +context: + build: 101 +build: + number: ${{ build }} +""" + +RECIPE_YAML_IN_LITERAL = """\ +build: + number: 100 +""" + +RECIPE_YAML_EXP_LITERAL = """\ +build: + number: 101 +""" + + +@pytest.mark.parametrize( + "recipe_yaml,expected_recipe_yaml", + [ + (RECIPE_YAML_IN_CONTEXT_1, RECIPE_YAML_EXP_CONTEXT_1), + (RECIPE_YAML_IN_CONTEXT_2, RECIPE_YAML_EXP_CONTEXT_2), + (RECIPE_YAML_IN_LITERAL, RECIPE_YAML_EXP_LITERAL), + ], +) +def test_update_build_number_recipe_yaml(recipe_yaml, expected_recipe_yaml): + out_recipe_yaml = update_build_number_recipe_yaml(recipe_yaml, 101) + assert out_recipe_yaml == expected_recipe_yaml + + +@pytest.mark.parametrize( + "recipe_yaml,expected_recipe_yaml", + [ + (RECIPE_YAML_IN_CONTEXT_1, RECIPE_YAML_EXP_CONTEXT_1), + (RECIPE_YAML_IN_CONTEXT_2, RECIPE_YAML_EXP_CONTEXT_2), + (RECIPE_YAML_IN_LITERAL, RECIPE_YAML_EXP_LITERAL), + ], +) +def test_update_build_number_recipe_yaml_function(recipe_yaml, expected_recipe_yaml): + out_recipe_yaml = update_build_number_recipe_yaml(recipe_yaml, lambda x: x + 1) + assert out_recipe_yaml == expected_recipe_yaml diff --git a/tests/test_migrators.py b/tests/test_migrators.py index eb0da28e4..f8c4a4cfc 100644 --- a/tests/test_migrators.py +++ b/tests/test_migrators.py @@ -3,6 +3,8 @@ import subprocess from pathlib import Path +import yaml + from conda_forge_tick.contexts import FeedstockContext from conda_forge_tick.feedstock_parser import populate_feedstock_attributes from conda_forge_tick.migrators import ( @@ -551,6 +553,94 @@ def run_test_migration( return pmy +def run_test_migration_recipe_yaml( + migrator: Migrator, + in_yaml: str, + output: str, + kwargs: dict, + prb: str, + mr_out: dict, + tmp_path: Path, + should_filter: bool = False, + make_body: bool = False, +): + if mr_out: + mr_out.update(bot_rerun=False) + + tmp_path.joinpath("recipe.yaml").write_text(in_yaml) + + # read the conda-forge.yml + cf_yml_path = tmp_path.parent / "conda-forge.yml" + cf_yml = cf_yml_path.read_text() if cf_yml_path.exists() else "{}" + + name = yaml.safe_load(in_yaml)["package"]["name"] + pmy = populate_feedstock_attributes( + name, sub_graph={}, recipe_yaml=in_yaml, conda_forge_yaml=cf_yml + ) + + # these are here for legacy migrators + pmy["version"] = pmy["meta_yaml"]["package"]["version"] + pmy["req"] = set() + for k in ["build", "host", "run"]: + req = pmy["meta_yaml"].get("requirements", {}) or {} + _set = req.get(k) or set() + pmy["req"] |= set(_set) + pmy.update(kwargs) + + try: + if "new_version" in kwargs: + pmy["version_pr_info"] = {"new_version": kwargs["new_version"]} + assert migrator.filter(pmy) == should_filter + finally: + pmy.pop("version_pr_info", None) + if should_filter: + return pmy + + migrator.run_pre_piggyback_migrations( + str(tmp_path), + pmy, + hash_type=pmy.get("hash_type", "sha256"), + ) + mr = migrator.migrate(str(tmp_path), pmy, hash_type=pmy.get("hash_type", "sha256")) + migrator.run_post_piggyback_migrations( + str(tmp_path), + pmy, + hash_type=pmy.get("hash_type", "sha256"), + ) + + if make_body: + fctx = FeedstockContext( + feedstock_name=name, + attrs=pmy, + ) + fctx.feedstock_dir = tmp_path.name + migrator.effective_graph.add_node(name) + migrator.effective_graph.nodes[name]["payload"] = MockLazyJson({}) + migrator.pr_body(fctx) + + assert mr_out == mr + if not mr: + return pmy + + pmy["pr_info"] = {} + pmy["pr_info"].update(PRed=[frozen_to_json_friendly(mr)]) + actual_output = tmp_path.joinpath("recipe.yaml").read_text() + assert actual_output == output + # TODO: fix subgraph here (need this to be xsh file) + if isinstance(migrator, Version): + pass + else: + assert prb in migrator.pr_body(None) + try: + if "new_version" in kwargs: + pmy["version_pr_info"] = {"new_version": kwargs["new_version"]} + assert migrator.filter(pmy) is True + finally: + pmy.pop("version_pr_info", None) + + return pmy + + def run_minimigrator( migrator: MiniMigrator, inp: str, diff --git a/tests/test_recipe_yaml/ipywidgets.yaml b/tests/test_recipe_yaml/ipywidgets.yaml index c7d41ce04..f675865d9 100644 --- a/tests/test_recipe_yaml/ipywidgets.yaml +++ b/tests/test_recipe_yaml/ipywidgets.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json context: version: "8.1.2" diff --git a/tests/test_recipe_yaml/mplb.yaml b/tests/test_recipe_yaml/mplb.yaml index 0f3be3764..94afc770f 100644 --- a/tests/test_recipe_yaml/mplb.yaml +++ b/tests/test_recipe_yaml/mplb.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json schema_version: 1 context: diff --git a/tests/test_recipe_yaml/version_cdiff.yaml b/tests/test_recipe_yaml/version_cdiff.yaml new file mode 100644 index 000000000..d149b1520 --- /dev/null +++ b/tests/test_recipe_yaml/version_cdiff.yaml @@ -0,0 +1,41 @@ +schema_version: 1 + +context: + org: GoogleContainerTools + name: container-diff + version: 0.14.0 + src_dir: "\"[\"src/github.com\", org, name]|join(\"/\")\"" + +package: + name: container-diff + version: ${{ version }} + +source: + - url: https://github.com/${{ org }}/${{ name }}/archive/v${{ version }}.tar.gz + sha256: 5dbafdc38524dad60286da2d7a7d303285de2e08e070ce3dcc1488dbfecd116b + target_directory: ${{ src_dir }} + - url: https://storage.googleapis.com/container-diff/v${{ version }}/container-diff-windows-amd64.exe + sha256: a55b75bb9b6894e8562e66a11f4cb12e6ea49e1774a136304c96312e966c2e66 # [win] + target_directory: bin + +build: + number: 100 + +requirements: + build: + - if: unix + then: ${{ compiler("go") }} + +tests: + - script: + - container-diff + +about: + license: Apache-2.0 + license_file: ${{ src_dir }}/LICENSE + summary: Diff your Docker containers + homepage: https://github.com/GoogleContainerTools/container-diff + +extra: + recipe-maintainers: + - jakirkham diff --git a/tests/test_recipe_yaml/version_cdiff_correct.yaml b/tests/test_recipe_yaml/version_cdiff_correct.yaml new file mode 100644 index 000000000..e1e36f5a9 --- /dev/null +++ b/tests/test_recipe_yaml/version_cdiff_correct.yaml @@ -0,0 +1,41 @@ +schema_version: 1 + +context: + org: GoogleContainerTools + name: container-diff + version: 0.15.0 + src_dir: "\"[\"src/github.com\", org, name]|join(\"/\")\"" + +package: + name: container-diff + version: ${{ version }} + +source: + - url: https://github.com/${{ org }}/${{ name }}/archive/v${{ version }}.tar.gz + sha256: 4bdd73a81b6f7a988cf270236471016525d0541f5fe04286043f3db28e4b250c + target_directory: ${{ src_dir }} + - url: https://storage.googleapis.com/container-diff/v${{ version }}/container-diff-windows-amd64.exe + sha256: 497123dfd22051c8facb4bfca67dbd0e3ed9a08ead8217eb2f5e534d70810822 # [win] + target_directory: bin + +build: + number: 0 + +requirements: + build: + - if: unix + then: ${{ compiler("go") }} + +tests: + - script: + - container-diff + +about: + license: Apache-2.0 + license_file: ${{ src_dir }}/LICENSE + summary: Diff your Docker containers + homepage: https://github.com/GoogleContainerTools/container-diff + +extra: + recipe-maintainers: + - jakirkham diff --git a/tests/test_version_migrator.py b/tests/test_version_migrator.py index c1351cb62..2a2c7dc33 100644 --- a/tests/test_version_migrator.py +++ b/tests/test_version_migrator.py @@ -5,7 +5,7 @@ import pytest from flaky import flaky -from test_migrators import run_test_migration +from test_migrators import run_test_migration, run_test_migration_recipe_yaml from conda_forge_tick.migrators import Version from conda_forge_tick.migrators.version import VersionMigrationError @@ -13,6 +13,7 @@ VERSION = Version(set()) YAML_PATH = os.path.join(os.path.dirname(__file__), "test_yaml") +RECIPE_YAML_PATH = Path(__file__).parent.joinpath("test_recipe_yaml") VARIANT_SOURCES_NOT_IMPLEMENTED = ( "Sources that depend on conda build config variants are not supported yet." @@ -174,6 +175,34 @@ def test_version_cupy(tmpdir, caplog): ) +def test_version_cdiff_recipe_yaml(tmp_path, caplog): + case = "cdiff" + new_ver = "8.5.0" + caplog.set_level( + logging.DEBUG, + logger="conda_forge_tick.migrators.version", + ) + + in_yaml = RECIPE_YAML_PATH.joinpath(f"version_{case}.yaml").read_text() + out_yaml = RECIPE_YAML_PATH.joinpath(f"version_{case}_correct.yaml").read_text() + + kwargs = {"new_version": new_ver} + + run_test_migration_recipe_yaml( + migrator=VERSION, + in_yaml=in_yaml, + output=out_yaml, + kwargs=kwargs, + prb="Dependencies have been updated if changed", + mr_out={ + "migrator_name": Version.name, + "migrator_version": Version.migrator_version, + "version": new_ver, + }, + tmp_path=tmp_path, + ) + + def test_version_rand_frac(tmpdir, caplog): case = "aws_sdk_cpp" new_ver = "1.11.132"