From 81870f85616628a88b3677aa071fc685d6639191 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Mon, 20 Apr 2020 13:17:46 -0500 Subject: [PATCH 1/4] Add a simple update --- conda_forge_tick/migrators/version.py | 118 +++++++++++++++++++------- 1 file changed, 88 insertions(+), 30 deletions(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index ae5bbc289..93dd1e6e7 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -445,19 +445,14 @@ def migrate( try: with open(os.path.join(recipe_dir, "meta.yaml"), "r") as fp: - cmeta = CondaMetaYAML(fp.read()) + old_meta_yaml = fp.read() + cmeta = CondaMetaYAML(old_meta_yaml) except Exception as e: attrs["new_version_errors"][version] = ( "We found a problem parsing the recipe: \n\n" + str(e) ) return {} - # cache round-tripped yaml for testing later - s = io.StringIO() - cmeta.dump(s) - s.seek(0) - old_meta_yaml = s.read() - # if is a git url, then we error if _recipe_has_git_url(cmeta): logger.critical("Migrations do not work on `git_url`s!") @@ -492,7 +487,69 @@ def migrate( attrs["new_version_errors"][version] = _fmt_error_message(errors, version) return {} - if len(list(_gen_key_selector(cmeta.meta, "source"))) > 0: + def _try_simple_update(): + s = io.StringIO() + cmeta.dump(s) + s.seek(0) + new_meta_text_with_version = s.read() + try: + new_meta = parse_meta_yaml(new_meta_text_with_version) + except Exception: + msg = "the yaml couldn't be rendered easily." + return {}, msg + source = meta_yaml["source"] + if isinstance(source, collections.abc.Mapping): + sources = [source] + else: + sources = source + + source_section = [] + in_source = False + for line in new_meta_text_with_version.split("\n"): + if line.startswith("source:"): + in_source = True + if in_source and len(line) != 0 and line[0] not in [" ", "\t"]: + in_source = False + break + if in_source: + source_section.append(line) + if '#' in line: + return {}, "there's a selector" + if '{%' in line: + return {}, "there's complicated jinja {%" + source_section = "\n".join(source_section) + + if source_section.count("url:") != len(sources): + return {}, "there are more URLs than expected" + if source_section.count(hash_type + ":") != len(sources): + return {}, "there are more hashes than expected" + changes = {} + for source in sources: + if hash_type not in source: + return {}, "the hash type was not found" + if re.fullmatch(r"^[0-9a-fA-F]$", source[hash_type]) is not None: + return {}, "unexpected hash value" + if "url" not in source: + return {}, "a URL was not found" + new_hash = hash_url(source["url"], timeout=120, hash_type=hash_type) + old_hash = source[hash_type] + if old_hash in changes and changes[old_hash] != new_hash: + return {}, "unexpected duplicate hash" + changes[old_hash] = new_hash + return changes, "" + + changes, msg = _try_simple_update() + + if len(changes) > 0: + new_meta_yaml = old_meta_yaml + for old_hash, new_hash in changes.items(): + new_meta_yaml = new_meta_yaml.replace(old_hash, new_hash) + with indir(recipe_dir): + with open("meta.yaml", "w") as fp: + fp.write(new_meta_yaml) + did_update = True + + else if len(list(_gen_key_selector(cmeta.meta, "source"))) > 0: did_update = True for src_key in _gen_key_selector(cmeta.meta, "source"): if isinstance(cmeta.meta[src_key], collections.abc.MutableSequence): @@ -508,34 +565,35 @@ def migrate( did_update &= _did_update errors |= _errors + if did_update: + # if the yaml did not change, then we did not migrate actually + cmeta.jinja2_vars["version"] = old_version + s = io.StringIO() + cmeta.dump(s) + s.seek(0) + still_the_same = s.read() == old_meta_yaml + cmeta.jinja2_vars["version"] = version # put back version + + if still_the_same and old_version != version: + did_update = False + errors.add( + "recipe did not appear to change even " + "though the bot said it should have" + ) + logger.critical( + "Recipe did not change in version migration " + "but the code indicates an update was done!" + ) + with indir(recipe_dir): + with open("meta.yaml", "w") as fp: + cmeta.dump(fp) + else: did_update = False errors.add("no source sections found in the recipe") - if did_update: - # if the yaml did not change, then we did not migrate actually - cmeta.jinja2_vars["version"] = old_version - s = io.StringIO() - cmeta.dump(s) - s.seek(0) - still_the_same = s.read() == old_meta_yaml - cmeta.jinja2_vars["version"] = version # put back version - - if still_the_same and old_version != version: - did_update = False - errors.add( - "recipe did not appear to change even " - "though the bot said it should have" - ) - logger.critical( - "Recipe did not change in version migration " - "but the code indicates an update was done!" - ) - if did_update: with indir(recipe_dir): - with open("meta.yaml", "w") as fp: - cmeta.dump(fp) self.set_build_number("meta.yaml") return super().migrate(recipe_dir, attrs) From 34b13eeaf1aaf2109c9dd73c57f9f716271be1e2 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Mon, 20 Apr 2020 13:22:35 -0500 Subject: [PATCH 2/4] else if -> elif --- conda_forge_tick/migrators/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index 93dd1e6e7..b33d58cbd 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -549,7 +549,7 @@ def _try_simple_update(): fp.write(new_meta_yaml) did_update = True - else if len(list(_gen_key_selector(cmeta.meta, "source"))) > 0: + elif len(list(_gen_key_selector(cmeta.meta, "source"))) > 0: did_update = True for src_key in _gen_key_selector(cmeta.meta, "source"): if isinstance(cmeta.meta[src_key], collections.abc.MutableSequence): From b8d0d8376f975430bb49c6eddcfa740f1649c47a Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Mon, 20 Apr 2020 13:26:41 -0500 Subject: [PATCH 3/4] round-tripped might change --- conda_forge_tick/migrators/version.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index b33d58cbd..f96e257a9 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -445,14 +445,20 @@ def migrate( try: with open(os.path.join(recipe_dir, "meta.yaml"), "r") as fp: - old_meta_yaml = fp.read() - cmeta = CondaMetaYAML(old_meta_yaml) + orig_meta_yaml = fp.read() + cmeta = CondaMetaYAML(orig_meta_yaml) except Exception as e: attrs["new_version_errors"][version] = ( "We found a problem parsing the recipe: \n\n" + str(e) ) return {} + # cache round-tripped yaml for testing later + s = io.StringIO() + cmeta.dump(s) + s.seek(0) + old_meta_yaml = s.read() + # if is a git url, then we error if _recipe_has_git_url(cmeta): logger.critical("Migrations do not work on `git_url`s!") @@ -541,7 +547,7 @@ def _try_simple_update(): changes, msg = _try_simple_update() if len(changes) > 0: - new_meta_yaml = old_meta_yaml + new_meta_yaml = orig_meta_yaml for old_hash, new_hash in changes.items(): new_meta_yaml = new_meta_yaml.replace(old_hash, new_hash) with indir(recipe_dir): From 9559005432133f3b408454d42c2be8f07ae09b26 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Mon, 20 Apr 2020 13:29:27 -0500 Subject: [PATCH 4/4] check did_update again --- conda_forge_tick/migrators/version.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_forge_tick/migrators/version.py b/conda_forge_tick/migrators/version.py index f96e257a9..13f487240 100644 --- a/conda_forge_tick/migrators/version.py +++ b/conda_forge_tick/migrators/version.py @@ -590,6 +590,8 @@ def _try_simple_update(): "Recipe did not change in version migration " "but the code indicates an update was done!" ) + + if did_update: with indir(recipe_dir): with open("meta.yaml", "w") as fp: cmeta.dump(fp)