From eb4fe51d8702417201fc48638384065d85d50941 Mon Sep 17 00:00:00 2001 From: Justin Calamari Date: Thu, 28 Jun 2018 03:48:20 -0400 Subject: [PATCH 1/2] Map feedstock to packages --- libcflib/harvester.py | 9 ++++++--- libcflib/preloader.py | 2 +- libcflib/rest/request_handler.py | 2 +- libcflib/tools.xsh | 9 ++++++--- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/libcflib/harvester.py b/libcflib/harvester.py index bfc1d4d..cf15308 100644 --- a/libcflib/harvester.py +++ b/libcflib/harvester.py @@ -23,23 +23,26 @@ def filter_file(filename): else: return True + # There are some meta.yaml files that were written weirdly and have some strange tag -# information in them. This is not parsable by safe loader, so we add some more +# information in them. This is not parsable by safe loader, so we add some more # fallback loading def yaml_construct_fallback(loader, node): return None + ruamel_yaml.add_constructor( - 'tag:yaml.org,2002:python/object/apply:builtins.getattr', + "tag:yaml.org,2002:python/object/apply:builtins.getattr", yaml_construct_fallback, constructor=ruamel_yaml.SafeConstructor, ) ruamel_yaml.add_constructor( - 'tag:yaml.org,2002:python/object:__builtin__.instancemethod', + "tag:yaml.org,2002:python/object:__builtin__.instancemethod", yaml_construct_fallback, constructor=ruamel_yaml.SafeConstructor, ) + def harvest(io_like): tf = tarfile.open(fileobj=io_like, mode="r:bz2") diff --git a/libcflib/preloader.py b/libcflib/preloader.py index 64bd827..2576aa5 100644 --- a/libcflib/preloader.py +++ b/libcflib/preloader.py @@ -110,7 +110,7 @@ def reap_package(root_path, package, dst_path, src_url, progress_callback=None): def reap(path, known_bad_packages=()): sorted_files = list(diff(path)) - print(f"TOTAL OUTSTANDING ARTIFACTS: {len(sorted_files)}") + print(f"TOTAL OUTSTANDING ARTIFACTS: {len(sorted_files)}") sorted_files = sorted_files[:500] progress = tqdm.tqdm(total=len(sorted_files)) with ThreadPoolExecutor(max_workers=20) as pool: diff --git a/libcflib/rest/request_handler.py b/libcflib/rest/request_handler.py index eac0267..8be9f20 100644 --- a/libcflib/rest/request_handler.py +++ b/libcflib/rest/request_handler.py @@ -57,7 +57,7 @@ def set_default_headers(self): self.set_header("Content-Type", "application/json") self.set_header("Access-Control-Allow-Origin", "*") self.set_header("Access-Control-Allow-Headers", "x-requested-with") - self.set_header('Access-Control-Allow-Methods', 'GET, OPTIONS') + self.set_header("Access-Control-Allow-Methods", "GET, OPTIONS") def write(self, chunk): """Writes the given chunk to the output buffer. This overrides (and almost diff --git a/libcflib/tools.xsh b/libcflib/tools.xsh index 0189413..d891be4 100644 --- a/libcflib/tools.xsh +++ b/libcflib/tools.xsh @@ -81,7 +81,7 @@ def render_meta_yaml(text): return content -def parse_meta_yaml(text): +def parse_meta_yaml(text, arch=None): """Parse the meta.yaml. Parameters @@ -96,9 +96,12 @@ def parse_meta_yaml(text): """ + config = Config() + if arch: + config.host_subdir = arch try: content = render_meta_yaml(text) - return parse(content, Config()) + return parse(content, config) except: return {} @@ -109,4 +112,4 @@ def indir(d): old_d = os.getcwd() ![cd @(d)] yield - ![cd @(old_d)] \ No newline at end of file + ![cd @(old_d)] From b1a455d25367025e89941ae0ec5924f07d0f9738 Mon Sep 17 00:00:00 2001 From: Justin Calamari Date: Thu, 28 Jun 2018 03:49:53 -0400 Subject: [PATCH 2/2] Add feedstockutils file --- libcflib/feedstockutils.xsh | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 libcflib/feedstockutils.xsh diff --git a/libcflib/feedstockutils.xsh b/libcflib/feedstockutils.xsh new file mode 100644 index 0000000..16da12c --- /dev/null +++ b/libcflib/feedstockutils.xsh @@ -0,0 +1,47 @@ +import re + +from libcflib.tools import parse_meta_yaml + +sel_pat = re.compile(".+?\s*(#.*)?\[[^\[\]]+\](?(1)[^\(\)]*)$") +ARCHS = ["linux-32", "linux-64", "linux-arm", "osx-64", "win-32", "win-64"] + + +def has_selectors(meta_yaml): + for line in meta_yaml.splitlines(): + line = line.rstrip() + if line.lstrip().startswith("#"): + continue + if sel_pat.match(line): + return True + return False + + +def get_packages_from_recipe(recipe): + outputs = recipe.get("outputs", recipe["package"]) + packages = set() + for package in outputs: + packages.add(package) + return packages + + +def extract_packages_from_feedstock(feedstock, arch=None): + packages = set() + if not arch and has_selectors(feedstock.meta_yaml): + for a in ARCHS: + packages.update(extract_packages_from_feedstock(feedstock, arch=a)) + elif arch: + return get_packages_from_recipe(parse_meta_yaml(feedstock.meta_yaml, arch=arch)) + else: + return get_packages_from_recipe(parse_meta_yaml(feedstock.meta_yaml)) + + +def get_feedstock_names(): + git init feedstock + with indir(feedstocks): + git remote add origin git@github.com:conda-forge/feedstocks.git + git config core.sparsecheckout true + echo "feedstocks/" >> .git/info/sparse-checkout + git pull --depth=1 origin master + feedstocks = $(ls feedstocks/).splitlines() + rm -rf feedstocks + return feedstocks