From 0ad3aec12f01f62fb07b1e7be329af2c8f89fd48 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Thu, 27 Nov 2025 15:13:32 +0100 Subject: [PATCH 1/3] Create fake-migrator nodes to debug inter-migration limitations. --- conda_forge_tick/make_migrators.py | 11 +- conda_forge_tick/status_report.py | 185 ++++++++++++++++++++++++++++- 2 files changed, 193 insertions(+), 3 deletions(-) diff --git a/conda_forge_tick/make_migrators.py b/conda_forge_tick/make_migrators.py index 2fdf7913d..e9b62f226 100644 --- a/conda_forge_tick/make_migrators.py +++ b/conda_forge_tick/make_migrators.py @@ -1004,6 +1004,11 @@ def load_migrators(skip_paused: bool = True) -> MutableSequence[Migrator]: pinning_migrators = [] longterm_migrators = [] all_names = get_all_keys_for_hashmap("migrators") + # Only load python314 and python314t migrators - filter BEFORE submitting to pool + allowed_migrators = {"python314", "python314t"} + all_names = [name for name in all_names if name in allowed_migrators] + print(f"Loading only: {all_names}", flush=True) + with executor("process", 2) as pool: futs = [pool.submit(_load, name) for name in all_names] @@ -1027,11 +1032,13 @@ def load_migrators(skip_paused: bool = True) -> MutableSequence[Migrator]: else: migrators.append(migrator) - version_migrator = _make_version_migrator(load_existing_graph()) + # Commented out - version migrator is slow + # version_migrator = _make_version_migrator(load_existing_graph()) RNG.shuffle(pinning_migrators) RNG.shuffle(longterm_migrators) - migrators = [version_migrator] + migrators + pinning_migrators + longterm_migrators + # migrators = [version_migrator] + migrators + pinning_migrators + longterm_migrators + migrators = migrators + pinning_migrators + longterm_migrators return migrators diff --git a/conda_forge_tick/status_report.py b/conda_forge_tick/status_report.py index 063ab229e..3ec454c27 100644 --- a/conda_forge_tick/status_report.py +++ b/conda_forge_tick/status_report.py @@ -24,6 +24,7 @@ ArchRebuild, GraphMigrator, MatplotlibBase, + MigrationYaml, MigrationYamlCreator, Migrator, OSXArm, @@ -52,6 +53,21 @@ "unstable", ] +# Cache for migrators by name, loaded on demand +_migrators_by_name_cache: Dict[str, Migrator] | None = None + + +def _get_migrators_by_name() -> Dict[str, Migrator]: + """Get mapping of migrator report_name to migrator instance. + + Uses a module-level cache to avoid reloading migrators multiple times. + """ + global _migrators_by_name_cache + if _migrators_by_name_cache is None: + migrators = load_migrators(skip_paused=False) + _migrators_by_name_cache = {m.report_name: m for m in migrators} + return _migrators_by_name_cache + def _sorted_set_json(obj: Any) -> Any: """If obj is a set, return sorted(obj). Else, raise TypeError. @@ -164,6 +180,40 @@ def write_version_migrator_status(migrator, mctx): ) +def _get_waiting_migrators(migrator: Migrator, attrs: dict) -> list[str]: + """Get list of migrators that this package is waiting for. + + Returns empty list if not waiting for any migrators. + """ + if not isinstance(migrator, MigrationYaml): + return [] + + migrator_payload = migrator.loaded_yaml.get("__migrator", {}) + wait_for_migrators = migrator_payload.get("wait_for_migrators", []) + + if not wait_for_migrators: + return [] + + # Check if we're actually waiting (i.e., migrators not all closed) + found_migrators = set() + for migration in attrs.get("pr_info", {}).get("PRed", []): + name = migration.get("data", {}).get("name", "") + if not name or name not in wait_for_migrators: + continue + found_migrators.add(name) + state = migration.get("PR", {}).get("state", "") + if state != "closed": + # Still waiting for this one + return list(wait_for_migrators) + + # Check if any migrators are missing + missing_migrators = set(wait_for_migrators) - found_migrators + if missing_migrators: + return list(wait_for_migrators) + + return [] + + def graph_migrator_status( migrator: Migrator, gx: nx.DiGraph, @@ -200,6 +250,7 @@ def graph_migrator_status( for node, node_attrs in gx2.nodes.items(): attrs = node_attrs["payload"] + # remove archived from status if attrs.get("archived", False): continue @@ -344,20 +395,133 @@ def graph_migrator_status( ) node_metadata["pr_status"] = pr_json["PR"].get("mergeable_state", "") + # Collect waiting migrators info for creating fake nodes + waiting_migrators_map: Dict[str, list[str]] = {} + for node, node_attrs in gx2.nodes.items(): + attrs = node_attrs["payload"] + + # remove archived from status + if attrs.get("archived", False): + continue + + # Check if waiting for other migrators + waiting_migrators = _get_waiting_migrators(migrator, attrs) + if waiting_migrators: + waiting_migrators_map[node] = waiting_migrators + + # Add fake migrator nodes and edges after processing regular nodes + # Create one fake node per waiting migrator PR: migrator_{migrator_name}_{node}_{pr_number} + # This represents the PR for the waiting migrator on the node itself (not its predecessors) + fake_migrator_nodes: Dict[str, Dict] = {} + + for node, migrator_names in waiting_migrators_map.items(): + node_attrs = gx2.nodes[node] + attrs = node_attrs["payload"] + + for migrator_name in migrator_names: + # Look up the correct migrator instance for the waiting migrator + # (not the current migrator being processed) + migrators_by_name = _get_migrators_by_name() + if migrator_name not in migrators_by_name: + # Migrator not found, skip + continue + waiting_migrator = migrators_by_name[migrator_name] + + # Check if this node has a PR for the waiting migrator + nuid = waiting_migrator.migrator_uid(attrs) + + nuid_data = frozen_to_json_friendly(nuid)["data"] + + # Look for a matching PR in this node's PRed list + # No need to copy since we're not modifying the PR data + matching_pr_json = None + for pr_json in attrs.get("pr_info", {}).get("PRed", []): + if pr_json and pr_json.get("data") == nuid_data: + matching_pr_json = pr_json + break + + if matching_pr_json is None: + # Node doesn't have a PR for this migrator yet - skip + continue + + # Get PR data + pr_data = matching_pr_json.get("PR", {}) + pr_number = pr_data.get("number") + if pr_number is None: + continue + + # Create fake node name: migrator_{migrator_name}_{node}_{pr_number} + # I'm not sure we could have 2 PRs for the same package in another migration, + # but just to be safe + fake_parent = f"migrator_{migrator_name}_{node}_{pr_number}" + + # Add fake node to graph if it doesn't exist + if fake_parent not in gx2.nodes(): + gx2.add_node(fake_parent, payload={}) + + pr_url = pr_data.get("html_url", "") + pr_status = pr_data.get("state", "") + + if not pr_url: + feedstock_name = attrs.get("feedstock_name", node) + pr_url = f"https://github.com/conda-forge/{feedstock_name}-feedstock/pull/{pr_number}" + + fake_migrator_nodes[fake_parent] = { + "pre_pr_migrator_status": "", + "pr_url": pr_url, + "pr_status": pr_status, + } + feedstock_metadata[fake_parent] = fake_migrator_nodes[fake_parent] + + # Set status based on PR state + if pr_status == "closed": + # PR is closed but package is still waiting - this is an error! + out["bot-error"].add(fake_parent) + print( + f"Package '{node}' waiting for migrator '{migrator_name}' but PR #{pr_number} is already closed. " + f"Waiting logic may be incorrect.", + flush=True, + ) + else: + # PR is open or in progress + out["in-pr"].add(fake_parent) + + # Add edge from fake migrator node to waiting package + # (migrator blocks package, so migrator -> package) + if node in gx2.nodes() and not gx2.has_edge(fake_parent, node): + gx2.add_edge(fake_parent, node) + + # Populate descendants and children for fake migrator nodes (must be done after all edges are added) + for node_name, node_metadata in fake_migrator_nodes.items(): + node_metadata["num_descendants"] = len(nx.descendants(gx2, node_name)) + node_metadata["immediate_children"] = [ + k + for k in sorted(gx2.successors(node_name)) + if not gx2[k].get("payload", {}).get("archived", False) + ] + out2: Dict = {} for k in out.keys(): + # Include all items, even if not in build_sequence (like fake migrator nodes) out2[k] = list( sorted( out[k], key=lambda x: ( - build_sequence.index(x) if x in build_sequence else -1, + build_sequence.index(x) + if x in build_sequence + else len(build_sequence), x, ), ), ) out2["_feedstock_status"] = feedstock_metadata + for (e0, e1), edge_attrs in gx2.edges.items(): + # Skip edges involving fake migrator nodes - handle separately + if e0.startswith("migrator_") or e1.startswith("migrator_"): + continue + if ( e0 not in out["done"] and e1 not in out["done"] @@ -366,6 +530,25 @@ def graph_migrator_status( ): gv.edge(e0, e1) + # Add nodes and edges for fake migrator parents in visualization + # (Metadata and awaiting-parents status already added above, before sorting) + for node_name in gx2.nodes(): + if node_name.startswith("migrator_"): + migrator_display_name = node_name.replace("migrator_", "") + + # Style migrator nodes differently (awaiting-parents color is #fde725) + gv.node( + node_name, + label=_clean_text(migrator_display_name), + fillcolor="#fde725", # Same color as awaiting-parents + style="filled,dashed", + fontcolor="black", + ) + # Add edges from migrator to waiting packages with dashed style + for successor in gx2.successors(node_name): + if successor not in out["done"]: + gv.edge(node_name, successor, style="dashed", color="orange") + print(" len(gv):", num_viz, flush=True) out2["_num_viz"] = num_viz From 1e3b64adb056168838f1d7888deb50aa116ce7f8 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Fri, 28 Nov 2025 14:02:17 +0100 Subject: [PATCH 2/3] multipr --- conda_forge_tick/status_report.py | 109 +++++++++++++++--------------- 1 file changed, 54 insertions(+), 55 deletions(-) diff --git a/conda_forge_tick/status_report.py b/conda_forge_tick/status_report.py index 3ec454c27..27b957541 100644 --- a/conda_forge_tick/status_report.py +++ b/conda_forge_tick/status_report.py @@ -429,67 +429,66 @@ def graph_migrator_status( # Check if this node has a PR for the waiting migrator nuid = waiting_migrator.migrator_uid(attrs) - + nuid_data = frozen_to_json_friendly(nuid)["data"] - - # Look for a matching PR in this node's PRed list - # No need to copy since we're not modifying the PR data - matching_pr_json = None + + # Find all matching PRs in this node's PRed list + # Create one fake node per matching PR + matching_prs = [] for pr_json in attrs.get("pr_info", {}).get("PRed", []): if pr_json and pr_json.get("data") == nuid_data: - matching_pr_json = pr_json - break - - if matching_pr_json is None: + matching_prs.append(pr_json) + + if not matching_prs: # Node doesn't have a PR for this migrator yet - skip continue + + # Create a fake node for each matching PR + for matching_pr_json in matching_prs: + # Get PR data + pr_data = matching_pr_json.get("PR", {}) + pr_number = pr_data.get("number") + if pr_number is None: + continue + + # Create fake node name: migrator_{migrator_name}_{node}_{pr_number} + fake_parent = f"migrator_{migrator_name}_{node}_{pr_number}" + + # Add fake node to graph if it doesn't exist + if fake_parent not in gx2.nodes(): + gx2.add_node(fake_parent, payload={}) + + pr_url = pr_data.get("html_url", "") + pr_status = pr_data.get("state", "") + + if not pr_url: + feedstock_name = attrs.get("feedstock_name", node) + pr_url = f"https://github.com/conda-forge/{feedstock_name}-feedstock/pull/{pr_number}" + + fake_migrator_nodes[fake_parent] = { + "pre_pr_migrator_status": "", + "pr_url": pr_url, + "pr_status": pr_status, + } + feedstock_metadata[fake_parent] = fake_migrator_nodes[fake_parent] + + # Set status based on PR state + if pr_status == "closed": + # PR is closed but package is still waiting - this is an error! + out["bot-error"].add(fake_parent) + print( + f"Package '{node}' waiting for migrator '{migrator_name}' but PR #{pr_number} is already closed. " + f"Waiting logic may be incorrect.", + flush=True, + ) + else: + # PR is open or in progress + out["in-pr"].add(fake_parent) - # Get PR data - pr_data = matching_pr_json.get("PR", {}) - pr_number = pr_data.get("number") - if pr_number is None: - continue - - # Create fake node name: migrator_{migrator_name}_{node}_{pr_number} - # I'm not sure we could have 2 PRs for the same package in another migration, - # but just to be safe - fake_parent = f"migrator_{migrator_name}_{node}_{pr_number}" - - # Add fake node to graph if it doesn't exist - if fake_parent not in gx2.nodes(): - gx2.add_node(fake_parent, payload={}) - - pr_url = pr_data.get("html_url", "") - pr_status = pr_data.get("state", "") - - if not pr_url: - feedstock_name = attrs.get("feedstock_name", node) - pr_url = f"https://github.com/conda-forge/{feedstock_name}-feedstock/pull/{pr_number}" - - fake_migrator_nodes[fake_parent] = { - "pre_pr_migrator_status": "", - "pr_url": pr_url, - "pr_status": pr_status, - } - feedstock_metadata[fake_parent] = fake_migrator_nodes[fake_parent] - - # Set status based on PR state - if pr_status == "closed": - # PR is closed but package is still waiting - this is an error! - out["bot-error"].add(fake_parent) - print( - f"Package '{node}' waiting for migrator '{migrator_name}' but PR #{pr_number} is already closed. " - f"Waiting logic may be incorrect.", - flush=True, - ) - else: - # PR is open or in progress - out["in-pr"].add(fake_parent) - - # Add edge from fake migrator node to waiting package - # (migrator blocks package, so migrator -> package) - if node in gx2.nodes() and not gx2.has_edge(fake_parent, node): - gx2.add_edge(fake_parent, node) + # Add edge from fake migrator node to waiting package + # (migrator blocks package, so migrator -> package) + if node in gx2.nodes() and not gx2.has_edge(fake_parent, node): + gx2.add_edge(fake_parent, node) # Populate descendants and children for fake migrator nodes (must be done after all edges are added) for node_name, node_metadata in fake_migrator_nodes.items(): From feb774b85b350b3f67e9b569b81d64c35a0a5d85 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Fri, 28 Nov 2025 14:50:19 +0100 Subject: [PATCH 3/3] dont know --- conda_forge_tick/make_migrators.py | 11 +-- conda_forge_tick/migrators/core.py | 107 +++++++++++++++++++---------- conda_forge_tick/status_report.py | 30 ++------ 3 files changed, 77 insertions(+), 71 deletions(-) diff --git a/conda_forge_tick/make_migrators.py b/conda_forge_tick/make_migrators.py index e9b62f226..2fdf7913d 100644 --- a/conda_forge_tick/make_migrators.py +++ b/conda_forge_tick/make_migrators.py @@ -1004,11 +1004,6 @@ def load_migrators(skip_paused: bool = True) -> MutableSequence[Migrator]: pinning_migrators = [] longterm_migrators = [] all_names = get_all_keys_for_hashmap("migrators") - # Only load python314 and python314t migrators - filter BEFORE submitting to pool - allowed_migrators = {"python314", "python314t"} - all_names = [name for name in all_names if name in allowed_migrators] - print(f"Loading only: {all_names}", flush=True) - with executor("process", 2) as pool: futs = [pool.submit(_load, name) for name in all_names] @@ -1032,13 +1027,11 @@ def load_migrators(skip_paused: bool = True) -> MutableSequence[Migrator]: else: migrators.append(migrator) - # Commented out - version migrator is slow - # version_migrator = _make_version_migrator(load_existing_graph()) + version_migrator = _make_version_migrator(load_existing_graph()) RNG.shuffle(pinning_migrators) RNG.shuffle(longterm_migrators) - # migrators = [version_migrator] + migrators + pinning_migrators + longterm_migrators - migrators = migrators + pinning_migrators + longterm_migrators + migrators = [version_migrator] + migrators + pinning_migrators + longterm_migrators return migrators diff --git a/conda_forge_tick/migrators/core.py b/conda_forge_tick/migrators/core.py index 7cdc1b147..2e6758201 100644 --- a/conda_forge_tick/migrators/core.py +++ b/conda_forge_tick/migrators/core.py @@ -704,6 +704,69 @@ def migrator_uid(self, attrs: "AttrsTypedDict") -> "MigrationUidTypedDict": return d + def get_blocking_predecessors( + self, attrs: "AttrsTypedDict", graph: nx.DiGraph | None = None + ) -> list[tuple[str, dict | None]]: + """Get list of predecessors (dependencies) that are blocking this package. + + Parameters + ---------- + attrs : AttrsTypedDict + The node attributes + graph : nx.DiGraph | None, optional + Graph to use for finding predecessors. If None, uses self.graph. + Useful when the node might not be in self.graph. + + Returns: + List of (node_name, pr_data) tuples for blocking predecessors. + pr_data is None if predecessor doesn't have this migration in PRed. + pr_data is a dict with PR info if predecessor has an open PR for this migration. + """ + use_graph = graph if graph is not None else self.graph + if use_graph is None: + return [] + + feedstock_name = attrs.get("feedstock_name") + if feedstock_name not in use_graph.nodes(): + return [] + + ignored_deps = getattr(self, "ignored_deps_per_node", {}) + + blocking = [] + for node, payload in _gen_active_feedstocks_payloads( + use_graph.predecessors(feedstock_name), + use_graph, + ): + if node in ignored_deps.get( + attrs.get("feedstock_name", None), + [], + ): + continue + + muid = frozen_to_json_friendly(self.migrator_uid(payload)) + + if muid not in _sanitized_muids( + payload.get("pr_info", {}).get("PRed", []), + ): + logger.debug("not yet built: %s", node) + blocking.append((node, None)) + continue + + m_pred_json = None + for pr_json in payload.get("pr_info", {}).get("PRed", []): + if pr_json.get("data") == muid["data"]: + m_pred_json = pr_json + break + + if ( + m_pred_json + and m_pred_json.get("PR", {"state": "open"}).get("state", "") == "open" + ): + logger.debug("not yet built: %s", node) + blocking.append((node, m_pred_json.get("PR", {}))) + + return blocking + def order( self, graph: nx.DiGraph, @@ -910,44 +973,12 @@ def all_predecessors_issued(self, attrs: "AttrsTypedDict") -> bool: return True def predecessors_not_yet_built(self, attrs: "AttrsTypedDict") -> bool: - # Check if all upstreams have been built - if self.graph is None: - raise ValueError("graph is None") - for node, payload in _gen_active_feedstocks_payloads( - self.graph.predecessors(attrs["feedstock_name"]), - self.graph, - ): - if node in self.ignored_deps_per_node.get( - attrs.get("feedstock_name", None), - [], - ): - continue - - muid = frozen_to_json_friendly(self.migrator_uid(payload)) - - if muid not in _sanitized_muids( - payload.get("pr_info", {}).get("PRed", []), - ): - logger.debug("not yet built: %s", node) - return True - - # This is due to some PRed_json loss due to bad graph deploy outage - for m_pred_json in payload.get("pr_info", {}).get("PRed", []): - if m_pred_json["data"] == muid["data"]: - break - else: - m_pred_json = None - - # note that if the bot is missing the PR we assume it is open - # so that errors halt the migration and can be fixed - if ( - m_pred_json - and m_pred_json.get("PR", {"state": "open"}).get("state", "") == "open" - ): - logger.debug("not yet built: %s", node) - return True - - return False + """Check if any predecessors are blocking this package. + + Returns True if any predecessor is blocking, False otherwise. + This method uses get_blocking_predecessors to determine blocking status. + """ + return len(self.get_blocking_predecessors(attrs)) > 0 def filter_not_in_migration(self, attrs, not_bad_str_start=""): if super().filter_not_in_migration(attrs, not_bad_str_start): diff --git a/conda_forge_tick/status_report.py b/conda_forge_tick/status_report.py index 27b957541..0eb76c7eb 100644 --- a/conda_forge_tick/status_report.py +++ b/conda_forge_tick/status_report.py @@ -194,7 +194,6 @@ def _get_waiting_migrators(migrator: Migrator, attrs: dict) -> list[str]: if not wait_for_migrators: return [] - # Check if we're actually waiting (i.e., migrators not all closed) found_migrators = set() for migration in attrs.get("pr_info", {}).get("PRed", []): name = migration.get("data", {}).get("name", "") @@ -203,10 +202,8 @@ def _get_waiting_migrators(migrator: Migrator, attrs: dict) -> list[str]: found_migrators.add(name) state = migration.get("PR", {}).get("state", "") if state != "closed": - # Still waiting for this one return list(wait_for_migrators) - # Check if any migrators are missing missing_migrators = set(wait_for_migrators) - found_migrators if missing_migrators: return list(wait_for_migrators) @@ -409,9 +406,6 @@ def graph_migrator_status( if waiting_migrators: waiting_migrators_map[node] = waiting_migrators - # Add fake migrator nodes and edges after processing regular nodes - # Create one fake node per waiting migrator PR: migrator_{migrator_name}_{node}_{pr_number} - # This represents the PR for the waiting migrator on the node itself (not its predecessors) fake_migrator_nodes: Dict[str, Dict] = {} for node, migrator_names in waiting_migrators_map.items(): @@ -419,42 +413,30 @@ def graph_migrator_status( attrs = node_attrs["payload"] for migrator_name in migrator_names: - # Look up the correct migrator instance for the waiting migrator - # (not the current migrator being processed) migrators_by_name = _get_migrators_by_name() if migrator_name not in migrators_by_name: - # Migrator not found, skip continue waiting_migrator = migrators_by_name[migrator_name] - # Check if this node has a PR for the waiting migrator nuid = waiting_migrator.migrator_uid(attrs) - nuid_data = frozen_to_json_friendly(nuid)["data"] - # Find all matching PRs in this node's PRed list - # Create one fake node per matching PR matching_prs = [] for pr_json in attrs.get("pr_info", {}).get("PRed", []): if pr_json and pr_json.get("data") == nuid_data: matching_prs.append(pr_json) if not matching_prs: - # Node doesn't have a PR for this migrator yet - skip continue - # Create a fake node for each matching PR for matching_pr_json in matching_prs: - # Get PR data pr_data = matching_pr_json.get("PR", {}) pr_number = pr_data.get("number") if pr_number is None: continue - # Create fake node name: migrator_{migrator_name}_{node}_{pr_number} fake_parent = f"migrator_{migrator_name}_{node}_{pr_number}" - # Add fake node to graph if it doesn't exist if fake_parent not in gx2.nodes(): gx2.add_node(fake_parent, payload={}) @@ -472,25 +454,25 @@ def graph_migrator_status( } feedstock_metadata[fake_parent] = fake_migrator_nodes[fake_parent] - # Set status based on PR state if pr_status == "closed": - # PR is closed but package is still waiting - this is an error! out["bot-error"].add(fake_parent) print( f"Package '{node}' waiting for migrator '{migrator_name}' but PR #{pr_number} is already closed. " f"Waiting logic may be incorrect.", flush=True, ) + blocking_preds = waiting_migrator.get_blocking_predecessors(attrs, gx2) + for pred_node, _ in blocking_preds: + if pred_node in gx2.nodes() and not gx2.has_edge( + pred_node, fake_parent + ): + gx2.add_edge(pred_node, fake_parent) else: - # PR is open or in progress out["in-pr"].add(fake_parent) - # Add edge from fake migrator node to waiting package - # (migrator blocks package, so migrator -> package) if node in gx2.nodes() and not gx2.has_edge(fake_parent, node): gx2.add_edge(fake_parent, node) - # Populate descendants and children for fake migrator nodes (must be done after all edges are added) for node_name, node_metadata in fake_migrator_nodes.items(): node_metadata["num_descendants"] = len(nx.descendants(gx2, node_name)) node_metadata["immediate_children"] = [