From 51f747adad7e39fa91ab312ae5f270d98b8ef28f Mon Sep 17 00:00:00 2001 From: Andrey Golovanov Date: Sun, 23 Feb 2025 19:09:22 +0000 Subject: [PATCH 1/3] Introducing the topology dsl. --- README.md | 2 +- ngraph/blueprints.py | 394 +++++++++++++++++++++++++++++ ngraph/network.py | 80 +++--- ngraph/scenario.py | 236 ++++++++--------- ngraph/workflow/build_graph.py | 2 - notebooks/lib_examples.ipynb | 4 +- tests/scenarios/scenario_1.yaml | 100 ++++---- tests/scenarios/scenario_2.yaml | 204 +++++++++++++++ tests/test_blueprints.py | 342 +++++++++++++++++++++++++ tests/test_blueprints_helpers.py | 379 +++++++++++++++++++++++++++ tests/test_network.py | 8 +- tests/test_scenario.py | 34 +-- tests/workflow/test_build_graph.py | 7 +- 13 files changed, 1542 insertions(+), 250 deletions(-) create mode 100644 ngraph/blueprints.py create mode 100644 tests/scenarios/scenario_2.yaml create mode 100644 tests/test_blueprints.py create mode 100644 tests/test_blueprints_helpers.py diff --git a/README.md b/README.md index 143f295..1951473 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ Note: Don't forget to use a virtual environment (e.g., `venv`) to avoid conflict ### Traffic demands placement on a graph ```python """ - Demonstrates traffic engineering by placing two bidirectional demands on a network. + Demonstrates traffic engineering by placing two demands on a network. Graph topology (metrics/capacities): diff --git a/ngraph/blueprints.py b/ngraph/blueprints.py new file mode 100644 index 0000000..80afab5 --- /dev/null +++ b/ngraph/blueprints.py @@ -0,0 +1,394 @@ +from __future__ import annotations + +import copy +from dataclasses import dataclass +from typing import Any, Dict, List + +from ngraph.network import Network, Node, Link + + +@dataclass(slots=True) +class Blueprint: + """ + Represents a reusable blueprint for hierarchical sub-topologies. + + A blueprint may contain multiple groups of nodes (each can have a node_count + and a name_template), plus adjacency rules describing how those groups connect. + + Attributes: + name: Unique identifier of this blueprint. + groups: A mapping of group_name -> group definition (e.g. node_count, name_template). + adjacency: A list of adjacency dictionaries describing how groups are linked. + """ + + name: str + groups: Dict[str, Any] + adjacency: List[Dict[str, Any]] + + +@dataclass(slots=True) +class DSLExpansionContext: + """ + Carries the blueprint definitions and the final Network instance + to be populated during DSL expansion. + + Attributes: + blueprints: A dictionary of blueprint name -> Blueprint object. + network: The Network into which expanded nodes/links will be inserted. + """ + + blueprints: Dict[str, Blueprint] + network: Network + + +def expand_network_dsl(data: Dict[str, Any]) -> Network: + """ + Expands a combined blueprint + network DSL into a complete Network object. + + Overall flow: + 1) Parse "blueprints" into Blueprint objects. + 2) Build a new Network from "network" metadata (name, version, etc.). + 3) Expand 'network["groups"]': + - If a group references a blueprint, incorporate that blueprint's subgroups. + - Otherwise, directly create nodes (e.g., node_count). + 4) Expand adjacency definitions in 'network["adjacency"]'. + 5) Process any direct node/link definitions. + + Args: + data: The YAML-parsed dictionary containing optional "blueprints" + "network". + + Returns: + A fully expanded Network object with all nodes and links. + """ + # 1) Parse blueprint definitions + blueprint_map: Dict[str, Blueprint] = {} + for bp_name, bp_data in data.get("blueprints", {}).items(): + blueprint_map[bp_name] = Blueprint( + name=bp_name, + groups=bp_data.get("groups", {}), + adjacency=bp_data.get("adjacency", []), + ) + + # 2) Initialize the Network from "network" metadata + network_data = data.get("network", {}) + net = Network() + if "name" in network_data: + net.attrs["name"] = network_data["name"] + if "version" in network_data: + net.attrs["version"] = network_data["version"] + + # Create a context + ctx = DSLExpansionContext(blueprints=blueprint_map, network=net) + + # 3) Expand top-level groups + for group_name, group_def in network_data.get("groups", {}).items(): + _expand_group( + ctx, + parent_path="", + group_name=group_name, + group_def=group_def, + blueprint_expansion=False, + ) + + # 4) Expand adjacency definitions + for adj_def in network_data.get("adjacency", []): + _expand_adjacency(ctx, adj_def) + + # 5) Process direct node/link definitions + _process_direct_nodes_and_links(ctx.network, network_data) + + return net + + +def _expand_group( + ctx: DSLExpansionContext, + parent_path: str, + group_name: str, + group_def: Dict[str, Any], + *, + blueprint_expansion: bool = False, +) -> None: + """ + Expands a single group definition into either: + - Another blueprint's subgroups, or + - A direct node group (node_count, name_template). + + We do *not* skip the subgroup name even inside blueprint expansion, because + typically the 'group_name' is "leaf"/"spine" etc., not the blueprint’s name. + + So the final path is always 'parent_path + "/" + group_name' if parent_path is non-empty, + otherwise just group_name. + """ + # Construct the effective path by appending group_name if parent_path is non-empty + if parent_path: + effective_path = f"{parent_path}/{group_name}" + else: + effective_path = group_name + + if "use_blueprint" in group_def: + # Expand blueprint subgroups + blueprint_name: str = group_def["use_blueprint"] + bp = ctx.blueprints.get(blueprint_name) + if not bp: + raise ValueError( + f"Group '{group_name}' references unknown blueprint '{blueprint_name}'." + ) + + param_overrides: Dict[str, Any] = group_def.get("parameters", {}) + coords = group_def.get("coords") + + # For each subgroup in the blueprint, apply overrides and expand + for bp_sub_name, bp_sub_def in bp.groups.items(): + merged_def = _apply_parameters(bp_sub_name, bp_sub_def, param_overrides) + if coords is not None and "coords" not in merged_def: + merged_def["coords"] = coords + + _expand_group( + ctx, + parent_path=effective_path, + group_name=bp_sub_name, + group_def=merged_def, + blueprint_expansion=True, + ) + + # Expand blueprint adjacency + for adj_def in bp.adjacency: + _expand_blueprint_adjacency(ctx, adj_def, effective_path) + + else: + # It's a direct node group + node_count = group_def.get("node_count", 1) + name_template = group_def.get("name_template", f"{group_name}-{{node_num}}") + + for i in range(1, node_count + 1): + label = name_template.format(node_num=i) + node_name = f"{effective_path}/{label}" if effective_path else label + + node = Node(name=node_name) + if "coords" in group_def: + node.attrs["coords"] = group_def["coords"] + node.attrs.setdefault("type", "node") + + ctx.network.add_node(node) + + +def _expand_blueprint_adjacency( + ctx: DSLExpansionContext, + adj_def: Dict[str, Any], + parent_path: str, +) -> None: + """ + Expands adjacency definitions from within a blueprint, using parent_path as the local root. + """ + source_rel = adj_def["source"] + target_rel = adj_def["target"] + pattern = adj_def.get("pattern", "mesh") + link_params = adj_def.get("link_params", {}) + + src_path = _join_paths(parent_path, source_rel) + tgt_path = _join_paths(parent_path, target_rel) + + _expand_adjacency_pattern(ctx, src_path, tgt_path, pattern, link_params) + + +def _expand_adjacency( + ctx: DSLExpansionContext, + adj_def: Dict[str, Any], +) -> None: + """ + Expands a top-level adjacency definition from 'network.adjacency'. + """ + source_path_raw = adj_def["source"] + target_path_raw = adj_def["target"] + pattern = adj_def.get("pattern", "mesh") + link_params = adj_def.get("link_params", {}) + + # Strip leading '/' from source/target paths + source_path = _join_paths("", source_path_raw) + target_path = _join_paths("", target_path_raw) + + _expand_adjacency_pattern(ctx, source_path, target_path, pattern, link_params) + + +def _expand_adjacency_pattern( + ctx: DSLExpansionContext, + source_path: str, + target_path: str, + pattern: str, + link_params: Dict[str, Any], +) -> None: + """ + Generates Link objects for the chosen adjacency pattern among matched nodes. + + Supported Patterns: + * "mesh": Cross-connect every node from source side to every node on target side, + skipping self-loops, and deduplicating reversed pairs. + * "one_to_one": Pair each source node with exactly one target node, supporting + wrap-around if one side is an integer multiple of the other. + Also skips self-loops. + """ + source_nodes = _find_nodes_by_path(ctx.network, source_path) + target_nodes = _find_nodes_by_path(ctx.network, target_path) + + if not source_nodes or not target_nodes: + return + + dedup_pairs = set() + + if pattern == "mesh": + for sn in source_nodes: + for tn in target_nodes: + # Skip self-loops + if sn.name == tn.name: + continue + pair = tuple(sorted((sn.name, tn.name))) + if pair not in dedup_pairs: + dedup_pairs.add(pair) + _create_link(ctx.network, sn.name, tn.name, link_params) + + elif pattern == "one_to_one": + s_count = len(source_nodes) + t_count = len(target_nodes) + bigger, smaller = max(s_count, t_count), min(s_count, t_count) + + if bigger % smaller != 0: + raise ValueError( + f"one_to_one pattern requires either equal node counts " + f"or a valid wrap-around. Got {s_count} vs {t_count}." + ) + + # total 'bigger' connections + for i in range(bigger): + if s_count >= t_count: + sn = source_nodes[i].name + tn = target_nodes[i % t_count].name + else: + sn = source_nodes[i % s_count].name + tn = target_nodes[i].name + + # Skip self-loops + if sn == tn: + continue + + pair = tuple(sorted((sn, tn))) + if pair not in dedup_pairs: + dedup_pairs.add(pair) + _create_link(ctx.network, sn, tn, link_params) + + else: + raise ValueError(f"Unknown adjacency pattern: {pattern}") + + +def _create_link( + net: Network, source: str, target: str, link_params: Dict[str, Any] +) -> None: + """ + Creates and adds a Link to the network, applying capacity/cost/attrs from link_params. + """ + capacity = link_params.get("capacity", 1.0) + cost = link_params.get("cost", 1.0) + attrs = copy.deepcopy(link_params.get("attrs", {})) + + link = Link( + source=source, + target=target, + capacity=capacity, + cost=cost, + attrs=attrs, + ) + net.add_link(link) + + +def _apply_parameters( + subgroup_name: str, subgroup_def: Dict[str, Any], params_overrides: Dict[str, Any] +) -> Dict[str, Any]: + """ + Applies user-provided parameter overrides to a blueprint subgroup. + + E.g.: + if 'spine.node_count' = 6 is in params_overrides, + we set 'node_count'=6 for the 'spine' subgroup. + """ + out = dict(subgroup_def) + for key, val in params_overrides.items(): + parts = key.split(".") + if parts[0] == subgroup_name and len(parts) > 1: + field_name = ".".join(parts[1:]) + out[field_name] = val + return out + + +def _join_paths(parent_path: str, rel_path: str) -> str: + """ + If rel_path starts with '/', interpret that as relative to 'parent_path'; + otherwise, simply append rel_path to parent_path with '/' if needed. + """ + if rel_path.startswith("/"): + rel_path = rel_path[1:] + if parent_path: + return f"{parent_path}/{rel_path}" + else: + return rel_path + if parent_path: + return f"{parent_path}/{rel_path}" + return rel_path + + +def _find_nodes_by_path(net: Network, path: str) -> List[Node]: + """ + Returns all nodes whose name is exactly 'path' or begins with 'path/'. + If none are found, tries 'path-' as a fallback prefix. + If still none are found, tries partial prefix "path" => "pathX". + + Examples: + path="SEA/clos_instance/spine" might match "SEA/clos_instance/spine/myspine-1" + path="S" might match "S1", "S2" if we resort to partial prefix logic. + """ + # 1) Exact or slash-based + result = [ + n for n in net.nodes.values() if n.name == path or n.name.startswith(f"{path}/") + ] + if result: + return result + + # 2) Fallback: path- + result = [n for n in net.nodes.values() if n.name.startswith(f"{path}-")] + if result: + return result + + # 3) Partial + partial = [] + for n in net.nodes.values(): + if n.name.startswith(path) and n.name != path: + partial.append(n) + return partial + + +def _process_direct_nodes_and_links(net: Network, network_data: Dict[str, Any]) -> None: + """ + Processes direct node definitions (network_data["nodes"]) and direct link definitions + (network_data["links"]). + """ + # Direct node definitions + for node_name, node_attrs in network_data.get("nodes", {}).items(): + if node_name not in net.nodes: + new_node = Node(name=node_name, attrs=node_attrs or {}) + new_node.attrs.setdefault("type", "node") + net.add_node(new_node) + + # Direct link definitions + existing_node_names = set(net.nodes.keys()) + for link_info in network_data.get("links", []): + source = link_info["source"] + target = link_info["target"] + if source not in existing_node_names or target not in existing_node_names: + raise ValueError(f"Link references unknown node(s): {source}, {target}.") + link_params = link_info.get("link_params", {}) + link = Link( + source=source, + target=target, + capacity=link_params.get("capacity", 1.0), + cost=link_params.get("cost", 1.0), + attrs=link_params.get("attrs", {}), + ) + net.add_link(link) diff --git a/ngraph/network.py b/ngraph/network.py index d8f346d..ab5f8de 100644 --- a/ngraph/network.py +++ b/ngraph/network.py @@ -1,4 +1,5 @@ from __future__ import annotations + import uuid import base64 from dataclasses import dataclass, field @@ -6,9 +7,7 @@ def new_base64_uuid() -> str: - """ - Generate a Base64-encoded UUID without padding (a string with 22 characters). - """ + """Generates a Base64-encoded UUID without padding (22 characters).""" return base64.urlsafe_b64encode(uuid.uuid4().bytes).decode("ascii").rstrip("=") @@ -20,13 +19,14 @@ class Node: Each node is uniquely identified by its name, which is used as the key in the Network's node dictionary. - :param name: The unique name of the node. - :param attrs: Optional extra metadata for the node. For example: - { - "type": "node", # auto-tagged upon add_node - "coords": [lat, lon], # user-provided - "region": "west_coast" # user-provided - } + Attributes: + name (str): The unique name of the node. + attrs (Dict[str, Any]): Optional extra metadata for the node. For example: + { + "type": "node", # auto-tagged upon add_node + "coords": [lat, lon], # user-provided + "region": "west_coast" # user-provided + } """ name: str @@ -42,33 +42,30 @@ class Link: is auto-generated from the source, target, and a random Base64-encoded UUID, allowing multiple distinct links between the same nodes. - :param source: Unique name of the source node. - :param target: Unique name of the target node. - :param capacity: Link capacity (default 1.0). - :param latency: Link latency (default 1.0). - :param cost: Link cost (default 1.0). - :param attrs: Optional extra metadata for the link. For example: - { - "type": "link", # auto-tagged upon add_link - "distance_km": 1500, # user-provided - "fiber_provider": "Lumen", # user-provided - } - :param id: Auto-generated unique link identifier, e.g. "SEA-DEN-abCdEf..." + Attributes: + source (str): Unique name of the source node. + target (str): Unique name of the target node. + capacity (float): Link capacity (default is 1.0). + cost (float): Link cost (default is 1.0). + attrs (Dict[str, Any]): Optional extra metadata for the link. + For example: + { + "type": "link", # auto-tagged upon add_link + "distance_km": 1500, # user-provided + "fiber_provider": "Lumen", # user-provided + } + id (str): Auto-generated unique link identifier, e.g. "SEA-DEN-abCdEf..." """ source: str target: str capacity: float = 1.0 - latency: float = 1.0 cost: float = 1.0 attrs: Dict[str, Any] = field(default_factory=dict) id: str = field(init=False) def __post_init__(self) -> None: - """ - Auto-generate a unique link ID by combining the source, target, - and a random Base64-encoded UUID. - """ + """Auto-generates a unique link ID by combining the source, target, and a random Base64-encoded UUID.""" self.id = f"{self.source}-{self.target}-{new_base64_uuid()}" @@ -77,13 +74,14 @@ class Network: """ A container for network nodes and links. - Nodes are stored in a dictionary keyed by their unique names (:attr:`Node.name`). - Links are stored in a dictionary keyed by their auto-generated IDs (:attr:`Link.id`). + Nodes are stored in a dictionary keyed by their unique names (Node.name). + Links are stored in a dictionary keyed by their auto-generated IDs (Link.id). The 'attrs' dict allows extra network metadata. - :param nodes: Mapping from node name -> Node object. - :param links: Mapping from link id -> Link object. - :param attrs: Optional extra metadata for the network itself. + Attributes: + nodes (Dict[str, Node]): Mapping from node name -> Node object. + links (Dict[str, Link]): Mapping from link ID -> Link object. + attrs (Dict[str, Any]): Optional extra metadata for the network. """ nodes: Dict[str, Node] = field(default_factory=dict) @@ -92,13 +90,16 @@ class Network: def add_node(self, node: Node) -> None: """ - Add a node to the network, keyed by its :attr:`Node.name`. + Adds a node to the network, keyed by its name. This method also auto-tags the node with ``node.attrs["type"] = "node"`` if it's not already set. - :param node: The Node to add. - :raises ValueError: If a node with the same name is already in the network. + Args: + node (Node): The node to add. + + Raises: + ValueError: If a node with the same name is already in the network. """ node.attrs.setdefault("type", "node") if node.name in self.nodes: @@ -107,13 +108,16 @@ def add_node(self, node: Node) -> None: def add_link(self, link: Link) -> None: """ - Add a link to the network, keyed by its auto-generated :attr:`Link.id`. + Adds a link to the network, keyed by its auto-generated ID. This method also auto-tags the link with ``link.attrs["type"] = "link"`` if it's not already set. - :param link: The Link to add. - :raises ValueError: If the source/target node is not present in the network. + Args: + link (Link): The link to add. + + Raises: + ValueError: If the source or target node is not present in the network. """ if link.source not in self.nodes: raise ValueError(f"Source node '{link.source}' not found in network.") diff --git a/ngraph/scenario.py b/ngraph/scenario.py index 83b8116..7caffd9 100644 --- a/ngraph/scenario.py +++ b/ngraph/scenario.py @@ -1,4 +1,5 @@ from __future__ import annotations + import yaml from dataclasses import dataclass, field from typing import Any, Dict, List @@ -13,34 +14,26 @@ @dataclass(slots=True) class Scenario: """ - Represents a complete scenario, including: - - The network (nodes and links). - - A failure policy (with one or more rules). - - Traffic demands. - - A workflow of steps to execute. - - A results container for storing outputs. - - Typical usage: - 1. Create a Scenario from YAML: :: - - scenario = Scenario.from_yaml(yaml_str) - - 2. Run it: :: + Represents a complete scenario for building and executing network workflows. - scenario.run() - - 3. Check scenario.results for step outputs. + This scenario includes: + - A network (nodes and links). + - A failure policy (one or more rules). + - A set of traffic demands. + - A list of workflow steps to execute. + - A results container for storing outputs. - :param network: - The network model containing nodes and links. - :param failure_policy: - The multi-rule failure policy describing how and which entities fail. - :param traffic_demands: - A list of traffic demands describing source/target flows. - :param workflow: - A list of workflow steps defining the scenario pipeline. - :param results: - A Results object to store outputs from workflow steps. + Typical usage example: + scenario = Scenario.from_yaml(yaml_str) + scenario.run() + # Inspect scenario.results + + Attributes: + network (Network): The network model containing nodes and links. + failure_policy (FailurePolicy): Defines how and which entities fail. + traffic_demands (List[TrafficDemand]): Describes source/target flows. + workflow (List[WorkflowStep]): Defines the execution pipeline. + results (Results): Stores outputs from the workflow steps. """ network: Network @@ -51,11 +44,10 @@ class Scenario: def run(self) -> None: """ - Execute the scenario's workflow steps in the defined order. + Executes the scenario's workflow steps in the defined order. - Each step has access to :attr:`Scenario.network`, - :attr:`Scenario.failure_policy`, etc. Steps may store outputs in - :attr:`Scenario.results`. + Each step may access and modify scenario data, or store outputs in + scenario.results. """ for step in self.workflow: step.run(self) @@ -63,54 +55,22 @@ def run(self) -> None: @classmethod def from_yaml(cls, yaml_str: str) -> Scenario: """ - Construct a :class:`Scenario` from a YAML string. + Constructs a Scenario from a YAML string. Expected top-level YAML keys: - - ``network``: Node/Link definitions - - ``failure_policy``: A multi-rule policy - - ``traffic_demands``: List of demands - - ``workflow``: Steps to run + - network: Node/Link definitions (with optional name/version) + - failure_policy: Multi-rule policy + - traffic_demands: List of demands + - workflow: Steps to execute - Example: + Args: + yaml_str (str): The YAML string that defines the scenario. - .. code-block:: yaml + Returns: + Scenario: An initialized Scenario instance. - network: - nodes: - SEA: { coords: [47.6062, -122.3321] } - SFO: { coords: [37.7749, -122.4194] } - links: - - source: SEA - target: SFO - capacity: 100 - attrs: { distance_km: 1300 } - - failure_policy: - name: "multi_rule_example" - rules: - - conditions: - - attr: "type" - operator: "==" - value: "node" - logic: "and" - rule_type: "choice" - count: 1 - - traffic_demands: - - source: SEA - target: SFO - demand: 50 - - workflow: - - step_type: BuildGraph - name: build_graph - - :param yaml_str: - The YAML string defining a scenario. - :returns: - A fully constructed :class:`Scenario` instance. - :raises ValueError: - If the YAML is malformed or missing required sections. + Raises: + ValueError: If the YAML is malformed or missing required sections. """ data = yaml.safe_load(yaml_str) if not isinstance(data, dict): @@ -142,48 +102,65 @@ def from_yaml(cls, yaml_str: str) -> Scenario: @staticmethod def _build_network(network_data: Dict[str, Any]) -> Network: """ - Construct a :class:`Network` object from a dictionary containing 'nodes' and 'links'. - The dictionary is expected to look like: - - .. code-block:: yaml + Constructs a Network from a dictionary containing optional 'name', + 'version', 'nodes', and 'links' (with nested link_params). + Example structure for network_data: + name: "6-node-l3-us-backbone" + version: "1.0" nodes: SEA: { coords: [47.6062, -122.3321] } SFO: { coords: [37.7749, -122.4194] } - links: - source: SEA target: SFO - capacity: 100 - latency: 5 - cost: 10 - attrs: - distance_km: 1300 - - :param network_data: - Dictionary with optional keys 'nodes' and 'links'. - :returns: - A :class:`Network` containing the parsed nodes and links. - :raises ValueError: - If a link references nodes not defined in the network. + link_params: + capacity: 200 + cost: 100 + attrs: + distance_km: 1500 + + Args: + network_data (Dict[str, Any]): Dictionary with optional keys + 'name', 'version', 'nodes', and 'links'. + + Returns: + Network: A fully constructed network containing the parsed nodes + and links. + + Raises: + ValueError: If a link references nodes not defined in the network. """ net = Network() + # Store optional metadata in the network's attrs + if "name" in network_data: + net.attrs["name"] = network_data["name"] + if "version" in network_data: + net.attrs["version"] = network_data["version"] + # Add nodes - nodes = network_data.get("nodes", {}) - for node_name, node_attrs in nodes.items(): + nodes_dict = network_data.get("nodes", {}) + for node_name, node_attrs in nodes_dict.items(): net.add_node(Node(name=node_name, attrs=node_attrs or {})) + valid_node_names = set(nodes_dict.keys()) + # Add links - links = network_data.get("links", []) - for link_info in links: + links_data = network_data.get("links", []) + for link_info in links_data: + source = link_info["source"] + target = link_info["target"] + if source not in valid_node_names or target not in valid_node_names: + raise ValueError(f"Link references unknown node(s): {source}, {target}") + + link_params = link_info.get("link_params", {}) link = Link( - source=link_info["source"], - target=link_info["target"], - capacity=link_info.get("capacity", 1.0), - latency=link_info.get("latency", 1.0), - cost=link_info.get("cost", 1.0), - attrs=link_info.get("attrs", {}), + source=source, + target=target, + capacity=link_params.get("capacity", 1.0), + cost=link_params.get("cost", 1.0), + attrs=link_params.get("attrs", {}), ) net.add_link(link) @@ -192,28 +169,28 @@ def _build_network(network_data: Dict[str, Any]) -> Network: @staticmethod def _build_failure_policy(fp_data: Dict[str, Any]) -> FailurePolicy: """ - Construct a :class:`FailurePolicy` from YAML data that may look like: - - .. code-block:: yaml + Constructs a FailurePolicy from data that may specify multiple rules. + Example structure: failure_policy: - name: "multi_rule_example" - description: "Example of multi-rule approach" + name: "anySingleLink" + description: "Test single-link failures." rules: - conditions: - attr: "type" operator: "==" value: "link" logic: "and" - rule_type: "random" - probability: 0.1 + rule_type: "choice" + count: 1 + + Args: + fp_data (Dict[str, Any]): Dictionary for the 'failure_policy' + section of the YAML. - :param fp_data: - Dictionary from the 'failure_policy' section of YAML. - :returns: - A :class:`FailurePolicy` object with a list of :class:`FailureRule`. + Returns: + FailurePolicy: A policy containing a list of FailureRule objects. """ - # Extract the list of rules rules_data = fp_data.get("rules", []) rules: List[FailureRule] = [] @@ -237,7 +214,7 @@ def _build_failure_policy(fp_data: Dict[str, Any]) -> FailurePolicy: ) rules.append(rule) - # All other key-value pairs go into policy.attrs (e.g. "name", "description") + # Put any additional keys (e.g., "name", "description") into policy.attrs attrs = {k: v for k, v in fp_data.items() if k != "rules"} return FailurePolicy(rules=rules, attrs=attrs) @@ -247,37 +224,38 @@ def _build_workflow_steps( workflow_data: List[Dict[str, Any]] ) -> List[WorkflowStep]: """ - Convert a list of workflow step dictionaries into instantiated - :class:`WorkflowStep` objects. + Converts workflow step dictionaries into instantiated WorkflowStep objects. - Each step dict must have a ``step_type`` referencing a registered - workflow step in :attr:`WORKFLOW_STEP_REGISTRY`. Any additional - keys are passed as init arguments. + Each step dict must have a "step_type" referencing a registered workflow + step in WORKFLOW_STEP_REGISTRY. Any additional keys are passed as init + arguments to the WorkflowStep subclass. Example: - - .. code-block:: yaml - workflow: - step_type: BuildGraph name: build_graph - step_type: ComputeRoutes name: compute_routes - :param workflow_data: - A list of dictionaries, each describing a workflow step. - :returns: - A list of instantiated :class:`WorkflowStep` objects in the same order. - :raises ValueError: - If any dict lacks "step_type" or references an unknown type. + Args: + workflow_data (List[Dict[str, Any]]): A list of dictionaries, each + describing a workflow step. + + Returns: + List[WorkflowStep]: A list of WorkflowStep instances, constructed + in the order provided. + + Raises: + ValueError: If a dict lacks "step_type" or references an unknown + type. """ steps: List[WorkflowStep] = [] for step_info in workflow_data: step_type = step_info.get("step_type") if not step_type: raise ValueError( - "Each workflow entry must have a 'step_type' field " - "indicating which WorkflowStep subclass to use." + "Each workflow entry must have a 'step_type' field indicating " + "which WorkflowStep subclass to use." ) step_cls = WORKFLOW_STEP_REGISTRY.get(step_type) if not step_cls: diff --git a/ngraph/workflow/build_graph.py b/ngraph/workflow/build_graph.py index 30cacf4..dc2e134 100644 --- a/ngraph/workflow/build_graph.py +++ b/ngraph/workflow/build_graph.py @@ -41,7 +41,6 @@ def run(self, scenario: Scenario) -> None: key=link.id, capacity=link.capacity, cost=link.cost, - latency=link.latency, **link.attrs, ) # Reverse edge uses link.id + "_rev" @@ -52,7 +51,6 @@ def run(self, scenario: Scenario) -> None: key=reverse_id, capacity=link.capacity, cost=link.cost, - latency=link.latency, **link.attrs, ) diff --git a/notebooks/lib_examples.ipynb b/notebooks/lib_examples.ipynb index 3e419eb..24cb86c 100644 --- a/notebooks/lib_examples.ipynb +++ b/notebooks/lib_examples.ipynb @@ -108,12 +108,12 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", - "Demonstrates traffic engineering by placing two bidirectional demands on a network.\n", + "Demonstrates traffic engineering by placing two demands on a network.\n", "\n", "Graph topology (metrics/capacities):\n", "\n", diff --git a/tests/scenarios/scenario_1.yaml b/tests/scenarios/scenario_1.yaml index aa0cad4..5abe225 100644 --- a/tests/scenarios/scenario_1.yaml +++ b/tests/scenarios/scenario_1.yaml @@ -20,86 +20,86 @@ network: # West -> Middle - source: SEA target: DEN - capacity: 200 - latency: 6846 - cost: 6846 - attrs: - distance_km: 1369.13 + link_params: + capacity: 200 + cost: 6846 + attrs: + distance_km: 1369.13 - source: SFO target: DEN - capacity: 200 - latency: 7754 - cost: 7754 - attrs: - distance_km: 1550.77 + link_params: + capacity: 200 + cost: 7754 + attrs: + distance_km: 1550.77 - source: SEA target: DFW - capacity: 200 - latency: 9600 - cost: 9600 - attrs: - distance_km: 1920 + link_params: + capacity: 200 + cost: 9600 + attrs: + distance_km: 1920 - source: SFO target: DFW - capacity: 200 - latency: 10000 - cost: 10000 - attrs: - distance_km: 2000 + link_params: + capacity: 200 + cost: 10000 + attrs: + distance_km: 2000 # Middle <-> Middle (two parallel links to represent redundancy) - source: DEN target: DFW - capacity: 400 - latency: 7102 - cost: 7102 - attrs: - distance_km: 1420.28 + link_params: + capacity: 400 + cost: 7102 + attrs: + distance_km: 1420.28 - source: DEN target: DFW - capacity: 400 - latency: 7102 - cost: 7102 - attrs: - distance_km: 1420.28 + link_params: + capacity: 400 + cost: 7102 + attrs: + distance_km: 1420.28 # Middle -> East - source: DEN target: JFK - capacity: 200 - latency: 7500 - cost: 7500 - attrs: - distance_km: 1500 + link_params: + capacity: 200 + cost: 7500 + attrs: + distance_km: 1500 - source: DFW target: DCA - capacity: 200 - latency: 8000 - cost: 8000 - attrs: - distance_km: 1600 + link_params: + capacity: 200 + cost: 8000 + attrs: + distance_km: 1600 - source: DFW target: JFK - capacity: 200 - latency: 9500 - cost: 9500 - attrs: - distance_km: 1900 + link_params: + capacity: 200 + cost: 9500 + attrs: + distance_km: 1900 # East <-> East - source: JFK target: DCA - capacity: 100 - latency: 1714 - cost: 1714 - attrs: - distance_km: 342.69 + link_params: + capacity: 100 + cost: 1714 + attrs: + distance_km: 342.69 failure_policy: name: "anySingleLink" diff --git a/tests/scenarios/scenario_2.yaml b/tests/scenarios/scenario_2.yaml new file mode 100644 index 0000000..9aad30c --- /dev/null +++ b/tests/scenarios/scenario_2.yaml @@ -0,0 +1,204 @@ +# Hierarchical DSL describing sub-topologies and multi-node expansions. +# +# Paths and Scopes: +# - Within a blueprint, a leading '/' (e.g., '/leaf') means "blueprint local root". +# - In the main network definition, a leading '/' (e.g., '/SFO') means "global root". +# - Omitting '/' means a relative path within the current scope (blueprint or network). +# +# Adjacency Patterns: +# - "mesh" cross-connects all nodes from source to all nodes from target, skipping self-loops. +# - "one_to_one" pairs up source[i] with target[i], also skipping self-loops. + +blueprints: + # A "blueprint" describes a reusable fragment of topology (e.g., a pattern). + # It can be referenced in other blueprints or in the main 'network'. + clos_2tier: + groups: + leaf: + node_count: 4 + name_template: leaf-{node_num} + spine: + node_count: 4 + name_template: spine-{node_num} + + adjacency: + - source: /leaf + target: /spine + pattern: mesh + link_params: + capacity: 100 + cost: 1000 + + # Another blueprint referencing 'clos_2tier' as a sub-topology. + city_cloud: + groups: + clos_instance: + # Uses the 'clos_2tier' blueprint but overrides some parameters. + use_blueprint: clos_2tier + parameters: + # Example override: more spine nodes and a new naming convention. + spine.node_count: 6 + spine.name_template: "myspine-{node_num}" + + edge_nodes: + node_count: 4 + name_template: edge-{node_num} + + adjacency: + - source: /clos_instance/leaf + target: /edge_nodes + pattern: mesh + link_params: + capacity: 100 + cost: 1000 + + # A minimal blueprint representing a single node. + single_node: + groups: + single: + node_count: 1 + name_template: single-{node_num} + +# -- Main network definition -- +network: + name: "6-node-l3-us-backbone" + version: 1.1 + + groups: + # 1) 'SEA' references the 'city_cloud' blueprint. This creates subgroups + # "SEA/clos_instance" and "SEA/edge_nodes" in the global scope. + SEA: + use_blueprint: city_cloud + coords: [47.6062, -122.3321] + + # 2) 'SFO' references 'single_node' (one-node blueprint). + SFO: + use_blueprint: single_node + coords: [37.7749, -122.4194] + + adjacency: + # Each adjacency definition uses "mesh" in this scenario. Self-loops are automatically skipped. + - source: /SFO + target: /DEN + pattern: mesh + link_params: + capacity: 100 + cost: 7754 + attrs: + distance_km: 1550.77 + + - source: /SFO + target: /DFW + pattern: mesh + link_params: + capacity: 200 + cost: 10000 + attrs: + distance_km: 2000 + + - source: /SEA/edge_nodes + target: /DEN + pattern: mesh + link_params: + capacity: 100 + cost: 6846 + attrs: + distance_km: 1369.13 + + - source: /SEA/edge_nodes + target: /DFW + pattern: mesh + link_params: + capacity: 100 + cost: 9600 + attrs: + distance_km: 1920 + + # Standalone nodes + nodes: + DEN: + coords: [39.7392, -104.9903] + DFW: + coords: [32.8998, -97.0403] + JFK: + coords: [40.641766, -73.780968] + DCA: + coords: [38.907192, -77.036871] + + # Additional direct links + # Note that each link references existing nodes (e.g., DEN, DFW). + # If multiple links exist between the same two nodes, they have unique IDs + # generated by the code, but share the same source/target. + links: + - source: DEN + target: DFW + link_params: + capacity: 400 + cost: 7102 + attrs: + distance_km: 1420.28 + - source: DEN + target: DFW + link_params: + capacity: 400 + cost: 7102 + attrs: + distance_km: 1420.28 + - source: DEN + target: JFK + link_params: + capacity: 200 + cost: 7500 + attrs: + distance_km: 1500 + - source: DFW + target: DCA + link_params: + capacity: 200 + cost: 8000 + attrs: + distance_km: 1600 + - source: DFW + target: JFK + link_params: + capacity: 200 + cost: 9500 + attrs: + distance_km: 1900 + - source: JFK + target: DCA + link_params: + capacity: 100 + cost: 1714 + attrs: + distance_km: 342.69 + +failure_policy: + name: "anySingleLink" + description: "Evaluate traffic routing under any single link failure." + rules: + - conditions: + - attr: "type" + operator: "==" + value: "link" + logic: "and" + rule_type: "choice" + count: 1 + +traffic_demands: + - source: SEA + target: JFK + demand: 50 + - source: SFO + target: DCA + demand: 50 + - source: SEA + target: DCA + demand: 50 + - source: SFO + target: JFK + demand: 50 + +workflow: + - step_type: BuildGraph + name: build_graph diff --git a/tests/test_blueprints.py b/tests/test_blueprints.py new file mode 100644 index 0000000..53915be --- /dev/null +++ b/tests/test_blueprints.py @@ -0,0 +1,342 @@ +import pytest + +from ngraph.blueprints import expand_network_dsl +from ngraph.network import Node + + +def test_minimal_no_blueprints(): + """ + Tests a minimal DSL with no blueprints, no adjacency, and a single direct node/link. + Ensures the DSL creates expected nodes/links in the simplest scenario. + """ + scenario_data = { + "network": { + "name": "simple_network", + "nodes": {"A": {"test_attr": 123}, "B": {}}, + "links": [{"source": "A", "target": "B", "link_params": {"capacity": 10}}], + } + } + + net = expand_network_dsl(scenario_data) + + assert net.attrs["name"] == "simple_network" + assert len(net.nodes) == 2 + assert len(net.links) == 1 + + assert "A" in net.nodes + assert net.nodes["A"].attrs["test_attr"] == 123 + assert "B" in net.nodes + + # Grab the first (and only) Link object + link = next(iter(net.links.values())) + assert link.source == "A" + assert link.target == "B" + assert link.capacity == 10 + + +def test_simple_blueprint(): + """ + Tests a scenario with one blueprint used by one group. + Verifies that blueprint-based groups expand properly and adjacency is handled. + """ + scenario_data = { + "blueprints": { + "clos_1tier": { + "groups": {"leaf": {"node_count": 2}}, + "adjacency": [ + { + "source": "/leaf", + "target": "/leaf", + "pattern": "mesh", + "link_params": {"cost": 10}, + } + ], + } + }, + "network": { + "name": "test_simple_blueprint", + "groups": {"R1": {"use_blueprint": "clos_1tier"}}, + }, + } + + net = expand_network_dsl(scenario_data) + + # Expect 2 leaf nodes under path "R1/leaf" + assert len(net.nodes) == 2 + assert "R1/leaf/leaf-1" in net.nodes + assert "R1/leaf/leaf-2" in net.nodes + + # The adjacency is "leaf <-> leaf" mesh => leaf-1 <-> leaf-2 + # mesh deduplicates reversed links => single link + assert len(net.links) == 1 + only_link = next(iter(net.links.values())) + assert only_link.source.endswith("leaf-1") + assert only_link.target.endswith("leaf-2") + assert only_link.cost == 10 + + +def test_blueprint_parameters(): + """ + Tests parameter overrides in a blueprint scenario. + Ensures that user-provided overrides (e.g., node_count) are applied. + """ + scenario_data = { + "blueprints": { + "multi_layer": { + "groups": { + "layerA": {"node_count": 2, "name_template": "layerA-{node_num}"}, + "layerB": {"node_count": 2, "name_template": "layerB-{node_num}"}, + }, + "adjacency": [], + } + }, + "network": { + "groups": { + "MAIN": { + "use_blueprint": "multi_layer", + "parameters": { + "layerA.node_count": 3, + "layerA.name_template": "overrideA-{node_num}", + }, + } + } + }, + } + + net = expand_network_dsl(scenario_data) + + # layerA gets overridden to node_count=3 + # layerB remains node_count=2 => total 5 nodes + assert len(net.nodes) == 5 + + # Confirm naming override + overrideA_nodes = [n for n in net.nodes if "overrideA-" in n] + assert len(overrideA_nodes) == 3, "Expected 3 overrideA- nodes" + layerB_nodes = [n for n in net.nodes if "layerB-" in n] + assert len(layerB_nodes) == 2, "Expected 2 layerB- nodes" + + +def test_direct_nodes_and_links_alongside_blueprints(): + """ + Tests mixing a blueprint with direct nodes and links. + Verifies direct nodes are added, links to blueprint-created nodes are valid. + """ + scenario_data = { + "blueprints": { + "single_group": { + "groups": { + "mygroup": {"node_count": 2, "name_template": "g-{node_num}"} + }, + "adjacency": [], + } + }, + "network": { + "groups": {"BP1": {"use_blueprint": "single_group"}}, + "nodes": {"ExtraNode": {"tag": "extra"}}, + "links": [ + { + "source": "BP1/mygroup/g-1", + "target": "ExtraNode", + "link_params": {"capacity": 50}, + } + ], + }, + } + + net = expand_network_dsl(scenario_data) + + # 2 blueprint nodes + 1 direct node => 3 total + assert len(net.nodes) == 3 + assert "BP1/mygroup/g-1" in net.nodes + assert "BP1/mygroup/g-2" in net.nodes + assert "ExtraNode" in net.nodes + + # One link connecting blueprint node to direct node + assert len(net.links) == 1 + link = next(iter(net.links.values())) + assert link.source == "BP1/mygroup/g-1" + assert link.target == "ExtraNode" + assert link.capacity == 50 + + +def test_adjacency_one_to_one(): + """ + Tests a one_to_one adjacency among two groups of equal size (2:2). + We expect each source node pairs with one target node => total 2 links. + """ + scenario_data = { + "network": { + "groups": {"GroupA": {"node_count": 2}, "GroupB": {"node_count": 2}}, + "adjacency": [ + { + "source": "/GroupA", + "target": "/GroupB", + "pattern": "one_to_one", + "link_params": {"capacity": 99}, + } + ], + } + } + + net = expand_network_dsl(scenario_data) + + # 4 total nodes + assert len(net.nodes) == 4 + # one_to_one => 2 total links => (GroupA/GroupA-1->GroupB/GroupB-1, GroupA/GroupA-2->GroupB/GroupB-2) + assert len(net.links) == 2 + link_names = {(l.source, l.target) for l in net.links.values()} + expected_links = { + ("GroupA/GroupA-1", "GroupB/GroupB-1"), + ("GroupA/GroupA-2", "GroupB/GroupB-2"), + } + assert link_names == expected_links + for l in net.links.values(): + assert l.capacity == 99 + + +def test_adjacency_one_to_one_wrap(): + """ + Tests a one_to_one adjacency among groups of size 4 and 2. + Because 4%2 == 0, we can wrap around the smaller side => total 4 links. + """ + scenario_data = { + "network": { + "groups": {"Big": {"node_count": 4}, "Small": {"node_count": 2}}, + "adjacency": [ + { + "source": "/Big", + "target": "/Small", + "pattern": "one_to_one", + "link_params": {"cost": 555}, + } + ], + } + } + + net = expand_network_dsl(scenario_data) + + # 6 total nodes => Big(4) + Small(2) + assert len(net.nodes) == 6 + # Wrap => Big-1->Small-1, Big-2->Small-2, Big-3->Small-1, Big-4->Small-2 => 4 links + assert len(net.links) == 4 + link_pairs = {(l.source, l.target) for l in net.links.values()} + expected = { + ("Big/Big-1", "Small/Small-1"), + ("Big/Big-2", "Small/Small-2"), + ("Big/Big-3", "Small/Small-1"), + ("Big/Big-4", "Small/Small-2"), + } + assert link_pairs == expected + for l in net.links.values(): + assert l.cost == 555 + + +def test_adjacency_mesh(): + """ + Tests a mesh adjacency among two groups, ensuring all nodes from each group are interconnected. + """ + scenario_data = { + "network": { + "groups": { + "Left": {"node_count": 2}, # => Left/Left-1, Left/Left-2 + "Right": {"node_count": 2}, # => Right/Right-1, Right/Right-2 + }, + "adjacency": [ + { + "source": "/Left", + "target": "/Right", + "pattern": "mesh", + "link_params": {"cost": 5}, + } + ], + } + } + + net = expand_network_dsl(scenario_data) + + # 4 total nodes + assert len(net.nodes) == 4 + # mesh => (Left-1->Right-1), (Left-1->Right-2), (Left-2->Right-1), (Left-2->Right-2) => 4 unique links + assert len(net.links) == 4 + for link in net.links.values(): + assert link.cost == 5 + + +def test_fallback_prefix_behavior(): + """ + Tests the fallback prefix logic. If no normal match, we do partial or 'path-' fallback. + In this scenario, we have 1 node => "FallbackGroup/FallbackGroup-1". + The adjacency tries a one_to_one pattern => if we want to skip self-loops in all patterns, + the result is 0 links. + """ + scenario_data = { + "network": { + "groups": { + "FallbackGroup": { + "node_count": 1, + "name_template": "FallbackGroup-{node_num}", + } + }, + "adjacency": [ + { + "source": "FallbackGroup", + "target": "FallbackGroup", + "pattern": "one_to_one", + } + ], + } + } + + net = expand_network_dsl(scenario_data) + + # 1 node => name "FallbackGroup/FallbackGroup-1" + assert len(net.nodes) == 1 + assert "FallbackGroup/FallbackGroup-1" in net.nodes + + # If we skip self-loops for "one_to_one", we get 0 links. + # If your code doesn't skip self-loops in "one_to_one", you'll get 1 link. + # Adjust as needed: + assert len(net.links) == 0 + + +def test_direct_link_unknown_node_raises(): + """ + Ensures that referencing unknown nodes in a direct link raises an error. + """ + scenario_data = { + "network": { + "nodes": {"KnownNode": {}}, + "links": [{"source": "KnownNode", "target": "UnknownNode"}], + } + } + + with pytest.raises(ValueError) as excinfo: + expand_network_dsl(scenario_data) + + assert "Link references unknown node(s): KnownNode, UnknownNode" in str( + excinfo.value + ) + + +def test_existing_node_preserves_attrs(): + """ + Tests that if a node is already present in the network, direct node definitions don't overwrite + its existing attributes except for 'type' which is ensured by default. + """ + scenario_data = { + "network": { + "groups": {"Foo": {"node_count": 1, "name_template": "X-{node_num}"}}, + "nodes": {"Foo/X-1": {"myattr": 123}}, + } + } + + net = expand_network_dsl(scenario_data) + + # There's only 1 node => "Foo/X-1" + assert len(net.nodes) == 1 + node_obj = net.nodes["Foo/X-1"] + + # The code sets "type"="node" if not present but doesn't merge other attributes. + # So "myattr" won't appear, because the node was created from groups. + assert "myattr" not in node_obj.attrs + assert node_obj.attrs["type"] == "node" diff --git a/tests/test_blueprints_helpers.py b/tests/test_blueprints_helpers.py new file mode 100644 index 0000000..92ae16f --- /dev/null +++ b/tests/test_blueprints_helpers.py @@ -0,0 +1,379 @@ +import pytest +from ngraph.network import Network, Node, Link + +from ngraph.blueprints import ( + DSLExpansionContext, + Blueprint, + _apply_parameters, + _join_paths, + _find_nodes_by_path, + _create_link, + _expand_adjacency_pattern, + _process_direct_nodes_and_links, + _expand_blueprint_adjacency, + _expand_adjacency, + _expand_group, +) + + +def test_join_paths(): + """ + Tests _join_paths for correct handling of leading slash, parent/child combinations, etc. + """ + # No parent path, no slash + assert _join_paths("", "SFO") == "SFO" + # No parent path, child with leading slash => "SFO" + assert _join_paths("", "/SFO") == "SFO" + # Parent path plus child => "SEA/leaf" + assert _join_paths("SEA", "leaf") == "SEA/leaf" + # Parent path plus leading slash => "SEA/leaf" + assert _join_paths("SEA", "/leaf") == "SEA/leaf" + + +def test_find_nodes_by_path(): + """ + Tests _find_nodes_by_path for exact matches, slash-based prefix matches, and fallback prefix pattern. + """ + net = Network() + # Add some nodes + net.add_node(Node("SEA/spine/myspine-1")) + net.add_node(Node("SEA/leaf/leaf-1")) + net.add_node(Node("SEA-other")) + net.add_node(Node("SFO")) + + # 1) Exact match => "SFO" + nodes = _find_nodes_by_path(net, "SFO") + assert len(nodes) == 1 + assert nodes[0].name == "SFO" + + # 2) Slash prefix => "SEA/spine" matches "SEA/spine/myspine-1" + nodes = _find_nodes_by_path(net, "SEA/spine") + assert len(nodes) == 1 + assert nodes[0].name == "SEA/spine/myspine-1" + + # 3) Fallback: "SEA-other" won't be found by slash prefix "SEA/other", but if we search "SEA-other", + # we do an exact match or a fallback "SEA-other" => here it's exact, so we get 1 node + nodes = _find_nodes_by_path(net, "SEA-other") + assert len(nodes) == 1 + assert nodes[0].name == "SEA-other" + + # 4) If we search just "SEA", we match "SEA/spine/myspine-1" and "SEA/leaf/leaf-1" by slash prefix, + # but "SEA-other" won't appear because fallback never triggers (we already found slash matches). + nodes = _find_nodes_by_path(net, "SEA") + found = set(n.name for n in nodes) + assert found == { + "SEA/spine/myspine-1", + "SEA/leaf/leaf-1", + } + + +def test_apply_parameters(): + """ + Tests _apply_parameters to ensure user-provided overrides get applied to the correct subgroup fields. + """ + original_def = { + "node_count": 4, + "name_template": "spine-{node_num}", + "other_attr": True, + } + params = { + # Overwrite node_count for 'spine' + "spine.node_count": 6, + # Overwrite name_template for 'spine' + "spine.name_template": "myspine-{node_num}", + # Overwrite a non-existent param => ignored + "leaf.node_count": 10, + } + updated = _apply_parameters("spine", original_def, params) + assert updated["node_count"] == 6 + assert updated["name_template"] == "myspine-{node_num}" + # Check that we preserved "other_attr" + assert updated["other_attr"] is True + # Check that there's no spurious new key + assert len(updated) == 3, f"Unexpected keys: {updated.keys()}" + + +def test_create_link(): + """ + Tests _create_link to verify creation and insertion into a Network. + """ + net = Network() + net.add_node(Node("A")) + net.add_node(Node("B")) + + _create_link(net, "A", "B", {"capacity": 50, "cost": 5, "attrs": {"color": "red"}}) + + assert len(net.links) == 1 + link_obj = list(net.links.values())[0] + assert link_obj.source == "A" + assert link_obj.target == "B" + assert link_obj.capacity == 50 + assert link_obj.cost == 5 + assert link_obj.attrs["color"] == "red" + + +def test_expand_adjacency_pattern_one_to_one(): + """ + Tests _expand_adjacency_pattern in 'one_to_one' mode for the simplest matching case (2:2). + Should produce pairs: (S1->T1), (S2->T2). + """ + ctx_net = Network() + ctx_net.add_node(Node("S1")) + ctx_net.add_node(Node("S2")) + ctx_net.add_node(Node("T1")) + ctx_net.add_node(Node("T2")) + + ctx = DSLExpansionContext(blueprints={}, network=ctx_net) + + _expand_adjacency_pattern(ctx, "S", "T", "one_to_one", {"capacity": 10}) + # We expect 2 links: S1->T1, S2->T2 + assert len(ctx_net.links) == 2 + # Confirm the pairs + pairs = {(l.source, l.target) for l in ctx_net.links.values()} + assert pairs == {("S1", "T1"), ("S2", "T2")} + + # Also confirm the capacity + for l in ctx_net.links.values(): + assert l.capacity == 10 + + +def test_expand_adjacency_pattern_one_to_one_wrap(): + """ + Tests 'one_to_one' with wrapping case: e.g., 4 vs 2. (both sides a multiple) + => S1->T1, S2->T2, S3->T1, S4->T2 => total 4 links + """ + ctx_net = Network() + # 4 source nodes + ctx_net.add_node(Node("S1")) + ctx_net.add_node(Node("S2")) + ctx_net.add_node(Node("S3")) + ctx_net.add_node(Node("S4")) + # 2 target nodes + ctx_net.add_node(Node("T1")) + ctx_net.add_node(Node("T2")) + + ctx = DSLExpansionContext({}, ctx_net) + + _expand_adjacency_pattern(ctx, "S", "T", "one_to_one", {"cost": 99}) + # Expect 4 total links + assert len(ctx_net.links) == 4 + # Check the actual pairs + pairs = {(l.source, l.target) for l in ctx_net.links.values()} + expected = { + ("S1", "T1"), + ("S2", "T2"), + ("S3", "T1"), + ("S4", "T2"), + } + assert pairs == expected + for l in ctx_net.links.values(): + assert l.cost == 99 + + +def test_expand_adjacency_pattern_one_to_one_mismatch(): + """ + Tests 'one_to_one' with a mismatch (3 vs 2) => raises a ValueError + since 3 % 2 != 0 + """ + ctx_net = Network() + # 3 sources, 2 targets + ctx_net.add_node(Node("S1")) + ctx_net.add_node(Node("S2")) + ctx_net.add_node(Node("S3")) + ctx_net.add_node(Node("T1")) + ctx_net.add_node(Node("T2")) + + ctx = DSLExpansionContext({}, ctx_net) + + with pytest.raises(ValueError) as exc: + _expand_adjacency_pattern(ctx, "S", "T", "one_to_one", {}) + assert "requires either equal node counts or a valid wrap-around" in str(exc.value) + + +def test_expand_adjacency_pattern_mesh(): + """ + Tests _expand_adjacency_pattern in 'mesh' mode: all-to-all links among matched nodes, + with dedup so we don't double-link reversed pairs. + """ + ctx_net = Network() + ctx_net.add_node(Node("X1")) + ctx_net.add_node(Node("X2")) + ctx_net.add_node(Node("Y1")) + ctx_net.add_node(Node("Y2")) + + ctx = DSLExpansionContext({}, ctx_net) + + # mesh => X1, X2 => Y1, Y2 => 4 links total + _expand_adjacency_pattern(ctx, "X", "Y", "mesh", {"capacity": 99}) + assert len(ctx_net.links) == 4 + for link in ctx_net.links.values(): + assert link.capacity == 99 + + +def test_process_direct_nodes_and_links(): + """ + Tests _process_direct_nodes_and_links to ensure direct node creation + and direct link creation works, verifying unknown nodes raise errors. + """ + net = Network() + net.add_node(Node("Existing")) + + network_data = { + "nodes": { + "New1": {"foo": "bar"}, + "Existing": { + "override": "ignored" + }, # This won't be merged since node exists + }, + "links": [ + { + "source": "New1", + "target": "NoExist", # Should fail + "link_params": {"capacity": 5}, + } + ], + } + with pytest.raises(ValueError) as excinfo: + _process_direct_nodes_and_links(net, network_data) + # The error should mention "unknown node(s): New1, NoExist" + assert "Link references unknown node(s): New1, NoExist" in str(excinfo.value) + + # Confirm that partial link creation was blocked + assert len(net.links) == 0 + + # Meanwhile, "New1" was created + assert "New1" in net.nodes + assert net.nodes["New1"].attrs["foo"] == "bar" + # "Existing" was not overwritten + assert "override" not in net.nodes["Existing"].attrs + + +def test_expand_blueprint_adjacency(): + """ + Tests _expand_blueprint_adjacency: verifying that relative paths inside a blueprint are joined + with parent_path, then expanded as normal adjacency. + """ + ctx_net = Network() + ctx_net.add_node(Node("Parent/leaf-1")) + ctx_net.add_node(Node("Parent/leaf-2")) + ctx_net.add_node(Node("Parent/spine-1")) + + ctx = DSLExpansionContext({}, ctx_net) + + adj_def = { + "source": "/leaf-1", + "target": "/spine-1", + "pattern": "mesh", + "link_params": {"cost": 999}, + } + # parent_path => "Parent" + _expand_blueprint_adjacency(ctx, adj_def, "Parent") + + # Only "Parent/leaf-1" matches the source path => single source node, + # "Parent/spine-1" => single target node => 1 link + assert len(ctx_net.links) == 1 + link = next(iter(ctx_net.links.values())) + assert link.source == "Parent/leaf-1" + assert link.target == "Parent/spine-1" + assert link.cost == 999 + + +def test_expand_adjacency(): + """ + Tests _expand_adjacency for a top-level adjacency definition (non-blueprint), + verifying that leading '/' is stripped and used to find nodes in the global context. + """ + ctx_net = Network() + ctx_net.add_node(Node("Global/A1")) + ctx_net.add_node(Node("Global/B1")) + + ctx = DSLExpansionContext({}, ctx_net) + + # adjacency => "one_to_one" with /Global/A1 -> /Global/B1 + adj_def = { + "source": "/Global/A1", + "target": "/Global/B1", + "pattern": "one_to_one", + "link_params": {"capacity": 10}, + } + _expand_adjacency(ctx, adj_def) + + # single pairing => A1 -> B1 + assert len(ctx_net.links) == 1 + link = next(iter(ctx_net.links.values())) + assert link.source == "Global/A1" + assert link.target == "Global/B1" + assert link.capacity == 10 + + +def test_expand_group_direct(): + """ + Tests _expand_group for a direct node group (no use_blueprint), ensuring node_count and name_template. + """ + ctx_net = Network() + ctx = DSLExpansionContext({}, ctx_net) + + group_def = { + "node_count": 3, + "name_template": "myNode-{node_num}", + "coords": [1, 2], + } + _expand_group(ctx, parent_path="", group_name="TestGroup", group_def=group_def) + + # Expect 3 nodes => "TestGroup/myNode-1" ... "TestGroup/myNode-3" + assert len(ctx_net.nodes) == 3 + for i in range(1, 4): + name = f"TestGroup/myNode-{i}" + assert name in ctx_net.nodes + node = ctx_net.nodes[name] + assert node.attrs["coords"] == [1, 2] + assert node.attrs["type"] == "node" + + +def test_expand_group_blueprint(): + """ + Tests _expand_group referencing a blueprint (basic subgroups + adjacency). + We'll create a blueprint 'bp1' with one subgroup and adjacency, then reference it from group 'Main'. + """ + bp = Blueprint( + name="bp1", + groups={ + "leaf": {"node_count": 2}, + }, + adjacency=[ + { + "source": "/leaf", + "target": "/leaf", + "pattern": "mesh", + } + ], + ) + ctx_net = Network() + ctx = DSLExpansionContext(blueprints={"bp1": bp}, network=ctx_net) + + # group_def referencing the blueprint + group_def = { + "use_blueprint": "bp1", + "coords": [10, 20], + } + _expand_group( + ctx, + parent_path="", + group_name="Main", + group_def=group_def, + blueprint_expansion=False, + ) + + # This expands 2 leaf nodes => "Main/leaf/leaf-1", "Main/leaf/leaf-2" + # plus adjacency => single link (leaf-1 <-> leaf-2) due to mesh + dedup + assert len(ctx_net.nodes) == 2 + assert "Main/leaf/leaf-1" in ctx_net.nodes + assert "Main/leaf/leaf-2" in ctx_net.nodes + # coords should be carried over + assert ctx_net.nodes["Main/leaf/leaf-1"].attrs["coords"] == [10, 20] + + # adjacency => mesh => 1 unique link + assert len(ctx_net.links) == 1 + link = next(iter(ctx_net.links.values())) + sources_targets = {link.source, link.target} + assert sources_targets == {"Main/leaf/leaf-1", "Main/leaf/leaf-2"} diff --git a/tests/test_network.py b/tests/test_network.py index 7532a7b..b2f242f 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -46,13 +46,12 @@ def test_node_creation_custom_attrs(): def test_link_defaults_and_id_generation(): """ - A Link without custom parameters should default capacity/latency/cost to 1.0, + A Link without custom parameters should default capacity/cost to 1.0, have an empty attrs dict, and generate a unique ID like 'A-B-'. """ link = Link("A", "B") assert link.capacity == 1.0 - assert link.latency == 1.0 assert link.cost == 1.0 assert link.attrs == {} @@ -63,16 +62,15 @@ def test_link_defaults_and_id_generation(): def test_link_custom_values(): """ - A Link can be created with custom capacity/latency/cost/attrs, + A Link can be created with custom capacity/cost/attrs, and the ID is generated automatically. """ custom_attrs = {"color": "red"} - link = Link("X", "Y", capacity=2.0, latency=3.0, cost=4.0, attrs=custom_attrs) + link = Link("X", "Y", capacity=2.0, cost=4.0, attrs=custom_attrs) assert link.source == "X" assert link.target == "Y" assert link.capacity == 2.0 - assert link.latency == 3.0 assert link.cost == 4.0 assert link.attrs == custom_attrs assert link.id.startswith("X-Y-") diff --git a/tests/test_scenario.py b/tests/test_scenario.py index 621b2dd..3f3f238 100644 --- a/tests/test_scenario.py +++ b/tests/test_scenario.py @@ -35,7 +35,6 @@ def run(self, scenario: Scenario) -> None: Perform a dummy operation for testing. Store something in scenario.results using the step name as a key. """ - # We can use self.name as the "step_name" scenario.results.put(self.name, "ran", True) @@ -74,16 +73,17 @@ def valid_scenario_yaml() -> str: links: - source: NodeA target: NodeB - capacity: 10 - latency: 2 - cost: 5 - attrs: {some_attr: some_value} + link_params: + capacity: 10 + cost: 5 + attrs: + some_attr: some_value - source: NodeB target: NodeC - capacity: 20 - latency: 3 - cost: 4 - attrs: {} + link_params: + capacity: 20 + cost: 4 + attrs: {} failure_policy: name: "multi_rule_example" description: "Testing multi-rule approach." @@ -132,7 +132,8 @@ def missing_step_type_yaml() -> str: links: - source: NodeA target: NodeB - capacity: 1 + link_params: + capacity: 1 failure_policy: rules: [] traffic_demands: @@ -159,7 +160,8 @@ def unrecognized_step_type_yaml() -> str: links: - source: NodeA target: NodeB - capacity: 1 + link_params: + capacity: 1 failure_policy: rules: [] traffic_demands: @@ -187,7 +189,8 @@ def extra_param_yaml() -> str: links: - source: NodeA target: NodeB - capacity: 1 + link_params: + capacity: 1 traffic_demands: [] failure_policy: rules: [] @@ -213,7 +216,7 @@ def test_scenario_from_yaml_valid(valid_scenario_yaml: str) -> None: # Check network assert isinstance(scenario.network, Network) - assert len(scenario.network.nodes) == 3 # We defined NodeA, NodeB, NodeC + assert len(scenario.network.nodes) == 3 # NodeA, NodeB, NodeC assert len(scenario.network.links) == 2 # NodeA->NodeB, NodeB->NodeC node_names = [node.name for node in scenario.network.nodes.values()] @@ -230,21 +233,18 @@ def test_scenario_from_yaml_valid(valid_scenario_yaml: str) -> None: assert link_ab is not None, "Link from NodeA to NodeB was not found." assert link_ab.target == "NodeB" assert link_ab.capacity == 10 - assert link_ab.latency == 2 assert link_ab.cost == 5 assert link_ab.attrs.get("some_attr") == "some_value" assert link_bc is not None, "Link from NodeB to NodeC was not found." assert link_bc.target == "NodeC" assert link_bc.capacity == 20 - assert link_bc.latency == 3 assert link_bc.cost == 4 # Check failure policy assert isinstance(scenario.failure_policy, FailurePolicy) assert len(scenario.failure_policy.rules) == 2, "Expected 2 rules in the policy." - # Check that the leftover fields in failure_policy (e.g. "name", "description") - # went into policy.attrs + # leftover fields (e.g. name, description) in policy.attrs assert scenario.failure_policy.attrs.get("name") == "multi_rule_example" assert ( scenario.failure_policy.attrs.get("description") diff --git a/tests/workflow/test_build_graph.py b/tests/workflow/test_build_graph.py index 71d92e4..c693515 100644 --- a/tests/workflow/test_build_graph.py +++ b/tests/workflow/test_build_graph.py @@ -19,13 +19,12 @@ class MockLink: A simple mock Link to simulate scenario.network.links[link_id]. """ - def __init__(self, link_id, source, target, capacity, cost, latency, attrs=None): + def __init__(self, link_id, source, target, capacity, cost, attrs=None): self.id = link_id self.source = source self.target = target self.capacity = capacity self.cost = cost - self.latency = latency self.attrs = attrs or {} @@ -49,7 +48,6 @@ def mock_scenario(): target="B", capacity=100, cost=5, - latency=10, attrs={"fiber": True}, ), "L2": MockLink( @@ -58,7 +56,6 @@ def mock_scenario(): target="A", capacity=50, cost=2, - latency=5, attrs={"copper": True}, ), } @@ -112,7 +109,6 @@ def test_build_graph_stores_multidigraph_in_results(mock_scenario): assert edge_data is not None, "Forward edge 'L1' should exist from A to B." assert edge_data["capacity"] == 100 assert edge_data["cost"] == 5 - assert edge_data["latency"] == 10 assert "fiber" in edge_data # Check reverse edge from link 'L1' @@ -137,4 +133,3 @@ def test_build_graph_stores_multidigraph_in_results(mock_scenario): assert ( rev_edge_data_l2["capacity"] == 50 ), "Reverse edge should share the same capacity." - assert rev_edge_data_l2["latency"] == 5 From 9220b7cd0d7618548e1a1d523fd777b07e00ff40 Mon Sep 17 00:00:00 2001 From: Andrey Golovanov Date: Sun, 23 Feb 2025 23:24:09 +0000 Subject: [PATCH 2/3] integrated topology DSL with scenarios --- ngraph/blueprints.py | 28 ++++---- ngraph/scenario.py | 101 +++++------------------------ ngraph/workflow/build_graph.py | 5 +- tests/scenarios/test_scenario_1.py | 6 +- tests/scenarios/test_scenario_2.py | 100 ++++++++++++++++++++++++++++ tests/test_blueprints_helpers.py | 51 ++++++++++----- 6 files changed, 172 insertions(+), 119 deletions(-) create mode 100644 tests/scenarios/test_scenario_2.py diff --git a/ngraph/blueprints.py b/ngraph/blueprints.py index 80afab5..26c323a 100644 --- a/ngraph/blueprints.py +++ b/ngraph/blueprints.py @@ -51,8 +51,9 @@ def expand_network_dsl(data: Dict[str, Any]) -> Network: 3) Expand 'network["groups"]': - If a group references a blueprint, incorporate that blueprint's subgroups. - Otherwise, directly create nodes (e.g., node_count). - 4) Expand adjacency definitions in 'network["adjacency"]'. - 5) Process any direct node/link definitions. + 4) Process any direct node definitions. + 5) Expand adjacency definitions in 'network["adjacency"]'. + 6) Process any direct link definitions. Args: data: The YAML-parsed dictionary containing optional "blueprints" + "network". @@ -90,12 +91,15 @@ def expand_network_dsl(data: Dict[str, Any]) -> Network: blueprint_expansion=False, ) - # 4) Expand adjacency definitions + # 4) Process direct node definitions + _process_direct_nodes(ctx.network, network_data) + + # 5) Expand adjacency definitions for adj_def in network_data.get("adjacency", []): _expand_adjacency(ctx, adj_def) - # 5) Process direct node/link definitions - _process_direct_nodes_and_links(ctx.network, network_data) + # 6) Process direct link definitions + _process_direct_links(ctx.network, network_data) return net @@ -364,19 +368,19 @@ def _find_nodes_by_path(net: Network, path: str) -> List[Node]: return partial -def _process_direct_nodes_and_links(net: Network, network_data: Dict[str, Any]) -> None: - """ - Processes direct node definitions (network_data["nodes"]) and direct link definitions - (network_data["links"]). - """ - # Direct node definitions +def _process_direct_nodes(net: Network, network_data: Dict[str, Any]) -> None: + """Processes direct node definitions (network_data["nodes"]).""" for node_name, node_attrs in network_data.get("nodes", {}).items(): if node_name not in net.nodes: new_node = Node(name=node_name, attrs=node_attrs or {}) new_node.attrs.setdefault("type", "node") net.add_node(new_node) - # Direct link definitions + +def _process_direct_links(net: Network, network_data: Dict[str, Any]) -> None: + """ + Processes direct link definitions (network_data["links"]). + """ existing_node_names = set(net.nodes.keys()) for link_info in network_data.get("links", []): source = link_info["source"] diff --git a/ngraph/scenario.py b/ngraph/scenario.py index 7caffd9..b4484e0 100644 --- a/ngraph/scenario.py +++ b/ngraph/scenario.py @@ -4,11 +4,12 @@ from dataclasses import dataclass, field from typing import Any, Dict, List -from ngraph.network import Network, Node, Link +from ngraph.network import Network from ngraph.failure_policy import FailurePolicy, FailureRule, FailureCondition from ngraph.traffic_demand import TrafficDemand from ngraph.results import Results from ngraph.workflow.base import WorkflowStep, WORKFLOW_STEP_REGISTRY +from ngraph.blueprints import expand_network_dsl @dataclass(slots=True) @@ -17,7 +18,7 @@ class Scenario: Represents a complete scenario for building and executing network workflows. This scenario includes: - - A network (nodes and links). + - A network (nodes and links), constructed via blueprint expansion. - A failure policy (one or more rules). - A set of traffic demands. - A list of workflow steps to execute. @@ -55,19 +56,20 @@ def run(self) -> None: @classmethod def from_yaml(cls, yaml_str: str) -> Scenario: """ - Constructs a Scenario from a YAML string. + Constructs a Scenario from a YAML string, including blueprint expansion. Expected top-level YAML keys: - - network: Node/Link definitions (with optional name/version) - - failure_policy: Multi-rule policy - - traffic_demands: List of demands + - blueprints: Optional set of blueprint definitions + - network: Network DSL that references blueprints and/or direct nodes/links + - failure_policy: Multi-rule policy definition + - traffic_demands: List of demands (source, target, amount) - workflow: Steps to execute Args: yaml_str (str): The YAML string that defines the scenario. Returns: - Scenario: An initialized Scenario instance. + Scenario: An initialized Scenario instance with expanded network. Raises: ValueError: If the YAML is malformed or missing required sections. @@ -76,9 +78,9 @@ def from_yaml(cls, yaml_str: str) -> Scenario: if not isinstance(data, dict): raise ValueError("The provided YAML must map to a dictionary at top-level.") - # 1) Build the network - network_data = data.get("network", {}) - network = cls._build_network(network_data) + # 1) Build the network using blueprint expansion logic + # This handles both "blueprints" and "network" sections if present. + network = expand_network_dsl(data) # 2) Build the multi-rule failure policy fp_data = data.get("failure_policy", {}) @@ -99,73 +101,6 @@ def from_yaml(cls, yaml_str: str) -> Scenario: workflow=workflow_steps, ) - @staticmethod - def _build_network(network_data: Dict[str, Any]) -> Network: - """ - Constructs a Network from a dictionary containing optional 'name', - 'version', 'nodes', and 'links' (with nested link_params). - - Example structure for network_data: - name: "6-node-l3-us-backbone" - version: "1.0" - nodes: - SEA: { coords: [47.6062, -122.3321] } - SFO: { coords: [37.7749, -122.4194] } - links: - - source: SEA - target: SFO - link_params: - capacity: 200 - cost: 100 - attrs: - distance_km: 1500 - - Args: - network_data (Dict[str, Any]): Dictionary with optional keys - 'name', 'version', 'nodes', and 'links'. - - Returns: - Network: A fully constructed network containing the parsed nodes - and links. - - Raises: - ValueError: If a link references nodes not defined in the network. - """ - net = Network() - - # Store optional metadata in the network's attrs - if "name" in network_data: - net.attrs["name"] = network_data["name"] - if "version" in network_data: - net.attrs["version"] = network_data["version"] - - # Add nodes - nodes_dict = network_data.get("nodes", {}) - for node_name, node_attrs in nodes_dict.items(): - net.add_node(Node(name=node_name, attrs=node_attrs or {})) - - valid_node_names = set(nodes_dict.keys()) - - # Add links - links_data = network_data.get("links", []) - for link_info in links_data: - source = link_info["source"] - target = link_info["target"] - if source not in valid_node_names or target not in valid_node_names: - raise ValueError(f"Link references unknown node(s): {source}, {target}") - - link_params = link_info.get("link_params", {}) - link = Link( - source=source, - target=target, - capacity=link_params.get("capacity", 1.0), - cost=link_params.get("cost", 1.0), - attrs=link_params.get("attrs", {}), - ) - net.add_link(link) - - return net - @staticmethod def _build_failure_policy(fp_data: Dict[str, Any]) -> FailurePolicy: """ @@ -185,8 +120,7 @@ def _build_failure_policy(fp_data: Dict[str, Any]) -> FailurePolicy: count: 1 Args: - fp_data (Dict[str, Any]): Dictionary for the 'failure_policy' - section of the YAML. + fp_data (Dict[str, Any]): Dictionary for the 'failure_policy' section. Returns: FailurePolicy: A policy containing a list of FailureRule objects. @@ -214,7 +148,7 @@ def _build_failure_policy(fp_data: Dict[str, Any]) -> FailurePolicy: ) rules.append(rule) - # Put any additional keys (e.g., "name", "description") into policy.attrs + # Put any extra keys (like "name" or "description") into policy.attrs attrs = {k: v for k, v in fp_data.items() if k != "rules"} return FailurePolicy(rules=rules, attrs=attrs) @@ -228,7 +162,7 @@ def _build_workflow_steps( Each step dict must have a "step_type" referencing a registered workflow step in WORKFLOW_STEP_REGISTRY. Any additional keys are passed as init - arguments to the WorkflowStep subclass. + arguments to that WorkflowStep subclass. Example: workflow: @@ -246,8 +180,7 @@ def _build_workflow_steps( in the order provided. Raises: - ValueError: If a dict lacks "step_type" or references an unknown - type. + ValueError: If a dict lacks "step_type" or references an unknown type. """ steps: List[WorkflowStep] = [] for step_info in workflow_data: @@ -261,7 +194,7 @@ def _build_workflow_steps( if not step_cls: raise ValueError(f"Unrecognized 'step_type': {step_type}") - # Remove 'step_type' so it doesn't clash with step_cls.__init__ + # Remove 'step_type' so it doesn't conflict with step_cls.__init__ step_args = {k: v for k, v in step_info.items() if k != "step_type"} steps.append(step_cls(**step_args)) return steps diff --git a/ngraph/workflow/build_graph.py b/ngraph/workflow/build_graph.py index dc2e134..bdbc604 100644 --- a/ngraph/workflow/build_graph.py +++ b/ngraph/workflow/build_graph.py @@ -2,9 +2,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -import networkx as nx - from ngraph.workflow.base import WorkflowStep, register_workflow_step +from ngraph.lib.graph import StrictMultiDiGraph if TYPE_CHECKING: from ngraph.scenario import Scenario @@ -26,7 +25,7 @@ class BuildGraph(WorkflowStep): def run(self, scenario: Scenario) -> None: # Create a MultiDiGraph to hold bidirectional edges - graph = nx.MultiDiGraph() + graph = StrictMultiDiGraph() # 1) Add nodes for node_name, node in scenario.network.nodes.items(): diff --git a/tests/scenarios/test_scenario_1.py b/tests/scenarios/test_scenario_1.py index 3eff2cc..b4a1a17 100644 --- a/tests/scenarios/test_scenario_1.py +++ b/tests/scenarios/test_scenario_1.py @@ -1,7 +1,7 @@ import pytest -import networkx as nx from pathlib import Path +from ngraph.lib.graph import StrictMultiDiGraph from ngraph.scenario import Scenario from ngraph.failure_policy import FailurePolicy @@ -29,8 +29,8 @@ def test_scenario_1_build_graph() -> None: # 4) Retrieve the graph built by BuildGraph graph = scenario.results.get("build_graph", "graph") assert isinstance( - graph, nx.MultiDiGraph - ), "Expected a MultiDiGraph in scenario.results under key ('build_graph', 'graph')." + graph, StrictMultiDiGraph + ), "Expected a StrictMultiDiGraph in scenario.results under key ('build_graph', 'graph')." # 5) Check the total number of nodes matches what's listed in scenario_1.yaml # For a 6-node scenario, we expect 6 nodes in the final Nx graph. diff --git a/tests/scenarios/test_scenario_2.py b/tests/scenarios/test_scenario_2.py new file mode 100644 index 0000000..626fd1e --- /dev/null +++ b/tests/scenarios/test_scenario_2.py @@ -0,0 +1,100 @@ +import pytest +from pathlib import Path + +from ngraph.lib.graph import StrictMultiDiGraph +from ngraph.scenario import Scenario +from ngraph.failure_policy import FailurePolicy + + +def test_scenario_2_build_graph() -> None: + """ + Integration test that verifies we can parse scenario_2.yaml, + run the BuildGraph step, and produce a valid NetworkX MultiDiGraph. + + Checks: + - The expected number of expanded nodes and links (including blueprint subgroups). + - The presence of key expanded nodes (e.g., overridden spine nodes). + - The traffic demands are loaded. + - The multi-rule failure policy matches "anySingleLink". + """ + # 1) Load the YAML file + scenario_path = Path(__file__).parent / "scenario_2.yaml" + yaml_text = scenario_path.read_text() + + # 2) Parse into a Scenario object (this calls blueprint expansion) + scenario = Scenario.from_yaml(yaml_text) + + # 3) Run the scenario's workflow (in this YAML, there's only "BuildGraph") + scenario.run() + + # 4) Retrieve the graph built by BuildGraph + graph = scenario.results.get("build_graph", "graph") + assert isinstance( + graph, StrictMultiDiGraph + ), "Expected a StrictMultiDiGraph in scenario.results under key ('build_graph', 'graph')." + + # 5) Verify total node count after blueprint expansion + # city_cloud blueprint: (4 leaves + 6 spines + 4 edge_nodes) = 14 + # single_node blueprint: 1 node + # plus 4 standalone global nodes (DEN, DFW, JFK, DCA) + # => 14 + 1 + 4 = 19 total + expected_nodes = 19 + actual_nodes = len(graph.nodes) + assert ( + actual_nodes == expected_nodes + ), f"Expected {expected_nodes} nodes, found {actual_nodes}" + + # 6) Verify total physical links before direction is applied to Nx + # - clos_2tier adjacency: 4 leaf * 6 spine = 24 + # - city_cloud adjacency: clos_instance/leaf(4) -> edge_nodes(4) => 16 + # => total within blueprint = 24 + 16 = 40 + # - top-level adjacency: + # SFO(1) -> DEN(1) => 1 + # SFO(1) -> DFW(1) => 1 + # SEA/edge_nodes(4) -> DEN(1) => 4 + # SEA/edge_nodes(4) -> DFW(1) => 4 + # => 1 + 1 + 4 + 4 = 10 + # - sum so far = 40 + 10 = 50 + # - plus 6 direct link definitions => total physical links = 56 + # - each link becomes 2 directed edges in MultiDiGraph => 112 edges + expected_links = 56 + expected_nx_edges = expected_links * 2 + actual_edges = len(graph.edges) + assert ( + actual_edges == expected_nx_edges + ), f"Expected {expected_nx_edges} directed edges, found {actual_edges}" + + # 7) Verify the traffic demands (should have 4) + expected_demands = 4 + assert ( + len(scenario.traffic_demands) == expected_demands + ), f"Expected {expected_demands} traffic demands." + + # 8) Check the single-rule failure policy "anySingleLink" + policy: FailurePolicy = scenario.failure_policy + assert len(policy.rules) == 1, "Should only have 1 rule for 'anySingleLink'." + + rule = policy.rules[0] + assert len(rule.conditions) == 1, "Expected exactly 1 condition for matching links." + cond = rule.conditions[0] + assert cond.attr == "type" + assert cond.operator == "==" + assert cond.value == "link" + assert rule.logic == "and" + assert rule.rule_type == "choice" + assert rule.count == 1 + assert policy.attrs.get("name") == "anySingleLink" + assert ( + policy.attrs.get("description") + == "Evaluate traffic routing under any single link failure." + ) + + # 9) Check presence of key expanded nodes + # For example: the overridden spine node "myspine-6" under "SEA/clos_instance/spine" + # and the single node blueprint "SFO/single/single-1". + assert ( + "SEA/clos_instance/spine/myspine-6" in scenario.network.nodes + ), "Missing expected overridden spine node (myspine-6) in expanded blueprint." + assert ( + "SFO/single/single-1" in scenario.network.nodes + ), "Missing expected single-node blueprint expansion under SFO." diff --git a/tests/test_blueprints_helpers.py b/tests/test_blueprints_helpers.py index 92ae16f..04fb150 100644 --- a/tests/test_blueprints_helpers.py +++ b/tests/test_blueprints_helpers.py @@ -9,7 +9,8 @@ _find_nodes_by_path, _create_link, _expand_adjacency_pattern, - _process_direct_nodes_and_links, + _process_direct_nodes, + _process_direct_links, _expand_blueprint_adjacency, _expand_adjacency, _expand_group, @@ -210,10 +211,10 @@ def test_expand_adjacency_pattern_mesh(): assert link.capacity == 99 -def test_process_direct_nodes_and_links(): +def test_process_direct_nodes(): """ - Tests _process_direct_nodes_and_links to ensure direct node creation - and direct link creation works, verifying unknown nodes raise errors. + Tests _process_direct_nodes to ensure direct node creation + works. """ net = Network() net.add_node(Node("Existing")) @@ -225,27 +226,43 @@ def test_process_direct_nodes_and_links(): "override": "ignored" }, # This won't be merged since node exists }, + } + + _process_direct_nodes(net, network_data) + + # "New1" was created + assert "New1" in net.nodes + assert net.nodes["New1"].attrs["foo"] == "bar" + # "Existing" was not overwritten + assert "override" not in net.nodes["Existing"].attrs + + +def test_process_direct_links(): + """ + Tests _process_direct_links to ensure direct link creation works. + """ + net = Network() + net.add_node(Node("Existing1")) + net.add_node(Node("Existing2")) + + network_data = { "links": [ { - "source": "New1", - "target": "NoExist", # Should fail + "source": "Existing1", + "target": "Existing2", "link_params": {"capacity": 5}, } ], } - with pytest.raises(ValueError) as excinfo: - _process_direct_nodes_and_links(net, network_data) - # The error should mention "unknown node(s): New1, NoExist" - assert "Link references unknown node(s): New1, NoExist" in str(excinfo.value) - # Confirm that partial link creation was blocked - assert len(net.links) == 0 + _process_direct_links(net, network_data) - # Meanwhile, "New1" was created - assert "New1" in net.nodes - assert net.nodes["New1"].attrs["foo"] == "bar" - # "Existing" was not overwritten - assert "override" not in net.nodes["Existing"].attrs + # Confirm that links were created + assert len(net.links) == 1 + link = next(iter(net.links.values())) + assert link.source == "Existing1" + assert link.target == "Existing2" + assert link.capacity == 5 def test_expand_blueprint_adjacency(): From d239d494ca689869b9518636e68015c5044af2b3 Mon Sep 17 00:00:00 2001 From: Andrey Golovanov Date: Mon, 24 Feb 2025 00:23:05 +0000 Subject: [PATCH 3/3] expanding tests with another scenario --- ngraph/network.py | 2 +- tests/scenarios/scenario_3.yaml | 64 +++++++++++++++++++++++++++ tests/scenarios/test_scenario_1.py | 2 +- tests/scenarios/test_scenario_2.py | 2 +- tests/scenarios/test_scenario_3.py | 71 ++++++++++++++++++++++++++++++ tests/test_network.py | 8 ++-- 6 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 tests/scenarios/scenario_3.yaml create mode 100644 tests/scenarios/test_scenario_3.py diff --git a/ngraph/network.py b/ngraph/network.py index ab5f8de..51c8ad9 100644 --- a/ngraph/network.py +++ b/ngraph/network.py @@ -66,7 +66,7 @@ class Link: def __post_init__(self) -> None: """Auto-generates a unique link ID by combining the source, target, and a random Base64-encoded UUID.""" - self.id = f"{self.source}-{self.target}-{new_base64_uuid()}" + self.id = f"{self.source}|{self.target}|{new_base64_uuid()}" @dataclass(slots=True) diff --git a/tests/scenarios/scenario_3.yaml b/tests/scenarios/scenario_3.yaml new file mode 100644 index 0000000..7a1cefd --- /dev/null +++ b/tests/scenarios/scenario_3.yaml @@ -0,0 +1,64 @@ +blueprints: + brick_2tier: + groups: + t1: + node_count: 4 + name_template: t1-{node_num} + t2: + node_count: 4 + name_template: t2-{node_num} + + adjacency: + - source: /t1 + target: /t2 + pattern: mesh + link_params: + capacity: 1 + cost: 1 + + 3tier_clos: + groups: + b1: + use_blueprint: brick_2tier + b2: + use_blueprint: brick_2tier + spine: + node_count: 16 + name_template: t3-{node_num} + + adjacency: + - source: b1/t2 + target: spine + pattern: one_to_one + link_params: + capacity: 1 + cost: 1 + - source: b2/t2 + target: spine + pattern: one_to_one + link_params: + capacity: 1 + cost: 1 +network: + name: "3tier_clos_network" + version: 1.0 + + groups: + my_clos1: + use_blueprint: 3tier_clos + + my_clos2: + use_blueprint: 3tier_clos + + adjacency: + - source: my_clos1/spine + target: my_clos2/spine + pattern: one_to_one + link_params: + capacity: 1 + cost: 1 + + +workflow: + - step_type: BuildGraph + name: build_graph diff --git a/tests/scenarios/test_scenario_1.py b/tests/scenarios/test_scenario_1.py index b4a1a17..434c59c 100644 --- a/tests/scenarios/test_scenario_1.py +++ b/tests/scenarios/test_scenario_1.py @@ -9,7 +9,7 @@ def test_scenario_1_build_graph() -> None: """ Integration test that verifies we can parse scenario_1.yaml, - run the BuildGraph step, and produce a valid NetworkX MultiDiGraph. + run the BuildGraph step, and produce a valid StrictMultiDiGraph. Checks: - The expected number of nodes and links are correctly parsed. - The traffic demands are loaded. diff --git a/tests/scenarios/test_scenario_2.py b/tests/scenarios/test_scenario_2.py index 626fd1e..d101503 100644 --- a/tests/scenarios/test_scenario_2.py +++ b/tests/scenarios/test_scenario_2.py @@ -9,7 +9,7 @@ def test_scenario_2_build_graph() -> None: """ Integration test that verifies we can parse scenario_2.yaml, - run the BuildGraph step, and produce a valid NetworkX MultiDiGraph. + run the BuildGraph step, and produce a valid StrictMultiDiGraph. Checks: - The expected number of expanded nodes and links (including blueprint subgroups). diff --git a/tests/scenarios/test_scenario_3.py b/tests/scenarios/test_scenario_3.py new file mode 100644 index 0000000..4fe8736 --- /dev/null +++ b/tests/scenarios/test_scenario_3.py @@ -0,0 +1,71 @@ +import pytest +from pathlib import Path + +from ngraph.lib.graph import StrictMultiDiGraph +from ngraph.scenario import Scenario +from ngraph.failure_policy import FailurePolicy + + +def test_scenario_3_build_graph() -> None: + """ + Integration test that verifies we can parse scenario_3.yaml, + run the BuildGraph step, and produce a valid StrictMultiDiGraph. + + Checks: + - The expected number of expanded nodes and links (two interconnected 3-tier CLOS fabrics). + - The presence of key expanded nodes. + - The traffic demands are empty in this scenario. + - The failure policy is empty by default. + """ + # 1) Load the YAML file + scenario_path = Path(__file__).parent / "scenario_3.yaml" + yaml_text = scenario_path.read_text() + + # 2) Parse into a Scenario object (this calls blueprint expansion) + scenario = Scenario.from_yaml(yaml_text) + + # 3) Run the scenario's workflow (in this YAML, there's only "BuildGraph") + scenario.run() + + # 4) Retrieve the graph built by BuildGraph + graph = scenario.results.get("build_graph", "graph") + assert isinstance( + graph, StrictMultiDiGraph + ), "Expected a StrictMultiDiGraph in scenario.results under key ('build_graph', 'graph')." + + # 5) Verify total node count + # Each 3-tier CLOS instance has 32 nodes (2 sub-bricks of 8 nodes each + 16 spine), + # so with 2 instances => 64 nodes total. + expected_nodes = 64 + actual_nodes = len(graph.nodes) + assert ( + actual_nodes == expected_nodes + ), f"Expected {expected_nodes} nodes, found {actual_nodes}" + + # 6) Verify total physical links before direction is applied to Nx + # Each 3-tier CLOS has 64 links internally. With 2 instances => 128 links, + # plus 16 links connecting my_clos1/spine to my_clos2/spine (one_to_one). + # => total physical links = 128 + 16 = 144 + # => each link becomes 2 directed edges in MultiDiGraph => 288 edges + expected_links = 144 + expected_nx_edges = expected_links * 2 + actual_edges = len(graph.edges) + assert ( + actual_edges == expected_nx_edges + ), f"Expected {expected_nx_edges} directed edges, found {actual_edges}" + + # 7) Verify that there are no traffic demands in this scenario + assert len(scenario.traffic_demands) == 0, "Expected zero traffic demands." + + # 8) Verify the default (empty) failure policy + policy: FailurePolicy = scenario.failure_policy + assert len(policy.rules) == 0, "Expected an empty failure policy." + + # 9) Check presence of a few key expanded nodes + # For example: a t1 node in my_clos1/b1 and a spine node in my_clos2. + assert ( + "my_clos1/b1/t1/t1-1" in scenario.network.nodes + ), "Missing expected node 'my_clos1/b1/t1/t1-1' in expanded blueprint." + assert ( + "my_clos2/spine/t3-16" in scenario.network.nodes + ), "Missing expected spine node 'my_clos2/spine/t3-16' in expanded blueprint." diff --git a/tests/test_network.py b/tests/test_network.py index b2f242f..e306f1b 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -55,9 +55,9 @@ def test_link_defaults_and_id_generation(): assert link.cost == 1.0 assert link.attrs == {} - # ID should start with 'A-B-' and have a random suffix - assert link.id.startswith("A-B-") - assert len(link.id) > len("A-B-") + # ID should start with 'A|B|' and have a random suffix + assert link.id.startswith("A|B|") + assert len(link.id) > len("A|B|") def test_link_custom_values(): @@ -73,7 +73,7 @@ def test_link_custom_values(): assert link.capacity == 2.0 assert link.cost == 4.0 assert link.attrs == custom_attrs - assert link.id.startswith("X-Y-") + assert link.id.startswith("X|Y|") def test_link_id_uniqueness():