fixed CI, fixed formatting, improved doc generation

Author: networmix

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.4
+    rev: v0.11.13
     hooks:
       - id: ruff
         args: [--fix]

Makefile

@@ -23,6 +23,7 @@ help:
 	@echo ""
 	@echo "Documentation:"
 	@echo "  make docs          - Generate API documentation"
+	@echo "  make docs-test     - Test API documentation generation"
 	@echo "  make docs-serve    - Serve documentation locally"
 	@echo ""
 	@echo "Build & Package:"
@@ -79,7 +80,12 @@ test-quick:
 # Documentation
 docs:
 	@echo "📚 Generating API documentation..."
-	@python dev/generate_api_docs.py
+	@echo "ℹ️  This regenerates docs/reference/api-full.md from source code"
+	@python dev/generate_api_docs.py --write-file
+
+docs-test:
+	@echo "🧪 Testing API documentation generation..."
+	@python dev/test_doc_generation.py
 
 docs-serve:
 	@echo "🌐 Serving documentation locally..."

dev/dev.md

@@ -6,11 +6,38 @@
 make setup         # Complete dev environment setup
 make check         # Run all quality checks + tests
 make test          # Run tests with coverage
+make docs          # Generate API documentation
 make docs-serve    # Serve docs locally
 ```
 
 **For all available commands**: `make help`
 
+## Documentation
+
+### Generating API Documentation
+
+The API documentation is auto-generated from source code docstrings:
+
+```bash
+# Generate API documentation
+make docs
+# or
+python dev/generate_api_docs.py
+```
+
+**Important**: API documentation is **not** regenerated during pytest runs to avoid constant file changes. The doc generation test is skipped by default. To test doc generation:
+
+```bash
+GENERATE_DOCS=true pytest tests/test_api_docs.py::test_api_doc_generation_output
+```
+
+### Documentation Types
+
+- `docs/reference/api.md` - Curated, example-driven API guide (manually maintained)
+- `docs/reference/api-full.md` - Complete auto-generated reference (regenerated via `make docs`)
+- `docs/reference/cli.md` - Command-line interface documentation
+- `docs/reference/dsl.md` - YAML DSL syntax reference
+
 ## Publishing
 
 **Manual**: `make clean && make build && make publish-test && make publish`

dev/generate_api_docs.py

@@ -2,8 +2,12 @@
 """
 Generate API documentation for NetGraph
 This script should be run from the project root directory.
+
+By default, outputs documentation to stdout.
+Use --write-file to write to docs/reference/api-full.md instead.
 """
 
+import argparse
 import dataclasses
 import importlib
 import inspect
@@ -145,8 +149,16 @@ def document_module(module_name):
     return doc
 
 
-def generate_api_documentation():
-    """Generate the complete API documentation."""
+def generate_api_documentation(output_to_file=False):
+    """Generate the complete API documentation.
+
+    Args:
+        output_to_file (bool): If True, write to docs/reference/api-full.md.
+                               If False, return the documentation string.
+
+    Returns:
+        str: The generated documentation (when output_to_file=False)
+    """
 
     # Modules to document (in order)
     modules = [
@@ -230,19 +242,45 @@ def generate_api_documentation():
 
     doc += footer
 
-    # Ensure output directory exists
-    output_path = Path("docs/reference/api-full.md")
-    output_path.parent.mkdir(parents=True, exist_ok=True)
+    if output_to_file:
+        # Ensure output directory exists
+        output_path = Path("docs/reference/api-full.md")
+        output_path.parent.mkdir(parents=True, exist_ok=True)
 
-    # Write to file
-    with open(output_path, "w", encoding="utf-8") as f:
-        f.write(doc)
+        # Write to file
+        with open(output_path, "w", encoding="utf-8") as f:
+            f.write(doc)
 
-    print("✅ API documentation generated successfully!")
-    print(f"📄 Written to: {output_path}")
-    print(f"📊 Size: {len(doc):,} characters")
-    print(f"📚 Modules documented: {len(modules)}")
+        print("✅ API documentation generated successfully!")
+        print(f"📄 Written to: {output_path}")
+        print(f"📊 Size: {len(doc):,} characters")
+        print(f"📚 Modules documented: {len(modules)}")
+    else:
+        # Return the documentation string
+        return doc
 
 
 if __name__ == "__main__":
-    generate_api_documentation()
+    parser = argparse.ArgumentParser(
+        description="Generate API documentation for NetGraph",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  python generate_api_docs.py                # Output to stdout
+  python generate_api_docs.py --write-file   # Write to docs/reference/api-full.md
+        """,
+    )
+    parser.add_argument(
+        "--write-file",
+        action="store_true",
+        help="Write documentation to docs/reference/api-full.md instead of stdout",
+    )
+
+    args = parser.parse_args()
+
+    if args.write_file:
+        generate_api_documentation(output_to_file=True)
+    else:
+        # Output to stdout
+        doc = generate_api_documentation(output_to_file=False)
+        print(doc)

docs/reference/api-full.md

@@ -10,7 +10,7 @@ For a curated, example-driven API guide, see **[api.md](api.md)**.
 > - **[CLI Reference](cli.md)** - Command-line interface
 > - **[DSL Reference](dsl.md)** - YAML syntax guide
 
-**Generated from source code on:** June 08, 2025 at 00:33 UTC
+**Generated from source code on:** June 08, 2025 at 01:15 UTC
 
 ---
 

pyproject.toml

@@ -44,7 +44,7 @@ dev = [
     "mkdocs-material",
     "pdoc",
     # style + type checking
-    "ruff>=0.5",
+    "ruff==0.11.13",
     "pyright",
     # pre-commit hooks
     "pre-commit",

tests/scenarios/test_scenario_1.py

@@ -27,32 +27,32 @@ def test_scenario_1_build_graph() -> None:
 
     # 4) Retrieve the graph built by BuildGraph
     graph = scenario.results.get("build_graph", "graph")
-    assert isinstance(
-        graph, StrictMultiDiGraph
-    ), "Expected a StrictMultiDiGraph in scenario.results under key ('build_graph', 'graph')."
+    assert isinstance(graph, StrictMultiDiGraph), (
+        "Expected a StrictMultiDiGraph in scenario.results under key ('build_graph', 'graph')."
+    )
 
     # 5) Check the total number of nodes matches what's listed in scenario_1.yaml
     #    For a 6-node scenario, we expect 6 nodes in the final Nx graph.
     expected_nodes = 6
     actual_nodes = len(graph.nodes)
-    assert (
-        actual_nodes == expected_nodes
-    ), f"Expected {expected_nodes} nodes, found {actual_nodes}"
+    assert actual_nodes == expected_nodes, (
+        f"Expected {expected_nodes} nodes, found {actual_nodes}"
+    )
 
     # 6) Each physical link from the YAML becomes 2 directed edges in MultiDiGraph.
     #    If the YAML has 10 link definitions, we expect 2 * 10 = 20 directed edges.
     expected_links = 10
     expected_nx_edges = expected_links * 2
     actual_edges = len(graph.edges)
-    assert (
-        actual_edges == expected_nx_edges
-    ), f"Expected {expected_nx_edges} directed edges, found {actual_edges}"
+    assert actual_edges == expected_nx_edges, (
+        f"Expected {expected_nx_edges} directed edges, found {actual_edges}"
+    )
 
     # 7) Verify the traffic demands.
     expected_demands = 4
-    assert (
-        len(scenario.traffic_demands) == expected_demands
-    ), f"Expected {expected_demands} traffic demands."
+    assert len(scenario.traffic_demands) == expected_demands, (
+        f"Expected {expected_demands} traffic demands."
+    )
 
     # 8) Check the multi-rule failure policy for "any single link".
     #    This should have exactly 1 rule that picks exactly 1 link from all links.

tests/scenarios/test_scenario_2.py

@@ -28,9 +28,9 @@ def test_scenario_2_build_graph() -> None:
 
     # 4) Retrieve the graph built by BuildGraph
     graph = scenario.results.get("build_graph", "graph")
-    assert isinstance(
-        graph, StrictMultiDiGraph
-    ), "Expected a StrictMultiDiGraph in scenario.results under key ('build_graph', 'graph')."
+    assert isinstance(graph, StrictMultiDiGraph), (
+        "Expected a StrictMultiDiGraph in scenario.results under key ('build_graph', 'graph')."
+    )
 
     # 5) Verify total node count after blueprint expansion
     #    city_cloud blueprint: (4 leaves + 6 spines + 4 edge_nodes) = 14
@@ -39,9 +39,9 @@ def test_scenario_2_build_graph() -> None:
     #    => 14 + 1 + 4 = 19 total
     expected_nodes = 19
     actual_nodes = len(graph.nodes)
-    assert (
-        actual_nodes == expected_nodes
-    ), f"Expected {expected_nodes} nodes, found {actual_nodes}"
+    assert actual_nodes == expected_nodes, (
+        f"Expected {expected_nodes} nodes, found {actual_nodes}"
+    )
 
     # 6) Verify total physical links before direction is applied to Nx
     #    - clos_2tier adjacency: 4 leaf * 6 spine = 24
@@ -59,15 +59,15 @@ def test_scenario_2_build_graph() -> None:
     expected_links = 56
     expected_nx_edges = expected_links * 2
     actual_edges = len(graph.edges)
-    assert (
-        actual_edges == expected_nx_edges
-    ), f"Expected {expected_nx_edges} directed edges, found {actual_edges}"
+    assert actual_edges == expected_nx_edges, (
+        f"Expected {expected_nx_edges} directed edges, found {actual_edges}"
+    )
 
     # 7) Verify the traffic demands (should have 4)
     expected_demands = 4
-    assert (
-        len(scenario.traffic_demands) == expected_demands
-    ), f"Expected {expected_demands} traffic demands."
+    assert len(scenario.traffic_demands) == expected_demands, (
+        f"Expected {expected_demands} traffic demands."
+    )
 
     # 8) Check the single-rule failure policy "anySingleLink"
     policy: FailurePolicy = scenario.failure_policy
@@ -87,9 +87,9 @@ def test_scenario_2_build_graph() -> None:
     # 9) Check presence of key expanded nodes
     #    For example: the overridden spine node "myspine-6" under "SEA/clos_instance/spine"
     #    and the single node blueprint "SFO/single/single-1".
-    assert (
-        "SEA/clos_instance/spine/myspine-6" in scenario.network.nodes
-    ), "Missing expected overridden spine node (myspine-6) in expanded blueprint."
-    assert (
-        "SFO/single/single-1" in scenario.network.nodes
-    ), "Missing expected single-node blueprint expansion under SFO."
+    assert "SEA/clos_instance/spine/myspine-6" in scenario.network.nodes, (
+        "Missing expected overridden spine node (myspine-6) in expanded blueprint."
+    )
+    assert "SFO/single/single-1" in scenario.network.nodes, (
+        "Missing expected single-node blueprint expansion under SFO."
+    )

tests/scenarios/test_scenario_3.py

@@ -31,17 +31,17 @@ def test_scenario_3_build_graph_and_capacity_probe() -> None:
 
     # 4) Retrieve the graph from the BuildGraph step
     graph = scenario.results.get("build_graph", "graph")
-    assert isinstance(
-        graph, StrictMultiDiGraph
-    ), "Expected a StrictMultiDiGraph in scenario.results under key ('build_graph', 'graph')."
+    assert isinstance(graph, StrictMultiDiGraph), (
+        "Expected a StrictMultiDiGraph in scenario.results under key ('build_graph', 'graph')."
+    )
 
     # 5) Verify total node count:
     #    Each 3-tier CLOS instance has 32 nodes -> 2 instances => 64 total.
     expected_nodes = 64
     actual_nodes = len(graph.nodes)
-    assert (
-        actual_nodes == expected_nodes
-    ), f"Expected {expected_nodes} nodes, found {actual_nodes}"
+    assert actual_nodes == expected_nodes, (
+        f"Expected {expected_nodes} nodes, found {actual_nodes}"
+    )
 
     # 6) Verify total physical links (before direction):
     #    Each 3-tier CLOS has 64 links internally => 2 instances => 128
@@ -50,9 +50,9 @@ def test_scenario_3_build_graph_and_capacity_probe() -> None:
     expected_links = 144
     expected_directed_edges = expected_links * 2
     actual_edges = len(graph.edges)
-    assert (
-        actual_edges == expected_directed_edges
-    ), f"Expected {expected_directed_edges} edges, found {actual_edges}"
+    assert actual_edges == expected_directed_edges, (
+        f"Expected {expected_directed_edges} edges, found {actual_edges}"
+    )
 
     # 7) Verify no traffic demands in this scenario
     assert len(scenario.traffic_demands) == 0, "Expected zero traffic demands."
@@ -62,24 +62,24 @@ def test_scenario_3_build_graph_and_capacity_probe() -> None:
     assert policy is None, "Expected no failure policy in this scenario."
 
     # 9) Check presence of some expanded nodes
-    assert (
-        "my_clos1/b1/t1/t1-1" in scenario.network.nodes
-    ), "Missing expected node 'my_clos1/b1/t1/t1-1' in expanded blueprint."
-    assert (
-        "my_clos2/spine/t3-16" in scenario.network.nodes
-    ), "Missing expected node 'my_clos2/spine/t3-16' in expanded blueprint."
+    assert "my_clos1/b1/t1/t1-1" in scenario.network.nodes, (
+        "Missing expected node 'my_clos1/b1/t1/t1-1' in expanded blueprint."
+    )
+    assert "my_clos2/spine/t3-16" in scenario.network.nodes, (
+        "Missing expected node 'my_clos2/spine/t3-16' in expanded blueprint."
+    )
 
     net = scenario.network
 
     # (A) Node attribute checks from node_overrides:
     # For "my_clos1/b1/t1/t1-1", we expect hw_component="LeafHW-A" and SRG="clos1-b1t1-SRG"
     node_a1 = net.nodes["my_clos1/b1/t1/t1-1"]
-    assert (
-        node_a1.attrs.get("hw_component") == "LeafHW-A"
-    ), "Expected hw_component=LeafHW-A for 'my_clos1/b1/t1/t1-1', but not found."
-    assert node_a1.attrs.get("shared_risk_groups") == [
-        "clos1-b1t1-SRG"
-    ], "Expected shared_risk_group=clos1-b1t1-SRG for 'my_clos1/b1/t1/t1-1'."
+    assert node_a1.attrs.get("hw_component") == "LeafHW-A", (
+        "Expected hw_component=LeafHW-A for 'my_clos1/b1/t1/t1-1', but not found."
+    )
+    assert node_a1.attrs.get("shared_risk_groups") == ["clos1-b1t1-SRG"], (
+        "Expected shared_risk_group=clos1-b1t1-SRG for 'my_clos1/b1/t1/t1-1'."
+    )
 
     # For "my_clos2/b2/t1/t1-1", check hw_component="LeafHW-B" and SRG="clos2-b2t1-SRG"
     node_b2 = net.nodes["my_clos2/b2/t1/t1-1"]
@@ -114,12 +114,12 @@ def test_scenario_3_build_graph_and_capacity_probe() -> None:
     )
     assert link_id_2, "Spine link (t3-2) not found for override check."
     for link_obj in link_id_2:
-        assert link_obj.attrs.get("shared_risk_groups") == [
-            "SpineSRG"
-        ], "Expected SRG=SpineSRG on spine<->spine link."
-        assert (
-            link_obj.attrs.get("hw_component") == "400G-LR4"
-        ), "Expected hw_component=400G-LR4 on spine<->spine link."
+        assert link_obj.attrs.get("shared_risk_groups") == ["SpineSRG"], (
+            "Expected SRG=SpineSRG on spine<->spine link."
+        )
+        assert link_obj.attrs.get("hw_component") == "400G-LR4", (
+            "Expected hw_component=400G-LR4 on spine<->spine link."
+        )
 
     # 10) The capacity probe step computed forward and reverse flows in 'combine' mode
     # with PROPORTIONAL flow placement.