Add JSON output to benchmarks and upload as artifacts

- Update benchmark-hardware.py to save results to JSON
- Update benchmark-jupyter.ipynb to save results to JSON
- Update benchmark-jupyterbook.md to save results to JSON
- Add CI step to collect and display benchmark results
- Add CI step to upload benchmark results as artifact

Author: mmcky

.github/workflows/ci.yml

@@ -45,8 +45,10 @@ jobs:
         shell: bash -l {0}
         run: |
           echo "=== Jupyter Kernel Execution ==="
-          jupyter nbconvert --to notebook --execute scripts/benchmark-jupyter.ipynb --output benchmark-jupyter-executed.ipynb
+          cd scripts
+          jupyter nbconvert --to notebook --execute benchmark-jupyter.ipynb --output benchmark-jupyter-executed.ipynb
           echo "Notebook executed successfully"
+          cd ..
       - name: Run Jupyter-Book Benchmark
         shell: bash -l {0}
         run: |
@@ -61,9 +63,40 @@ jobs:
           # Create minimal _toc.yml
           echo "format: jb-book" > benchmark_test/_toc.yml
           echo "root: benchmark-jupyterbook" >> benchmark_test/_toc.yml
-          # Build
-          jb build benchmark_test --path-output benchmark_build/
+          # Build (run from benchmark_test so JSON is written there)
+          cd benchmark_test
+          jb build . --path-output ../benchmark_build/
+          cd ..
           echo "Jupyter-Book build completed successfully"
+      - name: Collect and Display Benchmark Results
+        shell: bash -l {0}
+        run: |
+          echo "=== Collecting Benchmark Results ==="
+          mkdir -p benchmark_results
+          
+          # Copy results from each pathway
+          cp benchmark_results_bare_metal.json benchmark_results/ 2>/dev/null || echo "No bare metal results"
+          cp scripts/benchmark_results_jupyter.json benchmark_results/ 2>/dev/null || echo "No jupyter results"
+          cp benchmark_test/benchmark_results_jupyterbook.json benchmark_results/ 2>/dev/null || echo "No jupyterbook results"
+          
+          # Display summary
+          echo ""
+          echo "============================================================"
+          echo "BENCHMARK RESULTS SUMMARY"
+          echo "============================================================"
+          for f in benchmark_results/*.json; do
+            if [ -f "$f" ]; then
+              echo ""
+              echo "--- $(basename $f) ---"
+              cat "$f"
+            fi
+          done
+      - name: Upload Benchmark Results
+        uses: actions/upload-artifact@v5
+        with:
+          name: benchmark-results
+          path: benchmark_results/
+          if-no-files-found: warn
       - name: Download "build" folder (cache)
         uses: dawidd6/action-download-artifact@v11
         with:

scripts/benchmark-hardware.py

@@ -6,6 +6,16 @@
 import time
 import platform
 import os
+import json
+from datetime import datetime
+
+# Global results dictionary
+RESULTS = {
+    "pathway": "bare_metal",
+    "timestamp": datetime.now().isoformat(),
+    "system": {},
+    "benchmarks": {}
+}
 
 def get_cpu_info():
     """Get CPU information."""
@@ -16,12 +26,20 @@ def get_cpu_info():
     print(f"Processor: {platform.processor()}")
     print(f"Python: {platform.python_version()}")
 
-    # Try to get CPU frequency
+    RESULTS["system"]["platform"] = platform.platform()
+    RESULTS["system"]["processor"] = platform.processor()
+    RESULTS["system"]["python"] = platform.python_version()
+    RESULTS["system"]["cpu_count"] = os.cpu_count()
+    
+    # Try to get CPU model
+    cpu_model = None
+    cpu_mhz = None
     try:
         with open('/proc/cpuinfo', 'r') as f:
             for line in f:
                 if 'model name' in line:
-                    print(f"CPU Model: {line.split(':')[1].strip()}")
+                    cpu_model = line.split(':')[1].strip()
+                    print(f"CPU Model: {cpu_model}")
                     break
     except:
         pass
@@ -31,26 +49,33 @@ def get_cpu_info():
         with open('/proc/cpuinfo', 'r') as f:
             for line in f:
                 if 'cpu MHz' in line:
-                    print(f"CPU MHz: {line.split(':')[1].strip()}")
+                    cpu_mhz = line.split(':')[1].strip()
+                    print(f"CPU MHz: {cpu_mhz}")
                     break
     except:
         pass
 
+    RESULTS["system"]["cpu_model"] = cpu_model
+    RESULTS["system"]["cpu_mhz"] = cpu_mhz
+    
     # CPU count
     print(f"CPU Count: {os.cpu_count()}")
 
     # Check for GPU
+    gpu_info = None
     try:
         import subprocess
         result = subprocess.run(['nvidia-smi', '--query-gpu=name,memory.total', '--format=csv,noheader'], 
                               capture_output=True, text=True, timeout=5)
         if result.returncode == 0:
-            print(f"GPU: {result.stdout.strip()}")
+            gpu_info = result.stdout.strip()
+            print(f"GPU: {gpu_info}")
         else:
             print("GPU: None detected")
     except:
         print("GPU: None detected (nvidia-smi not available)")
 
+    RESULTS["system"]["gpu"] = gpu_info
     print()
 
 def benchmark_cpu_pure_python():
@@ -59,11 +84,14 @@ def benchmark_cpu_pure_python():
     print("CPU BENCHMARK: Pure Python")
     print("=" * 60)
 
+    results = {}
+    
     # Integer computation
     start = time.perf_counter()
     total = sum(i * i for i in range(10_000_000))
     elapsed = time.perf_counter() - start
     print(f"Integer sum (10M iterations): {elapsed:.3f} seconds")
+    results["integer_sum_10m"] = elapsed
 
     # Float computation
     start = time.perf_counter()
@@ -72,7 +100,10 @@ def benchmark_cpu_pure_python():
         total += (i * 0.1) ** 0.5
     elapsed = time.perf_counter() - start
     print(f"Float sqrt (1M iterations): {elapsed:.3f} seconds")
+    results["float_sqrt_1m"] = elapsed
     print()
+    
+    RESULTS["benchmarks"]["pure_python"] = results
 
 def benchmark_cpu_numpy():
     """NumPy CPU benchmark."""
@@ -82,6 +113,8 @@ def benchmark_cpu_numpy():
     print("CPU BENCHMARK: NumPy")
     print("=" * 60)
 
+    results = {}
+    
     # Matrix multiplication
     n = 3000
     A = np.random.randn(n, n)
@@ -91,6 +124,7 @@ def benchmark_cpu_numpy():
     C = A @ B
     elapsed = time.perf_counter() - start
     print(f"Matrix multiply ({n}x{n}): {elapsed:.3f} seconds")
+    results["matmul_3000x3000"] = elapsed
 
     # Element-wise operations
     x = np.random.randn(50_000_000)
@@ -99,7 +133,10 @@ def benchmark_cpu_numpy():
     y = np.cos(x**2) + np.sin(x)
     elapsed = time.perf_counter() - start
     print(f"Element-wise ops (50M elements): {elapsed:.3f} seconds")
+    results["elementwise_50m"] = elapsed
     print()
+    
+    RESULTS["benchmarks"]["numpy"] = results
 
 def benchmark_gpu_jax():
     """JAX benchmark (GPU if available, otherwise CPU)."""
@@ -125,6 +162,12 @@ def benchmark_gpu_jax():
         print(f"GPU Available: {has_gpu}")
         print()
 
+        results = {
+            "backend": default_backend,
+            "has_gpu": has_gpu,
+            "devices": str(devices)
+        }
+        
         # Warm-up JIT compilation
         print("Warming up JIT compilation...")
         n = 1000
@@ -141,12 +184,14 @@ def matmul(a, b):
         C = matmul(A, B).block_until_ready()
         warmup_time = time.perf_counter() - start
         print(f"Warm-up (includes JIT compile, {n}x{n}): {warmup_time:.3f} seconds")
+        results["matmul_1000x1000_warmup"] = warmup_time
 
         # Actual benchmark (compiled)
         start = time.perf_counter()
         C = matmul(A, B).block_until_ready()
         elapsed = time.perf_counter() - start
         print(f"Matrix multiply compiled ({n}x{n}): {elapsed:.3f} seconds")
+        results["matmul_1000x1000_compiled"] = elapsed
 
         # Larger matrix
         n = 3000
@@ -158,12 +203,14 @@ def matmul(a, b):
         C = matmul(A, B).block_until_ready()
         warmup_time = time.perf_counter() - start
         print(f"Warm-up (recompile for {n}x{n}): {warmup_time:.3f} seconds")
+        results["matmul_3000x3000_warmup"] = warmup_time
 
         # Benchmark compiled
         start = time.perf_counter()
         C = matmul(A, B).block_until_ready()
         elapsed = time.perf_counter() - start
         print(f"Matrix multiply compiled ({n}x{n}): {elapsed:.3f} seconds")
+        results["matmul_3000x3000_compiled"] = elapsed
 
         # Element-wise GPU benchmark
         x = jax.random.normal(key, (50_000_000,))
@@ -177,19 +224,24 @@ def elementwise_ops(x):
         y = elementwise_ops(x).block_until_ready()
         warmup_time = time.perf_counter() - start
         print(f"Element-wise warm-up (50M): {warmup_time:.3f} seconds")
+        results["elementwise_50m_warmup"] = warmup_time
 
         # Compiled
         start = time.perf_counter()
         y = elementwise_ops(x).block_until_ready()
         elapsed = time.perf_counter() - start
         print(f"Element-wise compiled (50M): {elapsed:.3f} seconds")
+        results["elementwise_50m_compiled"] = elapsed
 
         print()
+        RESULTS["benchmarks"]["jax"] = results
 
     except ImportError as e:
         print(f"JAX not available: {e}")
+        RESULTS["benchmarks"]["jax"] = {"error": str(e)}
     except Exception as e:
         print(f"JAX benchmark failed: {e}")
+        RESULTS["benchmarks"]["jax"] = {"error": str(e)}
 
 def benchmark_numba():
     """Numba CPU benchmark."""
@@ -201,6 +253,8 @@ def benchmark_numba():
         print("CPU BENCHMARK: Numba")
         print("=" * 60)
 
+        results = {}
+        
         @numba.jit(nopython=True)
         def numba_sum(n):
             total = 0
@@ -213,12 +267,14 @@ def numba_sum(n):
         result = numba_sum(10_000_000)
         warmup_time = time.perf_counter() - start
         print(f"Integer sum warm-up (includes compile): {warmup_time:.3f} seconds")
+        results["integer_sum_10m_warmup"] = warmup_time
 
         # Compiled run
         start = time.perf_counter()
         result = numba_sum(10_000_000)
         elapsed = time.perf_counter() - start
         print(f"Integer sum compiled (10M): {elapsed:.3f} seconds")
+        results["integer_sum_10m_compiled"] = elapsed
 
         @numba.jit(nopython=True, parallel=True)
         def numba_parallel_sum(arr):
@@ -234,19 +290,32 @@ def numba_parallel_sum(arr):
         result = numba_parallel_sum(arr)
         warmup_time = time.perf_counter() - start
         print(f"Parallel sum warm-up (50M): {warmup_time:.3f} seconds")
+        results["parallel_sum_50m_warmup"] = warmup_time
 
         # Compiled
         start = time.perf_counter()
         result = numba_parallel_sum(arr)
         elapsed = time.perf_counter() - start
         print(f"Parallel sum compiled (50M): {elapsed:.3f} seconds")
+        results["parallel_sum_50m_compiled"] = elapsed
 
         print()
+        RESULTS["benchmarks"]["numba"] = results
 
     except ImportError as e:
         print(f"Numba not available: {e}")
+        RESULTS["benchmarks"]["numba"] = {"error": str(e)}
     except Exception as e:
         print(f"Numba benchmark failed: {e}")
+        RESULTS["benchmarks"]["numba"] = {"error": str(e)}
+
+
+def save_results(output_path="benchmark_results_bare_metal.json"):
+    """Save benchmark results to JSON file."""
+    with open(output_path, 'w') as f:
+        json.dump(RESULTS, f, indent=2)
+    print(f"\nResults saved to: {output_path}")
+
 
 if __name__ == "__main__":
     print("\n" + "=" * 60)
@@ -259,6 +328,9 @@ def numba_parallel_sum(arr):
     benchmark_numba()
     benchmark_gpu_jax()
 
+    # Save results to JSON
+    save_results("benchmark_results_bare_metal.json")
+    
     print("=" * 60)
     print("BENCHMARK COMPLETE")
     print("=" * 60)