From 0205c41f32432c8134db6d6094fe33e9bdcc571e Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 11:17:57 +0100 Subject: [PATCH 01/27] Add REPL worker subprocess for Flask backend Co-Authored-By: KDW1 --- server/worker.py | 275 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 server/worker.py diff --git a/server/worker.py b/server/worker.py new file mode 100644 index 0000000..76c2bdc --- /dev/null +++ b/server/worker.py @@ -0,0 +1,275 @@ +""" +REPL Worker Subprocess for PathView Flask Backend. + +Direct port of worker.ts to Python. Reads JSON messages from stdin, +executes Python code, and writes JSON responses to stdout. + +Same message protocol (REPLRequest/REPLResponse as JSON lines over stdin/stdout). + +Threading model: +- Reader thread: reads JSON from stdin, dispatches to main thread or queues for streaming +- Main thread: processes init/exec/eval synchronously; for streaming, runs the loop +- Stdout lock: thread-safe writing to stdout (protocol messages only) +""" + +import sys +import io +import json +import threading +import traceback +import queue + +# Lock for thread-safe stdout writing (protocol messages only) +_stdout_lock = threading.Lock() + +# Worker state +_namespace = {} +_clean_globals = set() +_initialized = False + +# Streaming state +_streaming_active = False +_streaming_code_queue = queue.Queue() + +# Queue for messages from reader thread -> main thread +_message_queue = queue.Queue() + + +def send(response: dict) -> None: + """Send a JSON response to the parent process via stdout.""" + with _stdout_lock: + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + + +def _capture_output(func): + """Run func with stdout/stderr captured, sending output as messages.""" + old_stdout = sys.stdout + old_stderr = sys.stderr + captured_out = io.StringIO() + captured_err = io.StringIO() + sys.stdout = captured_out + sys.stderr = captured_err + try: + result = func() + finally: + sys.stdout = old_stdout + sys.stderr = old_stderr + out = captured_out.getvalue() + err = captured_err.getvalue() + if out: + send({"type": "stdout", "value": out}) + if err: + send({"type": "stderr", "value": err}) + return result + + +def initialize() -> None: + """Initialize the worker: import standard packages, capture clean globals.""" + global _initialized, _namespace, _clean_globals + + if _initialized: + send({"type": "ready"}) + return + + send({"type": "progress", "value": "Initializing Python worker..."}) + + # Set up the namespace with common imports + _namespace = {"__builtins__": __builtins__} + exec("import numpy as np", _namespace) + exec("import gc", _namespace) + exec("import json", _namespace) + + # Capture clean state for later cleanup + _clean_globals = set(_namespace.keys()) + + _initialized = True + send({"type": "ready"}) + + +def exec_code(msg_id: str, code: str) -> None: + """Execute Python code (no return value).""" + if not _initialized: + send({"type": "error", "id": msg_id, "error": "Worker not initialized"}) + return + + try: + def run(): + exec(code, _namespace) + _capture_output(run) + send({"type": "ok", "id": msg_id}) + except Exception as e: + tb = traceback.format_exc() + send({"type": "error", "id": msg_id, "error": str(e), "traceback": tb}) + + +def eval_expr(msg_id: str, expr: str) -> None: + """Evaluate Python expression and return JSON result.""" + if not _initialized: + send({"type": "error", "id": msg_id, "error": "Worker not initialized"}) + return + + try: + def run(): + # Mirror worker.ts: store result, then JSON-serialize + exec_code_str = f"_eval_result = {expr}" + exec(exec_code_str, _namespace) + to_json = _namespace.get("_to_json", str) + return json.dumps(_namespace["_eval_result"], default=to_json) + result = _capture_output(run) + send({"type": "value", "id": msg_id, "value": result}) + except Exception as e: + tb = traceback.format_exc() + send({"type": "error", "id": msg_id, "error": str(e), "traceback": tb}) + + +def run_streaming_loop(msg_id: str, expr: str) -> None: + """Run streaming loop - steps generator continuously and posts results.""" + global _streaming_active + + if not _initialized: + send({"type": "error", "id": msg_id, "error": "Worker not initialized"}) + return + + _streaming_active = True + # Clear any stale code from previous runs + while not _streaming_code_queue.empty(): + try: + _streaming_code_queue.get_nowait() + except queue.Empty: + break + + try: + while _streaming_active: + # Execute any queued code first (for runtime parameter changes) + # Errors in queued code are reported but don't stop the simulation + while True: + try: + code = _streaming_code_queue.get_nowait() + except queue.Empty: + break + try: + def run_queued(c=code): + exec(c, _namespace) + _capture_output(run_queued) + except Exception as e: + send({"type": "stderr", "value": f"Stream exec error: {e}"}) + + # Step the generator + def run_step(): + exec_code_str = f"_eval_result = {expr}" + exec(exec_code_str, _namespace) + to_json = _namespace.get("_to_json", str) + return json.dumps(_namespace["_eval_result"], default=to_json) + result = _capture_output(run_step) + + # Parse result + parsed = json.loads(result) + + # Check if stopped during Python execution - still send final data + if not _streaming_active: + if not parsed.get("done") and parsed.get("result"): + send({"type": "stream-data", "id": msg_id, "value": result}) + break + + # Check if simulation completed + if parsed.get("done"): + break + + # Send result and continue + send({"type": "stream-data", "id": msg_id, "value": result}) + + except Exception as e: + tb = traceback.format_exc() + send({"type": "error", "id": msg_id, "error": str(e), "traceback": tb}) + finally: + _streaming_active = False + # Always send done when loop ends + send({"type": "stream-done", "id": msg_id}) + + +def stop_streaming() -> None: + """Stop the streaming loop.""" + global _streaming_active + _streaming_active = False + + +def reader_thread() -> None: + """Read JSON messages from stdin and dispatch to the main thread.""" + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + send({"type": "error", "error": f"Invalid JSON: {line}"}) + continue + + msg_type = msg.get("type") + + # stream-stop and stream-exec are handled directly (thread-safe) + if msg_type == "stream-stop": + stop_streaming() + elif msg_type == "stream-exec": + code = msg.get("code") + if code and _streaming_active: + _streaming_code_queue.put(code) + else: + # All other messages go to main thread + _message_queue.put(msg) + + # stdin closed — signal main thread to exit + _message_queue.put(None) + + +def main() -> None: + """Main loop: process messages from the reader thread.""" + # Start the reader thread + t = threading.Thread(target=reader_thread, daemon=True) + t.start() + + while True: + msg = _message_queue.get() + if msg is None: + # stdin closed, exit + break + + msg_type = msg.get("type") + msg_id = msg.get("id") + code = msg.get("code") + expr = msg.get("expr") + + try: + if msg_type == "init": + initialize() + + elif msg_type == "exec": + if not msg_id or not isinstance(code, str): + raise ValueError("Invalid exec request: missing id or code") + exec_code(msg_id, code) + + elif msg_type == "eval": + if not msg_id or not isinstance(expr, str): + raise ValueError("Invalid eval request: missing id or expr") + eval_expr(msg_id, expr) + + elif msg_type == "stream-start": + if not msg_id or not isinstance(expr, str): + raise ValueError("Invalid stream-start request: missing id or expr") + # Run in the main thread (blocking until done/stopped) + run_streaming_loop(msg_id, expr) + + else: + raise ValueError(f"Unknown message type: {msg_type}") + + except Exception as e: + send({ + "type": "error", + "id": msg_id, + "error": str(e), + }) + + +if __name__ == "__main__": + main() From fd735ffd27318544a269e735e6674190e87019c5 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 11:42:19 +0100 Subject: [PATCH 02/27] Add Flask server and fix worker threading for Windows subprocess pipes Co-Authored-By: KDW1 --- server/app.py | 358 ++++++++++++++++++++++++++++++++++++++++ server/requirements.txt | 2 + server/worker.py | 100 ++++++----- 3 files changed, 419 insertions(+), 41 deletions(-) create mode 100644 server/app.py create mode 100644 server/requirements.txt diff --git a/server/app.py b/server/app.py new file mode 100644 index 0000000..76cb254 --- /dev/null +++ b/server/app.py @@ -0,0 +1,358 @@ +""" +Flask server for PathView backend. + +Manages worker subprocesses per session. Routes translate HTTP requests +into subprocess messages and relay responses back. + +Each session gets its own worker subprocess with an isolated Python namespace. +""" + +import os +import sys +import json +import subprocess +import threading +import time +import uuid +import atexit +from pathlib import Path + +from flask import Flask, Response, request, jsonify +from flask_cors import CORS + +app = Flask(__name__) +CORS(app) + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +SESSION_TTL = 3600 # 1 hour of inactivity before cleanup +CLEANUP_INTERVAL = 60 # Check for stale sessions every 60 seconds +REQUEST_TIMEOUT = 300 # 5 minutes default timeout for exec/eval +WORKER_SCRIPT = str(Path(__file__).parent / "worker.py") + +# --------------------------------------------------------------------------- +# Session management +# --------------------------------------------------------------------------- + +class Session: + """A worker subprocess bound to a session.""" + + def __init__(self, session_id: str): + self.session_id = session_id + self.last_active = time.time() + self.lock = threading.Lock() + self.process = subprocess.Popen( + [sys.executable, "-u", WORKER_SCRIPT], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, # line buffered + ) + self._initialized = False + + def send_message(self, msg: dict) -> None: + """Write a JSON message to the subprocess stdin.""" + self.last_active = time.time() + line = json.dumps(msg) + "\n" + self.process.stdin.write(line) + self.process.stdin.flush() + + def read_line(self) -> dict | None: + """Read one JSON line from the subprocess stdout.""" + line = self.process.stdout.readline() + if not line: + return None + return json.loads(line.strip()) + + def ensure_initialized(self) -> list[dict]: + """Initialize the worker if not already done. Returns any messages received.""" + if self._initialized: + return [] + messages = [] + self.send_message({"type": "init"}) + while True: + resp = self.read_line() + if resp is None: + raise RuntimeError("Worker process died during initialization") + messages.append(resp) + if resp.get("type") == "ready": + self._initialized = True + break + if resp.get("type") == "error": + raise RuntimeError(resp.get("error", "Unknown init error")) + return messages + + def is_alive(self) -> bool: + return self.process.poll() is None + + def kill(self) -> None: + """Kill the subprocess.""" + try: + self.process.stdin.close() + except Exception: + pass + try: + self.process.kill() + self.process.wait(timeout=5) + except Exception: + pass + + +# Global session store +_sessions: dict[str, Session] = {} +_sessions_lock = threading.Lock() + + +def get_or_create_session(session_id: str) -> Session: + """Get an existing session or create a new one.""" + with _sessions_lock: + session = _sessions.get(session_id) + if session and not session.is_alive(): + # Dead process, remove stale entry + _sessions.pop(session_id, None) + session = None + if session is None: + session = Session(session_id) + _sessions[session_id] = session + return session + + +def remove_session(session_id: str) -> None: + """Kill and remove a session.""" + with _sessions_lock: + session = _sessions.pop(session_id, None) + if session: + session.kill() + + +def cleanup_stale_sessions() -> None: + """Remove sessions that have been inactive beyond TTL.""" + while True: + time.sleep(CLEANUP_INTERVAL) + now = time.time() + stale = [] + with _sessions_lock: + for sid, session in _sessions.items(): + if now - session.last_active > SESSION_TTL: + stale.append(sid) + for sid in stale: + remove_session(sid) + + +# Start cleanup thread +_cleanup_thread = threading.Thread(target=cleanup_stale_sessions, daemon=True) +_cleanup_thread.start() + + +def _get_session_id() -> str: + """Extract session ID from request headers or generate one.""" + return request.headers.get("X-Session-ID") or str(uuid.uuid4()) + + +# --------------------------------------------------------------------------- +# Routes +# --------------------------------------------------------------------------- + +@app.route("/api/health", methods=["GET"]) +def health(): + return jsonify({"status": "ok"}) + + +@app.route("/api/exec", methods=["POST"]) +def api_exec(): + """Execute Python code in the session's worker.""" + session_id = _get_session_id() + data = request.get_json(force=True) + code = data.get("code", "") + msg_id = data.get("id", str(uuid.uuid4())) + + session = get_or_create_session(session_id) + with session.lock: + try: + session.ensure_initialized() + session.send_message({"type": "exec", "id": msg_id, "code": code}) + + # Collect responses until we get ok/error for this id + stdout_lines = [] + stderr_lines = [] + while True: + resp = session.read_line() + if resp is None: + return jsonify({"type": "error", "id": msg_id, "error": "Worker process died"}), 500 + resp_type = resp.get("type") + if resp_type == "stdout": + stdout_lines.append(resp.get("value", "")) + elif resp_type == "stderr": + stderr_lines.append(resp.get("value", "")) + elif resp_type == "ok" and resp.get("id") == msg_id: + result = {"type": "ok", "id": msg_id} + if stdout_lines: + result["stdout"] = "".join(stdout_lines) + if stderr_lines: + result["stderr"] = "".join(stderr_lines) + return jsonify(result) + elif resp_type == "error" and resp.get("id") == msg_id: + result = resp + if stdout_lines: + result["stdout"] = "".join(stdout_lines) + if stderr_lines: + result["stderr"] = "".join(stderr_lines) + return jsonify(result), 400 + + except Exception as e: + return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 + + +@app.route("/api/eval", methods=["POST"]) +def api_eval(): + """Evaluate a Python expression in the session's worker.""" + session_id = _get_session_id() + data = request.get_json(force=True) + expr = data.get("expr", "") + msg_id = data.get("id", str(uuid.uuid4())) + + session = get_or_create_session(session_id) + with session.lock: + try: + session.ensure_initialized() + session.send_message({"type": "eval", "id": msg_id, "expr": expr}) + + stdout_lines = [] + stderr_lines = [] + while True: + resp = session.read_line() + if resp is None: + return jsonify({"type": "error", "id": msg_id, "error": "Worker process died"}), 500 + resp_type = resp.get("type") + if resp_type == "stdout": + stdout_lines.append(resp.get("value", "")) + elif resp_type == "stderr": + stderr_lines.append(resp.get("value", "")) + elif resp_type == "value" and resp.get("id") == msg_id: + result = resp + if stdout_lines: + result["stdout"] = "".join(stdout_lines) + if stderr_lines: + result["stderr"] = "".join(stderr_lines) + return jsonify(result) + elif resp_type == "error" and resp.get("id") == msg_id: + result = resp + if stdout_lines: + result["stdout"] = "".join(stdout_lines) + if stderr_lines: + result["stderr"] = "".join(stderr_lines) + return jsonify(result), 400 + + except Exception as e: + return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 + + +@app.route("/api/stream", methods=["POST"]) +def api_stream(): + """Start a streaming simulation, returning results as SSE.""" + session_id = _get_session_id() + data = request.get_json(force=True) + expr = data.get("expr", "") + msg_id = data.get("id", str(uuid.uuid4())) + + session = get_or_create_session(session_id) + + def generate(): + with session.lock: + try: + session.ensure_initialized() + session.send_message({"type": "stream-start", "id": msg_id, "expr": expr}) + + while True: + resp = session.read_line() + if resp is None: + yield f"event: error\ndata: {json.dumps({'error': 'Worker process died'})}\n\n" + break + resp_type = resp.get("type") + + if resp_type == "stream-data": + yield f"event: data\ndata: {json.dumps({'done': False, 'result': json.loads(resp.get('value', '{}'))})}\n\n" + elif resp_type == "stream-done": + yield f"event: done\ndata: {{}}\n\n" + break + elif resp_type == "stdout": + yield f"event: stdout\ndata: {json.dumps(resp.get('value', ''))}\n\n" + elif resp_type == "stderr": + yield f"event: stderr\ndata: {json.dumps(resp.get('value', ''))}\n\n" + elif resp_type == "error": + yield f"event: error\ndata: {json.dumps({'error': resp.get('error', ''), 'traceback': resp.get('traceback', '')})}\n\n" + break + + except Exception as e: + yield f"event: error\ndata: {json.dumps({'error': str(e)})}\n\n" + + return Response( + generate(), + mimetype="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + }, + ) + + +@app.route("/api/stream/exec", methods=["POST"]) +def api_stream_exec(): + """Queue code to execute during an active stream.""" + session_id = _get_session_id() + data = request.get_json(force=True) + code = data.get("code", "") + + session = get_or_create_session(session_id) + try: + session.send_message({"type": "stream-exec", "code": code}) + return jsonify({"status": "queued"}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + +@app.route("/api/stream/stop", methods=["POST"]) +def api_stream_stop(): + """Stop an active streaming session.""" + session_id = _get_session_id() + + session = get_or_create_session(session_id) + try: + session.send_message({"type": "stream-stop"}) + return jsonify({"status": "stopped"}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + +@app.route("/api/session", methods=["DELETE"]) +def api_session_delete(): + """Kill a session's worker subprocess.""" + session_id = _get_session_id() + remove_session(session_id) + return jsonify({"status": "terminated"}) + + +# --------------------------------------------------------------------------- +# Cleanup on exit +# --------------------------------------------------------------------------- + +@atexit.register +def _cleanup_all_sessions(): + with _sessions_lock: + for session in _sessions.values(): + session.kill() + _sessions.clear() + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + port = int(os.environ.get("PORT", 5000)) + debug = os.environ.get("FLASK_DEBUG", "1") == "1" + print(f"PathView Flask backend starting on port {port}") + app.run(host="0.0.0.0", port=port, debug=debug, threaded=True) diff --git a/server/requirements.txt b/server/requirements.txt new file mode 100644 index 0000000..3535b45 --- /dev/null +++ b/server/requirements.txt @@ -0,0 +1,2 @@ +flask>=3.0 +flask-cors>=4.0 diff --git a/server/worker.py b/server/worker.py index 76c2bdc..6040cc0 100644 --- a/server/worker.py +++ b/server/worker.py @@ -7,8 +7,9 @@ Same message protocol (REPLRequest/REPLResponse as JSON lines over stdin/stdout). Threading model: -- Reader thread: reads JSON from stdin, dispatches to main thread or queues for streaming -- Main thread: processes init/exec/eval synchronously; for streaming, runs the loop +- Main thread: reads stdin, processes init/exec/eval synchronously +- During streaming: a reader thread handles stream-stop and stream-exec + while the main thread runs the streaming loop - Stdout lock: thread-safe writing to stdout (protocol messages only) """ @@ -31,9 +32,6 @@ _streaming_active = False _streaming_code_queue = queue.Queue() -# Queue for messages from reader thread -> main thread -_message_queue = queue.Queue() - def send(response: dict) -> None: """Send a JSON response to the parent process via stdout.""" @@ -64,6 +62,17 @@ def _capture_output(func): return result +def read_message(): + """Read one JSON message from stdin. Returns None on EOF.""" + line = sys.stdin.readline() + if not line: + return None + line = line.strip() + if not line: + return read_message() # skip blank lines + return json.loads(line) + + def initialize() -> None: """Initialize the worker: import standard packages, capture clean globals.""" global _initialized, _namespace, _clean_globals @@ -123,6 +132,34 @@ def run(): send({"type": "error", "id": msg_id, "error": str(e), "traceback": tb}) +def _streaming_reader_thread(stop_event: threading.Event) -> None: + """Read stdin during streaming, handling stream-stop and stream-exec.""" + global _streaming_active + while not stop_event.is_set(): + line = sys.stdin.readline() + if not line: + # EOF — stop streaming and signal main loop to exit + _streaming_active = False + break + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + send({"type": "error", "error": f"Invalid JSON: {line}"}) + continue + + msg_type = msg.get("type") + if msg_type == "stream-stop": + _streaming_active = False + elif msg_type == "stream-exec": + code = msg.get("code") + if code and _streaming_active: + _streaming_code_queue.put(code) + # Other messages during streaming are ignored (shouldn't happen) + + def run_streaming_loop(msg_id: str, expr: str) -> None: """Run streaming loop - steps generator continuously and posts results.""" global _streaming_active @@ -139,6 +176,11 @@ def run_streaming_loop(msg_id: str, expr: str) -> None: except queue.Empty: break + # Start reader thread to handle stream-stop and stream-exec + stop_event = threading.Event() + reader = threading.Thread(target=_streaming_reader_thread, args=(stop_event,), daemon=True) + reader.start() + try: while _streaming_active: # Execute any queued code first (for runtime parameter changes) @@ -184,6 +226,7 @@ def run_step(): send({"type": "error", "id": msg_id, "error": str(e), "traceback": tb}) finally: _streaming_active = False + stop_event.set() # Always send done when loop ends send({"type": "stream-done", "id": msg_id}) @@ -194,43 +237,10 @@ def stop_streaming() -> None: _streaming_active = False -def reader_thread() -> None: - """Read JSON messages from stdin and dispatch to the main thread.""" - for line in sys.stdin: - line = line.strip() - if not line: - continue - try: - msg = json.loads(line) - except json.JSONDecodeError: - send({"type": "error", "error": f"Invalid JSON: {line}"}) - continue - - msg_type = msg.get("type") - - # stream-stop and stream-exec are handled directly (thread-safe) - if msg_type == "stream-stop": - stop_streaming() - elif msg_type == "stream-exec": - code = msg.get("code") - if code and _streaming_active: - _streaming_code_queue.put(code) - else: - # All other messages go to main thread - _message_queue.put(msg) - - # stdin closed — signal main thread to exit - _message_queue.put(None) - - def main() -> None: - """Main loop: process messages from the reader thread.""" - # Start the reader thread - t = threading.Thread(target=reader_thread, daemon=True) - t.start() - + """Main loop: read messages from stdin and process them.""" while True: - msg = _message_queue.get() + msg = read_message() if msg is None: # stdin closed, exit break @@ -257,9 +267,17 @@ def main() -> None: elif msg_type == "stream-start": if not msg_id or not isinstance(expr, str): raise ValueError("Invalid stream-start request: missing id or expr") - # Run in the main thread (blocking until done/stopped) + # Blocking: runs streaming loop, reader thread handles + # stream-stop and stream-exec during this time run_streaming_loop(msg_id, expr) + elif msg_type == "stream-stop": + stop_streaming() + + elif msg_type == "stream-exec": + if isinstance(code, str) and _streaming_active: + _streaming_code_queue.put(code) + else: raise ValueError(f"Unknown message type: {msg_type}") From a32539c00ae1cc6a2d05d4b7d1c88aa63e9f903b Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 11:43:44 +0100 Subject: [PATCH 03/27] Add FlaskBackend class implementing Backend interface via HTTP/SSE Co-Authored-By: KDW1 --- src/lib/pyodide/backend/flask/backend.ts | 457 +++++++++++++++++++++++ 1 file changed, 457 insertions(+) create mode 100644 src/lib/pyodide/backend/flask/backend.ts diff --git a/src/lib/pyodide/backend/flask/backend.ts b/src/lib/pyodide/backend/flask/backend.ts new file mode 100644 index 0000000..842d159 --- /dev/null +++ b/src/lib/pyodide/backend/flask/backend.ts @@ -0,0 +1,457 @@ +/** + * Flask Backend + * Implements the Backend interface using a Flask server with subprocess workers + */ + +import type { Backend, BackendState } from '../types'; +import { backendState } from '../state'; +import { TIMEOUTS } from '$lib/constants/python'; +import { STATUS_MESSAGES } from '$lib/constants/messages'; + +/** + * Flask Backend Implementation + * + * Communicates with a Flask server that manages Python subprocess workers. + * Each browser session gets its own isolated Python process on the server. + * Supports streaming via Server-Sent Events (SSE). + */ +export class FlaskBackend implements Backend { + private host: string; + private sessionId: string; + private messageId = 0; + private _isStreaming = false; + private streamAbortController: AbortController | null = null; + + // Stream state + private streamState: { + onData: ((data: unknown) => void) | null; + onDone: (() => void) | null; + onError: ((error: Error) => void) | null; + } = { onData: null, onDone: null, onError: null }; + + // Output callbacks + private stdoutCallback: ((value: string) => void) | null = null; + private stderrCallback: ((value: string) => void) | null = null; + + constructor(host: string) { + this.host = host.replace(/\/$/, ''); // strip trailing slash + // Get or create session ID from sessionStorage + const stored = typeof sessionStorage !== 'undefined' ? sessionStorage.getItem('flask-session-id') : null; + if (stored) { + this.sessionId = stored; + } else { + this.sessionId = crypto.randomUUID(); + if (typeof sessionStorage !== 'undefined') { + sessionStorage.setItem('flask-session-id', this.sessionId); + } + } + } + + // ------------------------------------------------------------------------- + // Lifecycle + // ------------------------------------------------------------------------- + + async init(): Promise { + const state = this.getState(); + if (state.initialized || state.loading) return; + + backendState.update((s) => ({ + ...s, + loading: true, + error: null, + progress: 'Connecting to Flask server...' + })); + + try { + // Health check + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), TIMEOUTS.INIT); + + const resp = await fetch(`${this.host}/api/health`, { + signal: controller.signal + }); + clearTimeout(timeout); + + if (!resp.ok) { + throw new Error(`Server health check failed: ${resp.status}`); + } + + // Trigger worker initialization with a no-op exec + backendState.update((s) => ({ ...s, progress: 'Initializing Python worker...' })); + await this.exec('pass', TIMEOUTS.INIT); + + backendState.update((s) => ({ + ...s, + initialized: true, + loading: false, + progress: STATUS_MESSAGES.READY + })); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + backendState.update((s) => ({ + ...s, + loading: false, + error: `Flask backend error: ${msg}` + })); + throw error; + } + } + + terminate(): void { + // Abort any active stream + if (this.streamAbortController) { + this.streamAbortController.abort(); + this.streamAbortController = null; + } + + // Clear stream state + this._isStreaming = false; + this.streamState = { onData: null, onDone: null, onError: null }; + + // Kill server-side session (fire and forget) + fetch(`${this.host}/api/session`, { + method: 'DELETE', + headers: { 'X-Session-ID': this.sessionId } + }).catch(() => {}); + + // Reset state + backendState.reset(); + } + + // ------------------------------------------------------------------------- + // State + // ------------------------------------------------------------------------- + + getState(): BackendState { + return backendState.get(); + } + + subscribe(callback: (state: BackendState) => void): () => void { + return backendState.subscribe(callback); + } + + isReady(): boolean { + return this.getState().initialized; + } + + isLoading(): boolean { + return this.getState().loading; + } + + getError(): string | null { + return this.getState().error; + } + + // ------------------------------------------------------------------------- + // Execution + // ------------------------------------------------------------------------- + + async exec(code: string, timeout: number = TIMEOUTS.SIMULATION): Promise { + const id = this.generateId(); + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + + try { + const resp = await fetch(`${this.host}/api/exec`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ id, code }), + signal: controller.signal + }); + + const data = await resp.json(); + + // Forward stdout/stderr from response + if (data.stdout && this.stdoutCallback) this.stdoutCallback(data.stdout); + if (data.stderr && this.stderrCallback) this.stderrCallback(data.stderr); + + if (data.type === 'error') { + const errorMsg = data.traceback ? `${data.error}\n${data.traceback}` : data.error; + throw new Error(errorMsg); + } + } catch (error) { + if (error instanceof DOMException && error.name === 'AbortError') { + throw new Error('Execution timeout'); + } + throw error; + } finally { + clearTimeout(timeoutId); + } + } + + async evaluate(expr: string, timeout: number = TIMEOUTS.SIMULATION): Promise { + const id = this.generateId(); + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + + try { + const resp = await fetch(`${this.host}/api/eval`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ id, expr }), + signal: controller.signal + }); + + const data = await resp.json(); + + // Forward stdout/stderr from response + if (data.stdout && this.stdoutCallback) this.stdoutCallback(data.stdout); + if (data.stderr && this.stderrCallback) this.stderrCallback(data.stderr); + + if (data.type === 'error') { + const errorMsg = data.traceback ? `${data.error}\n${data.traceback}` : data.error; + throw new Error(errorMsg); + } + + if (data.value === undefined) { + throw new Error('No value returned from eval'); + } + + return JSON.parse(data.value) as T; + } catch (error) { + if (error instanceof DOMException && error.name === 'AbortError') { + throw new Error('Evaluation timeout'); + } + throw error; + } finally { + clearTimeout(timeoutId); + } + } + + // ------------------------------------------------------------------------- + // Streaming + // ------------------------------------------------------------------------- + + startStreaming( + expr: string, + onData: (data: T) => void, + onDone: () => void, + onError: (error: Error) => void + ): void { + if (!this.isReady()) { + onError(new Error('Backend not initialized')); + return; + } + + // Stop any existing stream + if (this._isStreaming) { + this.stopStreaming(); + } + + const id = this.generateId(); + this._isStreaming = true; + this.streamState = { + onData: onData as (data: unknown) => void, + onDone, + onError + }; + + this.streamAbortController = new AbortController(); + + // Start SSE stream + this.consumeSSEStream(id, expr, this.streamAbortController.signal); + } + + stopStreaming(): void { + if (!this._isStreaming) return; + + // Send stop to server + fetch(`${this.host}/api/stream/stop`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + } + }).catch(() => {}); + + // Abort the SSE connection + if (this.streamAbortController) { + this.streamAbortController.abort(); + this.streamAbortController = null; + } + } + + isStreaming(): boolean { + return this._isStreaming; + } + + execDuringStreaming(code: string): void { + if (!this._isStreaming) { + console.warn('Cannot exec during streaming: no active stream'); + return; + } + + // Fire and forget + fetch(`${this.host}/api/stream/exec`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ code }) + }).catch(() => {}); + } + + // ------------------------------------------------------------------------- + // Output Callbacks + // ------------------------------------------------------------------------- + + onStdout(callback: (value: string) => void): void { + this.stdoutCallback = callback; + } + + onStderr(callback: (value: string) => void): void { + this.stderrCallback = callback; + } + + // ------------------------------------------------------------------------- + // Private Methods + // ------------------------------------------------------------------------- + + private generateId(): string { + return `repl_${++this.messageId}`; + } + + /** + * Consume an SSE stream from /api/stream, dispatching events to callbacks. + */ + private async consumeSSEStream(id: string, expr: string, signal: AbortSignal): Promise { + try { + const resp = await fetch(`${this.host}/api/stream`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ id, expr }), + signal + }); + + if (!resp.ok || !resp.body) { + throw new Error(`Stream request failed: ${resp.status}`); + } + + const reader = resp.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + let currentEvent = ''; + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + // Process complete SSE messages (separated by double newlines) + const parts = buffer.split('\n\n'); + buffer = parts.pop() || ''; // Keep incomplete last part + + for (const part of parts) { + if (!part.trim()) continue; + + // Parse SSE fields + let eventType = ''; + let eventData = ''; + + for (const line of part.split('\n')) { + if (line.startsWith('event: ')) { + eventType = line.slice(7); + } else if (line.startsWith('data: ')) { + eventData = line.slice(6); + } + } + + if (!eventType) continue; + + this.handleSSEEvent(eventType, eventData); + + if (eventType === 'done' || eventType === 'error') { + return; + } + } + } + } catch (error) { + if (signal.aborted) { + // Aborted by stopStreaming — call onDone + this._isStreaming = false; + if (this.streamState.onDone) { + this.streamState.onDone(); + } + this.streamState = { onData: null, onDone: null, onError: null }; + return; + } + this._isStreaming = false; + if (this.streamState.onError) { + this.streamState.onError(error instanceof Error ? error : new Error(String(error))); + } + this.streamState = { onData: null, onDone: null, onError: null }; + } + } + + private handleSSEEvent(eventType: string, data: string): void { + switch (eventType) { + case 'data': { + if (this.streamState.onData) { + try { + const parsed = JSON.parse(data); + this.streamState.onData(parsed); + } catch { + // Ignore parse errors + } + } + break; + } + case 'stdout': { + if (this.stdoutCallback) { + try { + this.stdoutCallback(JSON.parse(data)); + } catch { + this.stdoutCallback(data); + } + } + break; + } + case 'stderr': { + if (this.stderrCallback) { + try { + this.stderrCallback(JSON.parse(data)); + } catch { + this.stderrCallback(data); + } + } + break; + } + case 'done': { + this._isStreaming = false; + if (this.streamState.onDone) { + this.streamState.onDone(); + } + this.streamState = { onData: null, onDone: null, onError: null }; + break; + } + case 'error': { + this._isStreaming = false; + if (this.streamState.onError) { + try { + const parsed = JSON.parse(data); + const msg = parsed.traceback + ? `${parsed.error}\n${parsed.traceback}` + : parsed.error || 'Unknown error'; + this.streamState.onError(new Error(msg)); + } catch { + this.streamState.onError(new Error(data || 'Stream error')); + } + } + this.streamState = { onData: null, onDone: null, onError: null }; + backendState.update((s) => ({ ...s, error: 'Stream error' })); + break; + } + } + } +} From 36d8496ae6b9a7e0258380e04f879c85f585165f Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 11:46:18 +0100 Subject: [PATCH 04/27] Integrate FlaskBackend into registry with URL param switching Co-Authored-By: KDW1 --- package.json | 3 ++- src/lib/pyodide/backend/index.ts | 25 +++++++++++++++++++++++-- src/lib/pyodide/backend/registry.ts | 14 ++++++++++++-- src/routes/+page.svelte | 4 ++++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index d8024cc..a2a30f9 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "lint": "eslint .", - "format": "prettier --write ." + "format": "prettier --write .", + "server": "python server/app.py" }, "devDependencies": { "@sveltejs/adapter-static": "^3.0.0", diff --git a/src/lib/pyodide/backend/index.ts b/src/lib/pyodide/backend/index.ts index aea26ee..5a227a1 100644 --- a/src/lib/pyodide/backend/index.ts +++ b/src/lib/pyodide/backend/index.ts @@ -20,21 +20,42 @@ export { getBackendType, hasBackend, terminateBackend, + setFlaskHost, type BackendType } from './registry'; -// Re-export PyodideBackend for direct use if needed +// Re-export backend implementations export { PyodideBackend } from './pyodide/backend'; +export { FlaskBackend } from './flask/backend'; // ============================================================================ // Backward-Compatible Convenience Functions // These delegate to the current backend and maintain API compatibility // ============================================================================ -import { getBackend } from './registry'; +import { getBackend, switchBackend, setFlaskHost } from './registry'; import { backendState } from './state'; import { consoleStore } from '$lib/stores/console'; +/** + * Initialize backend from URL parameters. + * Reads `?backend=flask` and `?host=...` from the current URL. + * Call this early in page mount, before any backend usage. + */ +export function initBackendFromUrl(): void { + if (typeof window === 'undefined') return; + const params = new URLSearchParams(window.location.search); + const backendParam = params.get('backend'); + const hostParam = params.get('host'); + + if (backendParam === 'flask') { + if (hostParam) { + setFlaskHost(hostParam); + } + switchBackend('flask'); + } +} + // Alias for backward compatibility export const replState = { subscribe: backendState.subscribe diff --git a/src/lib/pyodide/backend/registry.ts b/src/lib/pyodide/backend/registry.ts index e1eaadd..263ae5b 100644 --- a/src/lib/pyodide/backend/registry.ts +++ b/src/lib/pyodide/backend/registry.ts @@ -5,11 +5,20 @@ import type { Backend } from './types'; import { PyodideBackend } from './pyodide/backend'; +import { FlaskBackend } from './flask/backend'; -export type BackendType = 'pyodide' | 'local' | 'remote'; +export type BackendType = 'pyodide' | 'flask' | 'remote'; let currentBackend: Backend | null = null; let currentBackendType: BackendType | null = null; +let flaskHost = 'http://localhost:5000'; + +/** + * Set the Flask backend host URL + */ +export function setFlaskHost(host: string): void { + flaskHost = host; +} /** * Get the current backend, creating a Pyodide backend if none exists @@ -29,7 +38,8 @@ export function createBackend(type: BackendType): Backend { switch (type) { case 'pyodide': return new PyodideBackend(); - case 'local': + case 'flask': + return new FlaskBackend(flaskHost); case 'remote': throw new Error(`Backend type '${type}' not yet implemented`); default: diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 7aaad8b..268e53f 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -40,6 +40,7 @@ import { openEventDialog } from '$lib/stores/eventDialog'; import type { MenuItemType } from '$lib/components/ContextMenu.svelte'; import { pyodideState, simulationState, initPyodide, stopSimulation, continueStreamingSimulation } from '$lib/pyodide/bridge'; + import { initBackendFromUrl } from '$lib/pyodide/backend'; import { runGraphStreamingSimulation, validateGraphSimulation } from '$lib/pyodide/pathsimRunner'; import { consoleStore } from '$lib/stores/console'; import { newGraph, saveFile, saveAsFile, setupAutoSave, clearAutoSave, debouncedAutoSave, openImportDialog, importFromUrl, currentFileName } from '$lib/schema/fileOps'; @@ -381,6 +382,9 @@ const continueTooltip = { text: "Continue", shortcut: "Shift+Enter" }; onMount(() => { + // Check URL params for backend selection (e.g. ?backend=flask&host=http://localhost:5000) + initBackendFromUrl(); + // Subscribe to stores (with cleanup) const unsubPinnedPreviews = pinnedPreviewsStore.subscribe((pinned) => { showPinnedPreviews = pinned; From 4b4df3d0750465f7938066051289e0b69efaae1b Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 11:58:25 +0100 Subject: [PATCH 05/27] Add dynamic package installation from PYTHON_PACKAGES config Co-Authored-By: KDW1 --- server/app.py | 24 ++++++++++- server/worker.py | 55 ++++++++++++++++++++++-- src/lib/pyodide/backend/flask/backend.ts | 35 ++++++++++++++- 3 files changed, 107 insertions(+), 7 deletions(-) diff --git a/server/app.py b/server/app.py index 76cb254..6f8eff4 100644 --- a/server/app.py +++ b/server/app.py @@ -67,12 +67,15 @@ def read_line(self) -> dict | None: return None return json.loads(line.strip()) - def ensure_initialized(self) -> list[dict]: + def ensure_initialized(self, packages: list[dict] | None = None) -> list[dict]: """Initialize the worker if not already done. Returns any messages received.""" if self._initialized: return [] messages = [] - self.send_message({"type": "init"}) + init_msg = {"type": "init"} + if packages: + init_msg["packages"] = packages + self.send_message(init_msg) while True: resp = self.read_line() if resp is None: @@ -161,6 +164,23 @@ def health(): return jsonify({"status": "ok"}) +@app.route("/api/init", methods=["POST"]) +def api_init(): + """Initialize a session's worker with packages from the frontend config.""" + session_id = _get_session_id() + data = request.get_json(force=True) + packages = data.get("packages", []) + + session = get_or_create_session(session_id) + with session.lock: + try: + messages = session.ensure_initialized(packages=packages) + # Return all progress/stdout/stderr messages collected during init + return jsonify({"type": "ready", "messages": messages}) + except Exception as e: + return jsonify({"type": "error", "error": str(e)}), 500 + + @app.route("/api/exec", methods=["POST"]) def api_exec(): """Execute Python code in the session's worker.""" diff --git a/server/worker.py b/server/worker.py index 6040cc0..e73ba13 100644 --- a/server/worker.py +++ b/server/worker.py @@ -16,6 +16,7 @@ import sys import io import json +import subprocess import threading import traceback import queue @@ -73,8 +74,43 @@ def read_message(): return json.loads(line) -def initialize() -> None: - """Initialize the worker: import standard packages, capture clean globals.""" +def _install_package(pip_spec: str, pre: bool = False) -> None: + """Install a package via pip if not already available.""" + cmd = [sys.executable, "-m", "pip", "install", pip_spec, "--quiet"] + if pre: + cmd.append("--pre") + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"pip install failed for {pip_spec}: {result.stderr.strip()}") + + +def _ensure_package(pkg: dict) -> None: + """Ensure a package is installed and importable. Mirrors the Pyodide worker loop.""" + import_name = pkg.get("import", "") + pip_spec = pkg.get("pip", import_name) + required = pkg.get("required", False) + pre = pkg.get("pre", False) + + send({"type": "progress", "value": f"Installing {import_name}..."}) + + try: + # Try importing first — skip pip if already installed + exec(f"import {import_name}", _namespace) + except ImportError: + # Not installed — pip install then import + _install_package(pip_spec, pre) + exec(f"import {import_name}", _namespace) + + # Log version if available + try: + version = eval(f"{import_name}.__version__", _namespace) + send({"type": "stdout", "value": f"{import_name} {version} loaded successfully\n"}) + except Exception: + send({"type": "stdout", "value": f"{import_name} loaded successfully\n"}) + + +def initialize(packages: list[dict] | None = None) -> None: + """Initialize the worker: install packages, import standard libs, capture clean globals.""" global _initialized, _namespace, _clean_globals if _initialized: @@ -89,6 +125,19 @@ def initialize() -> None: exec("import gc", _namespace) exec("import json", _namespace) + # Install and import packages from the frontend config (single source of truth) + if packages: + send({"type": "progress", "value": "Installing dependencies..."}) + for pkg in packages: + try: + _ensure_package(pkg) + except Exception as e: + if pkg.get("required", False): + raise RuntimeError( + f"Failed to install required package {pkg.get('pip', pkg.get('import', '?'))}: {e}" + ) + send({"type": "stderr", "value": f"Optional package {pkg.get('import', '?')} failed: {e}\n"}) + # Capture clean state for later cleanup _clean_globals = set(_namespace.keys()) @@ -252,7 +301,7 @@ def main() -> None: try: if msg_type == "init": - initialize() + initialize(packages=msg.get("packages")) elif msg_type == "exec": if not msg_id or not isinstance(code, str): diff --git a/src/lib/pyodide/backend/flask/backend.ts b/src/lib/pyodide/backend/flask/backend.ts index 842d159..7f91c35 100644 --- a/src/lib/pyodide/backend/flask/backend.ts +++ b/src/lib/pyodide/backend/flask/backend.ts @@ -7,6 +7,7 @@ import type { Backend, BackendState } from '../types'; import { backendState } from '../state'; import { TIMEOUTS } from '$lib/constants/python'; import { STATUS_MESSAGES } from '$lib/constants/messages'; +import { PYTHON_PACKAGES } from '$lib/constants/dependencies'; /** * Flask Backend Implementation @@ -76,9 +77,39 @@ export class FlaskBackend implements Backend { throw new Error(`Server health check failed: ${resp.status}`); } - // Trigger worker initialization with a no-op exec + // Initialize worker with packages from the shared config (single source of truth) backendState.update((s) => ({ ...s, progress: 'Initializing Python worker...' })); - await this.exec('pass', TIMEOUTS.INIT); + + const initController = new AbortController(); + const initTimeout = setTimeout(() => initController.abort(), TIMEOUTS.INIT); + + const initResp = await fetch(`${this.host}/api/init`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ packages: PYTHON_PACKAGES }), + signal: initController.signal + }); + clearTimeout(initTimeout); + + const initData = await initResp.json(); + + if (initData.type === 'error') { + throw new Error(initData.error); + } + + // Forward any stdout/stderr messages from init + if (initData.messages) { + for (const msg of initData.messages) { + if (msg.type === 'stdout' && this.stdoutCallback) this.stdoutCallback(msg.value); + if (msg.type === 'stderr' && this.stderrCallback) this.stderrCallback(msg.value); + if (msg.type === 'progress') { + backendState.update((s) => ({ ...s, progress: msg.value })); + } + } + } backendState.update((s) => ({ ...s, From b183c18c68fc14658be38b7767bccdbb323a44fe Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 12:03:19 +0100 Subject: [PATCH 06/27] Update README with Flask backend documentation Co-Authored-By: KDW1 --- README.md | 106 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 88 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b7f7f82..c9ccad0 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ # PathView - System Modeling in the Browser -A web-based visual node editor for building and simulating dynamic systems with [PathSim](https://github.com/pathsim/pathsim) as the backend. Runs entirely in the browser via Pyodide - no server required. The UI is hosted at [view.pathsim.org](https://view.pathsim.org), free to use for everyone. +A web-based visual node editor for building and simulating dynamic systems with [PathSim](https://github.com/pathsim/pathsim) as the backend. Runs entirely in the browser via Pyodide by default — no server required. Optionally, a Flask backend enables server-side Python execution with any packages (including those with native dependencies that Pyodide can't run). The UI is hosted at [view.pathsim.org](https://view.pathsim.org), free to use for everyone. ## Tech Stack @@ -24,6 +24,15 @@ npm install npm run dev ``` +To use the Flask backend (server-side Python): + +```bash +pip install -r server/requirements.txt +npm run server # Start Flask backend on port 5000 +npm run dev # Start Vite dev server (separate terminal) +# Open http://localhost:5173/?backend=flask +``` + For production: ```bash @@ -61,7 +70,8 @@ src/ │ ├── routing/ # Orthogonal wire routing (A* pathfinding) │ ├── pyodide/ # Python runtime (backend, bridge) │ │ └── backend/ # Modular backend system (registry, state, types) -│ │ └── pyodide/ # Pyodide Web Worker implementation +│ │ ├── pyodide/ # Pyodide Web Worker implementation +│ │ └── flask/ # Flask HTTP/SSE backend implementation │ ├── schema/ # File I/O (save/load, component export) │ ├── simulation/ # Simulation metadata │ │ └── generated/ # Auto-generated defaults @@ -72,6 +82,11 @@ src/ ├── routes/ # SvelteKit pages └── app.css # Global styles with CSS variables +server/ +├── app.py # Flask server (subprocess management, HTTP routes) +├── worker.py # REPL worker subprocess (Python execution) +└── requirements.txt # Server Python dependencies + scripts/ ├── config/ # Configuration files for extraction │ ├── schemas/ # JSON schemas for validation @@ -100,8 +115,8 @@ scripts/ │ v ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ Plot/Console │<────│ bridge.ts │<────│ REPL Worker │ -│ (results) │ │ (queue + rAF) │ │ (Pyodide) │ +│ Plot/Console │<────│ bridge.ts │<────│ Backend │ +│ (results) │ │ (queue + rAF) │ │ (Pyodide/Flask) │ └─────────────────┘ └─────────────────┘ └─────────────────┘ ``` @@ -153,6 +168,7 @@ Key files: `src/lib/routing/` (pathfinder, grid builder, route calculator) | **Backend** | Modular Python execution interface | `pyodide/backend/` | | **Backend Registry** | Factory for swappable backends | `pyodide/backend/registry.ts` | | **PyodideBackend** | Web Worker Pyodide implementation | `pyodide/backend/pyodide/` | +| **FlaskBackend** | HTTP/SSE Flask server implementation | `pyodide/backend/flask/` | | **Simulation Bridge** | High-level simulation API | `pyodide/bridge.ts` | | **Schema** | File/component save/load operations | `schema/fileOps.ts`, `schema/componentOps.ts` | | **Export Utils** | SVG/CSV/Python file downloads | `utils/download.ts`, `export/svg/`, `utils/csvExport.ts` | @@ -326,29 +342,36 @@ The Python runtime uses a modular backend architecture, allowing different execu ┌──────────────┼──────────────┐ ▼ ▼ ▼ ┌───────────┐ ┌───────────┐ ┌───────────┐ - │ Pyodide │ │ Local │ │ Remote │ + │ Pyodide │ │ Flask │ │ Remote │ │ Backend │ │ Backend │ │ Backend │ - │ (Worker) │ │ (Flask) │ │ (Server) │ + │ (default) │ │ (HTTP) │ │ (future) │ └───────────┘ └───────────┘ └───────────┘ - │ (future) (future) - ▼ - ┌───────────┐ - │ Web Worker│ - │ (Pyodide) │ - └───────────┘ + │ │ + ▼ ▼ + ┌───────────┐ ┌───────────┐ + │ Web Worker│ │ Flask │──> Python subprocess + │ (Pyodide) │ │ Server │ (one per session) + └───────────┘ └───────────┘ ``` ### Backend Registry ```typescript -import { getBackend, switchBackend } from '$lib/pyodide/backend'; +import { getBackend, switchBackend, setFlaskHost } from '$lib/pyodide/backend'; // Get current backend (defaults to Pyodide) const backend = getBackend(); -// Switch to a different backend type (future) -// switchBackend('local'); // Use local Python via Flask -// switchBackend('remote'); // Use remote server +// Switch to Flask backend +setFlaskHost('http://localhost:5000'); +switchBackend('flask'); +``` + +Backend selection can also be controlled via URL parameters: + +``` +http://localhost:5173/?backend=flask # Flask on default port +http://localhost:5173/?backend=flask&host=http://myserver:5000 # Custom host ``` ### REPL Protocol @@ -426,6 +449,52 @@ await stopSimulation(); execDuringStreaming('source.amplitude = 2.0'); ``` +### Flask Backend + +The Flask backend enables server-side Python execution for packages that Pyodide can't run (e.g., FESTIM or other packages with native C/Fortran dependencies). It mirrors the Web Worker architecture: one subprocess per session with the same REPL protocol. + +``` +Browser Tab Flask Server Worker Subprocess +┌──────────────┐ ┌──────────────────┐ ┌──────────────────┐ +│ FlaskBackend │ HTTP/SSE │ app.py │ stdin │ worker.py │ +│ exec() │──POST────────→│ route → session │──JSON───→│ exec(code, ns) │ +│ eval() │──POST────────→│ subprocess mgr │──JSON───→│ eval(expr, ns) │ +│ stream() │──POST (SSE)──→│ pipe SSE relay │←─JSON────│ streaming loop │ +│ inject() │──POST────────→│ → code queue │──JSON───→│ queue drain │ +│ stop() │──POST────────→│ → stop flag │──JSON───→│ stop check │ +└──────────────┘ └──────────────────┘ └──────────────────┘ +``` + +**Setup:** + +```bash +pip install -r server/requirements.txt +npm run server # Starts Flask on port 5000 +npm run dev # Starts Vite dev server (separate terminal) +``` + +Then open `http://localhost:5173/?backend=flask`. + +**Key properties:** +- **Process isolation** — each session gets its own Python subprocess +- **Namespace persistence** — variables persist across exec/eval calls within a session +- **Dynamic packages** — packages from `PYTHON_PACKAGES` (the same config used by Pyodide) are pip-installed on first init +- **Session TTL** — stale sessions cleaned up after 1 hour of inactivity +- **Streaming** — simulations stream via SSE, with the same code injection support as Pyodide + +**API routes:** + +| Route | Method | Action | +|-------|--------|--------| +| `/api/health` | GET | Health check | +| `/api/init` | POST | Initialize worker with packages | +| `/api/exec` | POST | Execute Python code | +| `/api/eval` | POST | Evaluate expression, return JSON | +| `/api/stream` | POST | Start streaming simulation (SSE) | +| `/api/stream/exec` | POST | Inject code during streaming | +| `/api/stream/stop` | POST | Stop streaming | +| `/api/session` | DELETE | Kill session subprocess | + --- ## State Management @@ -583,7 +652,8 @@ https://view.pathsim.org/?modelgh=pathsim/pathview/static/examples/feedback-syst | Script | Purpose | |--------|---------| -| `npm run dev` | Start development server | +| `npm run dev` | Start Vite development server | +| `npm run server` | Start Flask backend server (port 5000) | | `npm run build` | Production build | | `npm run preview` | Preview production build | | `npm run check` | TypeScript/Svelte type checking | @@ -665,7 +735,7 @@ Port labels show the name of each input/output port alongside the node. Toggle g 2. **Subsystems are nested graphs** - The Interface node inside a subsystem mirrors its parent's ports (inverted direction). -3. **No server required** - Everything runs client-side via Pyodide WebAssembly. +3. **No server required by default** - Everything runs client-side via Pyodide. The optional Flask backend enables server-side execution for packages with native dependencies. 4. **Registry pattern** - Nodes and events are registered centrally for extensibility. From 7491986d30d2ae08600a35790ed85658be7d4863 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 12:15:02 +0100 Subject: [PATCH 07/27] Add backend protocol and toolbox integration spec docs Co-Authored-By: KDW1 --- docs/backend-protocol-spec.md | 1151 +++++++++++++++++++++++++++++++++ docs/toolbox-spec.md | 699 ++++++++++++++++++++ 2 files changed, 1850 insertions(+) create mode 100644 docs/backend-protocol-spec.md create mode 100644 docs/toolbox-spec.md diff --git a/docs/backend-protocol-spec.md b/docs/backend-protocol-spec.md new file mode 100644 index 0000000..6a4df23 --- /dev/null +++ b/docs/backend-protocol-spec.md @@ -0,0 +1,1151 @@ +# PathView REPL Backend Protocol Specification + +**Version:** 1.0.0 +**Status:** Normative + +The PathView REPL backend protocol defines the contract that any Python execution backend must implement. It covers the TypeScript interface, the wire-level message protocol between main thread and worker, the HTTP API used by the Flask reference implementation, and the streaming semantics that enable live simulation. + +This document is the authoritative reference for anyone building a new backend (remote server, cloud worker, WebSocket bridge, etc.). + +--- + +## Table of Contents + +- [1. Overview](#1-overview) +- [2. Backend Interface](#2-backend-interface) + - [2.1 Lifecycle Methods](#21-lifecycle-methods) + - [2.2 State Methods](#22-state-methods) + - [2.3 Execution Methods](#23-execution-methods) + - [2.4 Streaming Methods](#24-streaming-methods) + - [2.5 Output Callbacks](#25-output-callbacks) +- [3. REPL Message Protocol](#3-repl-message-protocol) + - [3.1 Request Messages (main -> worker)](#31-request-messages-main---worker) + - [3.2 Response Messages (worker -> main)](#32-response-messages-worker---main) +- [4. Message Flows](#4-message-flows) + - [4.1 Init Flow](#41-init-flow) + - [4.2 Exec Flow](#42-exec-flow) + - [4.3 Eval Flow](#43-eval-flow) + - [4.4 Streaming Flow](#44-streaming-flow) + - [4.5 Code Injection During Streaming](#45-code-injection-during-streaming) + - [4.6 Stop Streaming](#46-stop-streaming) +- [5. Streaming Semantics](#5-streaming-semantics) +- [6. HTTP API (Flask Reference Implementation)](#6-http-api-flask-reference-implementation) + - [6.1 Route Summary](#61-route-summary) + - [6.2 Route Details](#62-route-details) +- [7. SSE Event Format](#7-sse-event-format) +- [8. State Management](#8-state-management) +- [9. Backend Registration](#9-backend-registration) +- [10. Worked Example](#10-worked-example) + - [10.1 Abstract Message Flow](#101-abstract-message-flow) + - [10.2 HTTP Equivalents](#102-http-equivalents) + +--- + +## 1. Overview + +PathView uses a modular backend system for Python execution. The `Backend` interface defines a transport-agnostic contract that decouples the UI from the execution environment. Two implementations currently exist: + +| Backend | Transport | Environment | +|---------|-----------|-------------| +| `PyodideBackend` | Web Worker `postMessage` | Browser (Pyodide WASM) | +| `FlaskBackend` | HTTP + SSE | Local/remote Flask server | + +The `Backend` interface is defined in `src/lib/pyodide/backend/types.ts`. Implementations are registered in `src/lib/pyodide/backend/registry.ts` and re-exported from `src/lib/pyodide/backend/index.ts`. + +This spec defines the protocol at two levels: + +1. **Abstract level** -- the TypeScript `Backend` interface and the `REPLRequest`/`REPLResponse` message types. +2. **Wire level** -- the HTTP routes and SSE event format used by HTTP-based backends. + +Any new backend must implement the abstract interface. HTTP-based backends should additionally follow the wire-level conventions documented in sections 6 and 7. + +--- + +## 2. Backend Interface + +```typescript +interface Backend { + // Lifecycle + init(): Promise; + terminate(): void; + + // State + getState(): BackendState; + subscribe(callback: (state: BackendState) => void): () => void; + isReady(): boolean; + isLoading(): boolean; + getError(): string | null; + + // Execution + exec(code: string, timeout?: number): Promise; + evaluate(expr: string, timeout?: number): Promise; + + // Streaming + startStreaming( + expr: string, + onData: (data: T) => void, + onDone: () => void, + onError: (error: Error) => void + ): void; + stopStreaming(): void; + isStreaming(): boolean; + execDuringStreaming(code: string): void; + + // Output + onStdout(callback: (value: string) => void): void; + onStderr(callback: (value: string) => void): void; +} + +interface BackendState { + initialized: boolean; + loading: boolean; + error: string | null; + progress: string; +} +``` + +### 2.1 Lifecycle Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `init` | `init(): Promise` | Initialize the backend. Load the runtime, connect to the server, install packages, etc. The promise resolves when the backend is ready to execute code. Must set `BackendState.loading = true` at the start and `initialized = true` on success. Called once at application startup. Idempotent -- calling it when already initialized or loading is a no-op. | +| `terminate` | `terminate(): void` | Tear down the backend and release all resources. Reject pending requests, abort active streams, destroy workers or connections. Must call `backendState.reset()` to return state to initial values. | + +### 2.2 State Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `getState` | `getState(): BackendState` | Return a snapshot of the current backend state. | +| `subscribe` | `subscribe(callback: (state: BackendState) => void): () => void` | Subscribe to state changes. Returns an unsubscribe function. Delegates to the shared `backendState` Svelte store. | +| `isReady` | `isReady(): boolean` | Return `true` if `initialized` is `true`. Shorthand for `getState().initialized`. | +| `isLoading` | `isLoading(): boolean` | Return `true` if the backend is currently initializing. Shorthand for `getState().loading`. | +| `getError` | `getError(): string \| null` | Return the current error message, or `null` if no error. Shorthand for `getState().error`. | + +### 2.3 Execution Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `exec` | `exec(code: string, timeout?: number): Promise` | Execute Python code with no return value. The promise resolves on success and rejects on error. The `timeout` parameter (milliseconds) is optional; implementations should default to a reasonable value. Called for code setup, imports, variable definitions, and simulation construction. | +| `evaluate` | `evaluate(expr: string, timeout?: number): Promise` | Evaluate a Python expression and return the result as a parsed JSON value. The backend must serialize the result to JSON on the Python side and deserialize it on the TypeScript side. Rejects if the expression errors or the result is not JSON-serializable. | + +### 2.4 Streaming Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `startStreaming` | `startStreaming(expr: string, onData: (data: T) => void, onDone: () => void, onError: (error: Error) => void): void` | Begin an autonomous streaming loop. The backend repeatedly evaluates `expr`, calling `onData` with each parsed JSON result. The loop continues until the expression returns `{done: true}`, `stopStreaming()` is called, or an error occurs. `onDone` is always called when the loop exits (regardless of reason). `onError` is called if the main expression throws. If a stream is already active, it is stopped before the new one begins. | +| `stopStreaming` | `stopStreaming(): void` | Request the streaming loop to stop. The backend finishes the current step and then sends `stream-done`. This is a request, not an immediate abort -- `onDone` will still fire. | +| `isStreaming` | `isStreaming(): boolean` | Return `true` if a streaming loop is currently active. | +| `execDuringStreaming` | `execDuringStreaming(code: string): void` | Queue Python code to be executed between streaming steps. The code is drained and executed before the next evaluation of the stream expression. Used for runtime parameter changes and event injection. Errors in queued code are reported via stderr but do not stop the stream. No-op if no stream is active. | + +### 2.5 Output Callbacks + +| Method | Signature | Description | +|--------|-----------|-------------| +| `onStdout` | `onStdout(callback: (value: string) => void): void` | Register a callback for captured stdout output. Only one callback is active at a time (last registration wins). Called during `exec`, `evaluate`, and streaming whenever Python code writes to stdout. | +| `onStderr` | `onStderr(callback: (value: string) => void): void` | Register a callback for captured stderr output. Same semantics as `onStdout`. | + +--- + +## 3. REPL Message Protocol + +The REPL message protocol defines the typed messages exchanged between the main thread and the execution worker. For the `PyodideBackend`, these are `postMessage` payloads. For the `FlaskBackend`, they are mapped onto HTTP request/response bodies and SSE events. The types are defined in `src/lib/pyodide/backend/types.ts`. + +### 3.1 Request Messages (main -> worker) + +#### `init` + +Initialize the Python runtime and install packages. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"init"` | yes | Message type discriminant. | + +The Pyodide worker reads its package list from the compiled-in `PYTHON_PACKAGES` constant. HTTP-based backends receive the package list in the request body (see [Section 6](#6-http-api-flask-reference-implementation)). + +#### `exec` + +Execute Python code (no return value expected). + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"exec"` | yes | Message type discriminant. | +| `id` | `string` | yes | Unique request ID for correlating the response. Convention: `"repl_N"`. | +| `code` | `string` | yes | Python code to execute. | + +#### `eval` + +Evaluate a Python expression and return the JSON-serialized result. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"eval"` | yes | Message type discriminant. | +| `id` | `string` | yes | Unique request ID. | +| `expr` | `string` | yes | Python expression to evaluate. The result must be JSON-serializable. | + +#### `stream-start` + +Begin an autonomous streaming loop. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stream-start"` | yes | Message type discriminant. | +| `id` | `string` | yes | Unique stream ID. All `stream-data` and `stream-done` responses reference this ID. | +| `expr` | `string` | yes | Python expression to evaluate each iteration. Should return `{done: bool, result: any}`. | + +#### `stream-stop` + +Request the streaming loop to stop. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stream-stop"` | yes | Message type discriminant. | + +No additional fields. The worker finishes the current iteration and then sends `stream-done`. + +#### `stream-exec` + +Queue code for execution between streaming steps. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stream-exec"` | yes | Message type discriminant. | +| `code` | `string` | yes | Python code to queue. Executed before the next evaluation of the stream expression. | + +### 3.2 Response Messages (worker -> main) + +#### `ready` + +Sent after successful initialization. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"ready"` | yes | Message type discriminant. | + +#### `ok` + +Sent after successful `exec` completion. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"ok"` | yes | Message type discriminant. | +| `id` | `string` | yes | The request ID from the originating `exec` request. | + +#### `value` + +Sent after successful `eval` completion. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"value"` | yes | Message type discriminant. | +| `id` | `string` | yes | The request ID from the originating `eval` request. | +| `value` | `string` | yes | JSON-serialized result of the evaluated expression. The main thread parses this with `JSON.parse()`. | + +#### `error` + +Sent when any operation fails. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"error"` | yes | Message type discriminant. | +| `id` | `string` | no | The request ID of the failed operation. May be absent for global errors (e.g., init failures). | +| `error` | `string` | yes | Human-readable error message. | +| `traceback` | `string` | no | Python traceback string, if available. | + +#### `stdout` + +Captured standard output from Python execution. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stdout"` | yes | Message type discriminant. | +| `value` | `string` | yes | The captured output text. | + +#### `stderr` + +Captured standard error from Python execution. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stderr"` | yes | Message type discriminant. | +| `value` | `string` | yes | The captured error text. | + +#### `progress` + +Loading progress updates during initialization. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"progress"` | yes | Message type discriminant. | +| `value` | `string` | yes | Human-readable progress message (e.g., `"Installing pathsim..."`). | + +#### `stream-data` + +A single result from the streaming loop. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stream-data"` | yes | Message type discriminant. | +| `id` | `string` | yes | The stream ID from the originating `stream-start` request. | +| `value` | `string` | yes | JSON-serialized step result. Convention: `{"done": false, "result": {...}}`. | + +#### `stream-done` + +The streaming loop has exited. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stream-done"` | yes | Message type discriminant. | +| `id` | `string` | yes | The stream ID from the originating `stream-start` request. | + +This message is **always** sent when the loop exits, whether it completed naturally (`done: true`), was stopped via `stream-stop`, or errored. + +--- + +## 4. Message Flows + +### 4.1 Init Flow + +``` +main worker + | | + |──── init ──────────────────────>| + | | (load Pyodide / connect) + |<──── progress("Loading...") ────| + |<──── progress("Installing...") ─| + |<──── stdout("pathsim loaded") ──| + |<──── progress("Installing...") ─| + |<──── ready ─────────────────────| + | | +``` + +The worker may send zero or more `progress`, `stdout`, and `stderr` messages during initialization. The sequence ends with exactly one `ready` or `error`. + +### 4.2 Exec Flow + +``` +main worker + | | + |──── exec {id, code} ──────────>| + | | (execute Python code) + |<──── stdout("...") ────────────| (zero or more) + |<──── stderr("...") ────────────| (zero or more) + |<──── ok {id} ──────────────────| (success) + | | + | ── OR on failure ── | + | | + |<──── error {id, error, tb?} ───| + | | +``` + +### 4.3 Eval Flow + +``` +main worker + | | + |──── eval {id, expr} ──────────>| + | | (evaluate expression) + |<──── stdout("...") ────────────| (zero or more) + |<──── value {id, value} ────────| (success, value is JSON string) + | | + | ── OR on failure ── | + | | + |<──── error {id, error, tb?} ───| + | | +``` + +### 4.4 Streaming Flow + +``` +main worker + | | + |── stream-start {id, expr} ────>| + | | (enter streaming loop) + |<── stream-data {id, value} ────| \ + |<── stream-data {id, value} ────| } repeated until done + |<── stream-data {id, value} ────| / + | | (expr returns {done: true}) + |<── stream-done {id} ───────────| + | | +``` + +### 4.5 Code Injection During Streaming + +``` +main worker + | | + |── stream-start {id, expr} ────>| + |<── stream-data {id, value} ────| + |<── stream-data {id, value} ────| + | | + |── stream-exec {code} ─────────>| (queued) + | | + | | [drain queue: exec code] + | | [eval expr] + |<── stream-data {id, value} ────| + |<── stream-data {id, value} ────| + |<── stream-done {id} ───────────| + | | +``` + +If the queued code errors, the worker sends a `stderr` message but continues the streaming loop: + +``` + |── stream-exec {code} ─────────>| + | | [exec code → error] + |<── stderr("Stream exec error") | + | | [eval expr → continues] + |<── stream-data {id, value} ────| +``` + +### 4.6 Stop Streaming + +``` +main worker + | | + |── stream-start {id, expr} ────>| + |<── stream-data {id, value} ────| + |<── stream-data {id, value} ────| + | | + |── stream-stop ─────────────────>| + | | (finish current step) + |<── stream-data {id, value} ────| (optional: final partial result) + |<── stream-done {id} ───────────| (always sent) + | | +``` + +The worker does not abort the currently executing Python step. It sets a flag and exits the loop after the current step completes. + +--- + +## 5. Streaming Semantics + +The streaming loop is the core mechanism for live simulation. It runs autonomously on the worker side after receiving `stream-start`. + +### Loop Algorithm + +``` +function runStreamingLoop(id, expr): + active = true + codeQueue = [] + + try: + while active: + // 1. Drain and execute queued code + while codeQueue is not empty: + code = codeQueue.dequeue() + try: + exec(code) + except error: + send stderr("Stream exec error: " + error) + + // 2. Evaluate the stream expression + result = eval(expr) // JSON: {done: bool, result: any} + + // 3. Check for stop (set by stream-stop handler) + if not active: + if result is not done and has data: + send stream-data(id, result) + break + + // 4. Check for natural completion + if result.done: + break + + // 5. Send result and continue + send stream-data(id, result) + + except error: + send error(id, error, traceback?) + finally: + active = false + send stream-done(id) // ALWAYS sent +``` + +### Key Rules + +1. **Code queue drain.** On each iteration, ALL queued code snippets are drained and executed before the expression is evaluated. This ensures parameter changes take effect on the next step. + +2. **Queue isolation.** Errors in queued code are reported via `stderr` but do **not** stop the streaming loop. The loop continues with the next evaluation. + +3. **Expression errors are fatal.** If the main stream expression throws, the loop exits and sends `error` followed by `stream-done`. + +4. **`stream-done` is always sent.** Regardless of whether the loop completed naturally (`done: true`), was stopped (`stream-stop`), or errored, the `stream-done` message is always the last message for that stream ID. + +5. **Result convention.** The stream expression should return a JSON-serializable object with the shape `{done: boolean, result: any}`. When `done` is `true`, the loop exits without sending the final result as `stream-data`. + +6. **Stop semantics.** `stream-stop` sets a flag. The worker does not preempt a running Python evaluation. The flag is checked after the current step completes. + +7. **Single stream.** Only one stream can be active at a time. Starting a new stream while one is active will stop the existing stream first. + +--- + +## 6. HTTP API (Flask Reference Implementation) + +The Flask backend (`src/lib/pyodide/backend/flask/backend.ts`) maps the abstract protocol onto HTTP requests. All routes except `/api/health` require the `X-Session-ID` header to identify the browser session. Each session gets an isolated Python process on the server. + +### 6.1 Route Summary + +| Route | Method | Description | +|-------|--------|-------------| +| `/api/health` | GET | Server health check | +| `/api/init` | POST | Initialize Python worker with packages | +| `/api/exec` | POST | Execute Python code | +| `/api/eval` | POST | Evaluate Python expression | +| `/api/stream` | POST | Start streaming (returns SSE stream) | +| `/api/stream/exec` | POST | Queue code during streaming | +| `/api/stream/stop` | POST | Stop active stream | +| `/api/session` | DELETE | Terminate session and destroy worker | + +### 6.2 Route Details + +#### GET /api/health + +Server health check. No authentication required. + +**Request:** +``` +GET /api/health +``` + +**Response:** +```json +{ + "status": "ok" +} +``` + +--- + +#### POST /api/init + +Initialize the Python runtime and install packages. + +**Request:** +``` +POST /api/init +Content-Type: application/json +X-Session-ID: +``` + +```json +{ + "packages": [ + { + "pip": "pathsim==0.16.5", + "import": "pathsim", + "required": true, + "pre": true + }, + { + "pip": "pathsim-chem>=0.2rc2", + "import": "pathsim_chem", + "required": false, + "pre": true + } + ] +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `packages` | `PackageConfig[]` | no | Packages to install. If omitted, only the base runtime is initialized. | +| `packages[].pip` | `string` | yes | pip install specifier (e.g., `"pathsim==0.16.5"`). | +| `packages[].import` | `string` | yes | Python import name for verification (e.g., `"pathsim"`). | +| `packages[].required` | `boolean` | yes | If `true`, failure to install this package is a fatal error. | +| `packages[].pre` | `boolean` | yes | If `true`, pass `pre=True` to pip (allow pre-release versions). | + +**Success Response:** +```json +{ + "type": "ready", + "messages": [ + { "type": "progress", "value": "Installing pathsim..." }, + { "type": "stdout", "value": "pathsim 0.16.5 loaded successfully" }, + { "type": "progress", "value": "Installing pathsim_chem..." }, + { "type": "stdout", "value": "pathsim_chem 0.2rc3.dev1 loaded successfully" } + ] +} +``` + +The `messages` array contains all `progress`, `stdout`, and `stderr` messages that were generated during initialization, in order. + +**Error Response:** +```json +{ + "type": "error", + "error": "Failed to install required package pathsim==0.16.5: ..." +} +``` + +--- + +#### POST /api/exec + +Execute Python code with no return value. + +**Request:** +``` +POST /api/exec +Content-Type: application/json +X-Session-ID: +``` + +```json +{ + "id": "repl_1", + "code": "x = 42\nprint('hello')" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `id` | `string` | yes | Request ID for correlation. | +| `code` | `string` | yes | Python code to execute. | + +**Success Response:** +```json +{ + "type": "ok", + "id": "repl_1", + "stdout": "hello", + "stderr": "" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"ok"` | yes | Success discriminant. | +| `id` | `string` | yes | Echoed request ID. | +| `stdout` | `string` | no | Captured stdout output during execution. | +| `stderr` | `string` | no | Captured stderr output during execution. | + +**Error Response:** +```json +{ + "type": "error", + "id": "repl_1", + "error": "NameError: name 'y' is not defined", + "traceback": "Traceback (most recent call last):\n ...", + "stdout": "", + "stderr": "" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"error"` | yes | Error discriminant. | +| `id` | `string` | yes | Echoed request ID. | +| `error` | `string` | yes | Error message. | +| `traceback` | `string` | no | Python traceback. | +| `stdout` | `string` | no | Any stdout captured before the error. | +| `stderr` | `string` | no | Any stderr captured before the error. | + +--- + +#### POST /api/eval + +Evaluate a Python expression and return the JSON-serialized result. + +**Request:** +``` +POST /api/eval +Content-Type: application/json +X-Session-ID: +``` + +```json +{ + "id": "repl_2", + "expr": "json.dumps({'x': x, 'y': [1,2,3]})" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `id` | `string` | yes | Request ID. | +| `expr` | `string` | yes | Python expression to evaluate. Result must be JSON-serializable. | + +**Success Response:** +```json +{ + "type": "value", + "id": "repl_2", + "value": "{\"x\": 42, \"y\": [1, 2, 3]}", + "stdout": "", + "stderr": "" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"value"` | yes | Success discriminant. | +| `id` | `string` | yes | Echoed request ID. | +| `value` | `string` | yes | JSON-serialized result. The client parses this with `JSON.parse()`. | +| `stdout` | `string` | no | Captured stdout. | +| `stderr` | `string` | no | Captured stderr. | + +**Error Response:** Same shape as exec error response. + +--- + +#### POST /api/stream + +Start a streaming loop. Returns a Server-Sent Events stream. + +**Request:** +``` +POST /api/stream +Content-Type: application/json +X-Session-ID: +``` + +```json +{ + "id": "repl_3", + "expr": "step_simulation()" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `id` | `string` | yes | Stream ID. | +| `expr` | `string` | yes | Python expression to evaluate each iteration. Should return `{done: bool, result: any}`. | + +**Response:** `Content-Type: text/event-stream` (SSE). See [Section 7](#7-sse-event-format). + +--- + +#### POST /api/stream/exec + +Queue code for execution between streaming steps. + +**Request:** +``` +POST /api/stream/exec +Content-Type: application/json +X-Session-ID: +``` + +```json +{ + "code": "controller.set_gain(2.0)" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `code` | `string` | yes | Python code to queue. | + +**Response:** +```json +{ + "status": "queued" +} +``` + +--- + +#### POST /api/stream/stop + +Stop the active streaming loop. + +**Request:** +``` +POST /api/stream/stop +Content-Type: application/json +X-Session-ID: +``` + +```json +{} +``` + +**Response:** +```json +{ + "status": "stopped" +} +``` + +--- + +#### DELETE /api/session + +Terminate the session and destroy the server-side Python worker process. + +**Request:** +``` +DELETE /api/session +X-Session-ID: +``` + +**Response:** +```json +{ + "status": "terminated" +} +``` + +--- + +## 7. SSE Event Format + +The `/api/stream` endpoint returns a standard Server-Sent Events stream. Each event has an `event` field (the event type) and a `data` field (JSON payload). Events are separated by double newlines. + +### Event Types + +#### `data` -- Streaming step result + +``` +event: data +data: {"done":false,"result":{"t":0.5,"values":[1.2,3.4]}} + +``` + +The `data` field is a JSON string matching the return value of the stream expression. + +#### `stdout` -- Captured standard output + +``` +event: stdout +data: "Step 50 complete" + +``` + +The `data` field is a JSON-encoded string. + +#### `stderr` -- Captured standard error + +``` +event: stderr +data: "Stream exec error: NameError: name 'foo' is not defined" + +``` + +The `data` field is a JSON-encoded string. + +#### `done` -- Stream completed + +``` +event: done +data: {} + +``` + +Sent when the streaming loop exits (any reason). This is always the last event in the stream. The SSE connection closes after this event. + +#### `error` -- Stream error + +``` +event: error +data: {"error":"ZeroDivisionError: division by zero","traceback":"Traceback ..."} + +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `error` | `string` | yes | Error message. | +| `traceback` | `string` | no | Python traceback. | + +Sent when the main stream expression throws. The `done` event is NOT sent after `error` in the SSE stream (the error event terminates the stream). The client-side `FlaskBackend` maps this to an `onError` callback. + +### Complete SSE Example + +``` +event: data +data: {"done":false,"result":{"t":0.0,"values":[0.0]}} + +event: stdout +data: "Simulation step 1" + +event: data +data: {"done":false,"result":{"t":0.5,"values":[0.25]}} + +event: data +data: {"done":false,"result":{"t":1.0,"values":[1.0]}} + +event: done +data: {} + +``` + +--- + +## 8. State Management + +All backends share a single `backendState` Svelte writable store defined in `src/lib/pyodide/backend/state.ts`. This store drives the UI loading indicators, error displays, and ready checks. + +```typescript +const initialState: BackendState = { + initialized: false, + loading: false, + error: null, + progress: '' +}; +``` + +### State Transitions + +Backends must follow this state machine: + +``` + init() called + [idle] ──────────────────────────> [loading] + initialized: false initialized: false + loading: false loading: true + error: null error: null + progress: '' progress: "Loading..." + | + ┌────────────────┼────────────────┐ + | | | + (progress) (success) (failure) + | | | + [loading] [ready] [error] + progress: initialized: true loading: false + "Installing..." loading: false error: "msg" + progress: "Ready" + | + terminate() + | + [idle] (backendState.reset()) +``` + +### Rules for Implementations + +1. **Set `loading = true` and `error = null`** at the start of `init()`. +2. **Update `progress`** during loading to give the user feedback (e.g., `"Loading Pyodide..."`, `"Installing pathsim..."`). +3. **Set `initialized = true` and `loading = false`** on successful initialization. +4. **Set `loading = false` and `error = `** on initialization failure. +5. **Call `backendState.reset()`** in `terminate()` to return to the idle state. +6. **Set `error`** when execution errors occur that affect the overall backend health (not for per-request errors, which are returned via the promise rejection or error callback). + +--- + +## 9. Backend Registration + +To add a new backend type to PathView: + +### Step 1: Implement the Backend Interface + +Create a new file (e.g., `src/lib/pyodide/backend/remote/backend.ts`): + +```typescript +import type { Backend, BackendState } from '../types'; +import { backendState } from '../state'; + +export class RemoteBackend implements Backend { + // ... implement all methods from the Backend interface +} +``` + +### Step 2: Add Type to BackendType Union + +In `src/lib/pyodide/backend/registry.ts`: + +```typescript +export type BackendType = 'pyodide' | 'flask' | 'remote'; +``` + +### Step 3: Add Case to createBackend() + +In `src/lib/pyodide/backend/registry.ts`: + +```typescript +export function createBackend(type: BackendType): Backend { + switch (type) { + case 'pyodide': + return new PyodideBackend(); + case 'flask': + return new FlaskBackend(flaskHost); + case 'remote': + return new RemoteBackend(remoteConfig); + default: + throw new Error(`Unknown backend type: ${type}`); + } +} +``` + +### Step 4: Re-export from index.ts + +In `src/lib/pyodide/backend/index.ts`: + +```typescript +export { RemoteBackend } from './remote/backend'; +``` + +The backend is now available via `switchBackend('remote')` or the `?backend=remote` URL parameter (if `initBackendFromUrl()` is updated to handle it). + +--- + +## 10. Worked Example + +This section traces a complete session: initialization with packages, code execution, expression evaluation, then a streaming simulation with code injection and stop. Both the abstract message protocol and the HTTP equivalents are shown. + +### 10.1 Abstract Message Flow + +``` +=== INITIALIZATION === + +main → worker: { type: "init" } +worker → main: { type: "progress", value: "Loading Pyodide..." } +worker → main: { type: "progress", value: "Installing dependencies..." } +worker → main: { type: "progress", value: "Installing pathsim..." } +worker → main: { type: "stdout", value: "pathsim 0.16.5 loaded successfully" } +worker → main: { type: "progress", value: "Installing pathsim_chem..." } +worker → main: { type: "stdout", value: "pathsim_chem 0.2rc3.dev1 loaded successfully" } +worker → main: { type: "ready" } + +=== EXEC: Set up simulation === + +main → worker: { type: "exec", id: "repl_1", code: "import json\nfrom pathsim import Simulation, Connection\nfrom pathsim.blocks import Integrator, Constant, Scope\nfrom pathsim.solvers import SSPRK22\n\nconstant = Constant(value=1.0)\nintegrator = Integrator(initial_value=0.0)\nscope = Scope()\nsim = Simulation(\n [constant, integrator, scope],\n [Connection(constant[0], integrator[0]),\n Connection(integrator[0], scope[0])],\n Solver=SSPRK22, dt=0.01\n)" } +worker → main: { type: "ok", id: "repl_1" } + +=== EXEC: Set up streaming generator === + +main → worker: { type: "exec", id: "repl_2", code: "sim_iter = sim.run_generator(duration=10, steps_per_yield=100)\ndef step_simulation():\n try:\n result = next(sim_iter)\n return {'done': False, 'result': result}\n except StopIteration:\n return {'done': True, 'result': None}" } +worker → main: { type: "ok", id: "repl_2" } + +=== EVAL: Check initial state === + +main → worker: { type: "eval", id: "repl_3", expr: "json.dumps({'t': 0, 'ready': True})" } +worker → main: { type: "value", id: "repl_3", value: "{\"t\": 0, \"ready\": true}" } + +=== STREAMING: Run simulation with live updates === + +main → worker: { type: "stream-start", id: "repl_4", expr: "json.dumps(step_simulation(), default=str)" } +worker → main: { type: "stream-data", id: "repl_4", value: "{\"done\":false,\"result\":{\"t\":1.0}}" } +worker → main: { type: "stream-data", id: "repl_4", value: "{\"done\":false,\"result\":{\"t\":2.0}}" } + +--- User changes a parameter at t=2.0 --- + +main → worker: { type: "stream-exec", code: "constant.set(value=2.0)" } + +--- Worker drains queue, applies change, then continues --- + +worker → main: { type: "stream-data", id: "repl_4", value: "{\"done\":false,\"result\":{\"t\":3.0}}" } +worker → main: { type: "stream-data", id: "repl_4", value: "{\"done\":false,\"result\":{\"t\":4.0}}" } + +--- User stops the simulation at t=4.0 --- + +main → worker: { type: "stream-stop" } + +--- Worker finishes current step --- + +worker → main: { type: "stream-data", id: "repl_4", value: "{\"done\":false,\"result\":{\"t\":5.0}}" } +worker → main: { type: "stream-done", id: "repl_4" } +``` + +### 10.2 HTTP Equivalents + +The same session expressed as HTTP requests (Flask backend): + +``` +=== INITIALIZATION === + +GET /api/health +→ 200 {"status": "ok"} + +POST /api/init +Headers: X-Session-ID: a1b2c3d4-... +Body: { + "packages": [ + {"pip": "pathsim==0.16.5", "import": "pathsim", "required": true, "pre": true}, + {"pip": "pathsim-chem>=0.2rc2", "import": "pathsim_chem", "required": false, "pre": true} + ] +} +→ 200 { + "type": "ready", + "messages": [ + {"type": "progress", "value": "Installing pathsim..."}, + {"type": "stdout", "value": "pathsim 0.16.5 loaded successfully"}, + {"type": "progress", "value": "Installing pathsim_chem..."}, + {"type": "stdout", "value": "pathsim_chem 0.2rc3.dev1 loaded successfully"} + ] +} + +=== EXEC === + +POST /api/exec +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {"id": "repl_1", "code": "import json\nfrom pathsim import ..."} +→ 200 {"type": "ok", "id": "repl_1"} + +POST /api/exec +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {"id": "repl_2", "code": "sim_iter = sim.run_generator(...)..."} +→ 200 {"type": "ok", "id": "repl_2"} + +=== EVAL === + +POST /api/eval +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {"id": "repl_3", "expr": "json.dumps({'t': 0, 'ready': True})"} +→ 200 {"type": "value", "id": "repl_3", "value": "{\"t\": 0, \"ready\": true}"} + +=== STREAMING === + +POST /api/stream +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {"id": "repl_4", "expr": "json.dumps(step_simulation(), default=str)"} +→ 200 (SSE stream) + + event: data + data: {"done":false,"result":{"t":1.0}} + + event: data + data: {"done":false,"result":{"t":2.0}} + +--- concurrent request --- + +POST /api/stream/exec +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {"code": "constant.set(value=2.0)"} +→ 200 {"status": "queued"} + +--- SSE continues --- + + event: data + data: {"done":false,"result":{"t":3.0}} + + event: data + data: {"done":false,"result":{"t":4.0}} + +--- concurrent request --- + +POST /api/stream/stop +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {} +→ 200 {"status": "stopped"} + +--- SSE concludes --- + + event: data + data: {"done":false,"result":{"t":5.0}} + + event: done + data: {} + +=== CLEANUP === + +DELETE /api/session +Headers: X-Session-ID: a1b2c3d4-... +→ 200 {"status": "terminated"} +``` + +--- + +## Notes for Backend Authors + +1. **JSON serialization.** All values crossing the boundary (eval results, stream data) must be valid JSON strings. The Python side should use `json.dumps()` with a `default` handler for non-serializable types. + +2. **ID correlation.** The `id` field in requests is echoed in responses. Clients use it to match responses to pending promises. Generate unique IDs (the convention is `"repl_N"` with an incrementing counter). + +3. **Timeout handling.** Clients set their own timeouts. Backends should not enforce timeouts unless they need to protect server resources. If a backend does enforce timeouts, it should send an `error` response with a clear timeout message. + +4. **Stdout/stderr capture.** Backends must capture Python's stdout and stderr and forward them as `stdout`/`stderr` messages. For Web Worker backends, this is done via Pyodide's `setStdout`/`setStderr`. For HTTP backends, captured output is included in the response body or sent as SSE events during streaming. + +5. **Session isolation.** HTTP-based backends must isolate sessions. Each `X-Session-ID` maps to a separate Python process or namespace. Leaking state between sessions is a correctness and security issue. + +6. **Idempotent init.** Calling `init()` when already initialized or loading should be a no-op, not an error. + +7. **Graceful terminate.** `terminate()` must clean up all resources: reject pending promises, abort streams, kill workers/processes, and reset state. It should not throw. diff --git a/docs/toolbox-spec.md b/docs/toolbox-spec.md new file mode 100644 index 0000000..891da7e --- /dev/null +++ b/docs/toolbox-spec.md @@ -0,0 +1,699 @@ +# PathView Toolbox Integration Specification + +**Version:** 1.0.0 + +A toolbox is a Python package that provides computational blocks (and optionally events) for the PathView visual simulation environment. Toolboxes extend PathView with new block types that appear in the Block Library panel. + +This document is the authoritative reference for third-party developers creating toolbox packages and integrating them into PathView. + +Existing toolboxes: `pathsim` (core simulation blocks), `pathsim-chem` (chemical engineering blocks). + +--- + +## Table of Contents + +- [1. Overview](#1-overview) +- [2. Python Package Requirements](#2-python-package-requirements) + - [2.1 Block Classes](#21-block-classes) + - [2.2 Event Classes](#22-event-classes) +- [3. Toolbox Config Directory](#3-toolbox-config-directory) +- [4. blocks.json Schema](#4-blocksjson-schema) +- [5. events.json Schema](#5-eventsjson-schema) +- [6. requirements-pyodide.txt](#6-requirements-pyodidetxt) +- [7. Extraction Pipeline](#7-extraction-pipeline) + - [7.1 Discovery](#71-discovery) + - [7.2 Block Extraction Flow](#72-block-extraction-flow) + - [7.3 Event Extraction Flow](#73-event-extraction-flow) + - [7.4 Dependency Extraction Flow](#74-dependency-extraction-flow) +- [8. Generated Output](#8-generated-output) + - [8.1 blocks.ts](#81-blocksts) + - [8.2 events.ts](#82-eventsts) + - [8.3 dependencies.ts](#83-dependenciests) +- [9. Block Metadata Contract (Block.info())](#9-block-metadata-contract-blockinfo) + - [9.1 Port Label Semantics](#91-port-label-semantics) + - [9.2 Parameter Type Inference](#92-parameter-type-inference) +- [10. Runtime Package Installation](#10-runtime-package-installation) +- [11. Step-by-Step Walkthrough](#11-step-by-step-walkthrough) +- [12. UI Configuration (Optional)](#12-ui-configuration-optional) + +--- + +## 1. Overview + +Each toolbox provides a set of block types (and optionally event types) that PathView discovers, extracts metadata from, and presents in the Block Library panel. The integration requires three things: + +1. **A Python package** with blocks that implement the `Block.info()` classmethod (and optionally an `events` submodule). +2. **A config directory** at `scripts/config//` containing `blocks.json` (and optionally `events.json`). +3. **An entry** in `scripts/config/requirements-pyodide.txt` so both the Pyodide and Flask backends install the package at runtime. + +The extraction pipeline (`npm run extract`) reads the config files, imports the Python packages, calls `Block.info()` on each class, and generates TypeScript source files that PathView consumes at build time. No manual registration beyond these three steps is needed. + +--- + +## 2. Python Package Requirements + +### 2.1 Block Classes + +The toolbox Python package must have an importable module containing block classes. For example, a package named `pathsim-controls` (installed as `pathsim_controls`) would expose blocks via `pathsim_controls.blocks`. + +Each block class must implement the `info()` classmethod returning a dict with the following keys: + +```python +@classmethod +def info(cls): + return { + "input_port_labels": {"x": 0, "y": 1}, # or None or {} + "output_port_labels": {"out": 0}, # or None or {} + "parameters": { + "gain": {"default": 1.0}, + "mode": {"default": "linear"} + }, + "description": "A proportional controller block." + } +``` + +| Key | Type | Description | +|-----|------|-------------| +| `input_port_labels` | `dict`, `None`, or `{}` | Defines input port names and indices. See [Port Label Semantics](#91-port-label-semantics). | +| `output_port_labels` | `dict`, `None`, or `{}` | Defines output port names and indices. See [Port Label Semantics](#91-port-label-semantics). | +| `parameters` | `dict` | Map of parameter names to dicts containing at minimum a `"default"` key. | +| `description` | `str` | RST-formatted docstring. The first line/sentence is used as the short description. | + +If a block does not implement `info()`, the extractor falls back to `__init__` signature introspection, but this is less reliable. All new toolbox blocks should implement `info()`. + +### 2.2 Event Classes + +Event classes live in a separate submodule (e.g., `pathsim_controls.events`). Events do not use an `info()` classmethod. Instead, the extractor inspects the `__init__` signature to discover parameters: + +```python +class ThresholdEvent: + """Triggers when a signal crosses a threshold value.""" + + def __init__(self, func_evt=None, func_act=None, threshold=0.0, tolerance=1e-4): + ... +``` + +Parameter names, default values, and the class docstring are extracted automatically. + +--- + +## 3. Toolbox Config Directory + +Each toolbox has a config directory at: + +``` +scripts/config// +``` + +The directory name should match the pip package name (e.g., `pathsim-chem`, `pathsim-controls`). + +Required contents: + +| File | Required | Description | +|------|----------|-------------| +| `blocks.json` | Yes | Block categories and class names to extract | +| `events.json` | No | Event class names to extract | + +Example directory structure: + +``` +scripts/config/ + schemas/ # JSON schemas (shared, do not modify) + blocks.schema.json + events.schema.json + pathsim/ # Core toolbox + blocks.json + events.json + simulation.json + pathsim-chem/ # Chemical engineering toolbox + blocks.json + pathsim-controls/ # Your new toolbox + blocks.json + events.json +``` + +--- + +## 4. blocks.json Schema + +A `blocks.json` file declares which block classes to extract from a toolbox and how to organize them into categories. + +```json +{ + "$schema": "../schemas/blocks.schema.json", + "toolbox": "pathsim-controls", + "importPath": "pathsim_controls.blocks", + "categories": { + "Controls": ["PIDController", "StateEstimator"] + }, + "extraDocstrings": [] +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `$schema` | string | No | JSON Schema reference for editor validation. Should be `"../schemas/blocks.schema.json"`. | +| `toolbox` | string | Yes | Toolbox identifier. Must match the config directory name. | +| `importPath` | string | Yes | Python import path to the blocks module (e.g., `"pathsim_controls.blocks"`). | +| `categories` | object | Yes | Map of category names to arrays of block entries. Category names appear as section headers in the Block Library panel. | +| `extraDocstrings` | string[] | No | Additional classes to extract docstrings from (not blocks themselves). Used by the core toolbox for `Subsystem` and `Interface`. | + +**Block entries** within each category array can be either: + +- **A string** -- the class name, imported from `importPath`: + ```json + "PIDController" + ``` +- **An object** -- for classes that live in a different module than `importPath`: + ```json + {"class": "PIDController", "import": "pathsim_controls.advanced"} + ``` + +**Real-world example** (`pathsim-chem/blocks.json`): + +```json +{ + "$schema": "../schemas/blocks.schema.json", + "toolbox": "pathsim-chem", + "importPath": "pathsim_chem.tritium", + "categories": { + "Chemical": ["Process", "Bubbler4", "Splitter", "GLC"] + } +} +``` + +--- + +## 5. events.json Schema + +An `events.json` file declares which event classes to extract from a toolbox. + +```json +{ + "$schema": "../schemas/events.schema.json", + "toolbox": "pathsim-controls", + "importPath": "pathsim_controls.events", + "events": ["ThresholdEvent", "TimerEvent"] +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `$schema` | string | No | JSON Schema reference for editor validation. Should be `"../schemas/events.schema.json"`. | +| `toolbox` | string | Yes | Toolbox identifier. Must match the config directory name. | +| `importPath` | string | Yes | Python import path to the events module (e.g., `"pathsim_controls.events"`). | +| `events` | string[] | Yes | List of event class names to extract from the module. | + +**Real-world example** (`pathsim/events.json`): + +```json +{ + "$schema": "../schemas/events.schema.json", + "toolbox": "pathsim", + "importPath": "pathsim.events", + "events": [ + "ZeroCrossing", + "ZeroCrossingUp", + "ZeroCrossingDown", + "Schedule", + "ScheduleList", + "Condition" + ] +} +``` + +--- + +## 6. requirements-pyodide.txt + +The file `scripts/config/requirements-pyodide.txt` is the single source of truth for runtime Python dependencies. Both the Pyodide web worker and the Flask backend install packages from this file. + +**Syntax:** + +``` +--pre +pathsim +pathsim-chem>=0.2rc2 # optional +pathsim-controls # optional +``` + +| Syntax Element | Description | +|----------------|-------------| +| `--pre` | Global flag. Applies to all packages below it. Allows pre-release versions (e.g., `rc`, `beta`). | +| `pathsim` | A required package. If installation fails, app startup is blocked. | +| `pathsim-chem>=0.2rc2 # optional` | An optional package with a version specifier. The `# optional` comment sets the package's `required` field to `false` -- installation failure will not block app startup. | +| `>=`, `==`, `~=`, `<`, `<=`, `>` | Standard pip version specifiers. All are supported. | + +**How the file is parsed** (from `ConfigLoader.load_requirements()`): + +1. Lines starting with `#` are ignored (pure comments). +2. The `--pre` flag sets `pre: true` for all subsequent packages. +3. For each package line, the pip spec is the text before any `#` comment. +4. If the comment contains `# optional` (case-insensitive), `required` is set to `false`. +5. The Python import name is derived by stripping version specifiers and replacing `-` with `_` (e.g., `pathsim-chem` becomes `pathsim_chem`). + +--- + +## 7. Extraction Pipeline + +The extraction is run via: + +```bash +npm run extract # Extract all (blocks, events, simulation, dependencies) +python scripts/extract.py # Same, invoked directly +``` + +Selective extraction is also supported: + +```bash +python scripts/extract.py --blocks # Blocks only +python scripts/extract.py --events # Events only +python scripts/extract.py --deps # Dependencies only +python scripts/extract.py --registry # JSON registry only (for pvm2py) +python scripts/extract.py --validate # Validate config files only +``` + +### 7.1 Discovery + +The `discover_toolboxes()` method scans `scripts/config/` for subdirectories that contain a `blocks.json` file (excluding the `schemas/` directory). Toolboxes are auto-discovered -- no manual registration step is needed beyond creating the config directory. + +### 7.2 Block Extraction Flow + +For each discovered toolbox: + +1. Load and validate `blocks.json`. +2. Import the Python module specified in `importPath`. +3. For each class name in each category: + a. Get the class object from the module via `getattr()`. + b. Call `cls.info()` to get the metadata dict. + c. Process `input_port_labels` and `output_port_labels` via `_process_port_labels()` -- converting the dict to a sorted list of names, `None` for variable ports, or `[]` for no ports. + d. For each parameter, infer the type from the default value (see [Parameter Type Inference](#92-parameter-type-inference)). + e. Extract parameter descriptions from the RST docstring. + f. Convert the full docstring to HTML using `docutils` (if available). +4. If `info()` is not available, fall back to `__init__` signature introspection. +5. Process `extraDocstrings` classes (docstring-only extraction, no parameter/port extraction). +6. Generate `src/lib/nodes/generated/blocks.ts`. + +### 7.3 Event Extraction Flow + +For each discovered toolbox that has an `events.json`: + +1. Load and validate `events.json`. +2. Import the Python module specified in `importPath`. +3. For each event class name: + a. Get the class object from the module. + b. Inspect the `__init__` signature for parameters (skipping `self`). + c. Infer parameter types from names and default values. + d. Extract parameter descriptions from the class docstring. + e. Convert the full docstring to HTML. +4. Generate `src/lib/events/generated/events.ts`. + +### 7.4 Dependency Extraction Flow + +1. Parse `scripts/config/requirements-pyodide.txt` into a list of package configs. +2. For each package, import the module and read `__version__` to get the installed version. +3. Pin exact versions for release builds (dev versions keep the original spec). +4. Load `scripts/config/pyodide.json` for Pyodide runtime configuration. +5. Generate `src/lib/constants/dependencies.ts` containing the `PYTHON_PACKAGES` array. + +--- + +## 8. Generated Output + +The extraction pipeline generates three TypeScript files. These are auto-generated and should not be edited by hand. + +### 8.1 blocks.ts + +**Path:** `src/lib/nodes/generated/blocks.ts` + +```typescript +export interface ExtractedParam { + type: string; // "integer", "number", "string", "boolean", "callable", "array", "any" + default: string | null; + description: string; + min?: number; + max?: number; + options?: string[]; +} + +export interface ExtractedBlock { + blockClass: string; + description: string; + docstringHtml: string; + params: Record; + inputs: string[] | null; // null = variable, [] = none, [...] = fixed named + outputs: string[] | null; // null = variable, [] = none, [...] = fixed named +} + +export const extractedBlocks: Record = { ... }; + +export const blockConfig: Record = { + Sources: ["Constant", "Source", ...], + Controls: ["PIDController", "StateEstimator"], + ... +}; + +export const blockImportPaths: Record = { + "Constant": "pathsim.blocks", + "PIDController": "pathsim_controls.blocks", + ... +}; +``` + +The `blockConfig` object maps category names (used as Block Library section headers) to arrays of block class names. The `blockImportPaths` object maps each block class name to its Python import path. + +### 8.2 events.ts + +**Path:** `src/lib/events/generated/events.ts` + +```typescript +import type { EventTypeDefinition } from '../types'; + +export const extractedEvents: EventTypeDefinition[] = [ + { + type: "pathsim.events.ZeroCrossing", // fully qualified Python path + name: "ZeroCrossing", // display name + eventClass: "ZeroCrossing", // class name + description: "...", + docstringHtml: "...", + params: [ + { name: "func_evt", type: "callable", default: "None", description: "..." }, + { name: "func_act", type: "callable", default: "None", description: "..." }, + { name: "tolerance", type: "number", default: "0.0001", description: "..." } + ] + } +]; +``` + +The `type` field uses the fully qualified Python import path (e.g., `"pathsim.events.ZeroCrossing"`). This is used in `.pvm` files to reference event types and for code generation. + +### 8.3 dependencies.ts + +**Path:** `src/lib/constants/dependencies.ts` + +```typescript +export interface PackageConfig { + pip: string; // pip install spec (e.g., "pathsim==0.16.5") + pre: boolean; // whether to use --pre flag + required: boolean; // whether failure blocks startup + import: string; // Python import name (e.g., "pathsim_chem") +} + +export const PYTHON_PACKAGES: PackageConfig[] = [ + { + "pip": "pathsim==0.16.5", + "required": true, + "pre": true, + "import": "pathsim" + }, + { + "pip": "pathsim-chem>=0.2rc2", + "required": false, + "pre": true, + "import": "pathsim_chem" + } +]; +``` + +This file also exports `PYODIDE_VERSION`, `PYODIDE_CDN_URL`, `PYODIDE_PRELOAD`, `PATHVIEW_VERSION`, and `EXTRACTED_VERSIONS`. + +--- + +## 9. Block Metadata Contract (Block.info()) + +The `info()` classmethod is the primary interface between a toolbox's Python code and PathView's extraction pipeline. + +```python +@classmethod +def info(cls): + return { + "input_port_labels": {"x": 0, "y": 1}, + "output_port_labels": {"out": 0}, + "parameters": { + "gain": {"default": 1.0}, + "mode": {"default": "linear"} + }, + "description": "A proportional controller block." + } +``` + +### 9.1 Port Label Semantics + +The values of `input_port_labels` and `output_port_labels` have precise semantics that control both extraction and UI behavior: + +| Value | Meaning | Extracted As | UI Behavior | +|-------|---------|--------------|-------------| +| `None` | Variable/unlimited ports | `null` in TypeScript | Add/remove port buttons shown | +| `{}` | No ports of this direction | `[]` (empty array) | No ports rendered | +| `{"name": index, ...}` | Fixed labeled ports | `["name", ...]` sorted by index | Ports locked, names displayed | + +The extraction function `_process_port_labels()` converts dicts to sorted name lists: + +```python +{"x": 0, "y": 1} # → ["x", "y"] (sorted by index value) +{"out": 0} # → ["out"] +None # → None (variable ports) +{} # → [] (no ports) +``` + +### 9.2 Parameter Type Inference + +The extractor infers TypeScript-side parameter types from Python default values using these rules (evaluated in order): + +| Condition | Inferred Type | +|-----------|---------------| +| Name starts with `func_` or `func`, or default is callable | `"callable"` | +| Default is `True` or `False` | `"boolean"` | +| Default is `int` (and not `bool`) | `"integer"` | +| Default is `float` | `"number"` | +| Default is `str` | `"string"` | +| Default is `list`, `tuple`, or ndarray | `"array"` | +| Default is `None` or anything else | `"any"` | + +Note: `bool` is checked before `int` because in Python `isinstance(True, int)` is `True`. + +--- + +## 10. Runtime Package Installation + +The `PYTHON_PACKAGES` constant (generated from `requirements-pyodide.txt`) drives package installation in both backends: + +**Pyodide backend** (`src/lib/pyodide/backend/pyodide/worker.ts`): +```typescript +for (const pkg of PYTHON_PACKAGES) { + await pyodide.runPythonAsync(` + import micropip + await micropip.install('${pkg.pip}'${pkg.pre ? ', pre=True' : ''}) + `); +} +``` + +**Flask backend** (`src/lib/pyodide/backend/flask/backend.ts`): +```typescript +await fetch(`${baseUrl}/api/init`, { + method: 'POST', + body: JSON.stringify({ packages: PYTHON_PACKAGES }), +}); +``` + +Both backends respect the `required` flag. If a package has `required: false` (from the `# optional` comment in requirements), installation failure is caught and logged but does not block app startup. If `required: true`, a failed installation aborts initialization. + +--- + +## 11. Step-by-Step Walkthrough + +This section walks through adding a hypothetical `pathsim-controls` toolbox with two blocks (`PIDController`, `StateEstimator`) and one event (`ThresholdEvent`). + +### Step 1: Add to requirements + +Edit `scripts/config/requirements-pyodide.txt`: + +``` +--pre +pathsim +pathsim-chem>=0.2rc2 # optional +pathsim-controls # optional +``` + +The `# optional` comment means PathView will continue loading if this package is unavailable. + +### Step 2: Create config directory + +Create the directory `scripts/config/pathsim-controls/`. + +**`scripts/config/pathsim-controls/blocks.json`:** + +```json +{ + "$schema": "../schemas/blocks.schema.json", + "toolbox": "pathsim-controls", + "importPath": "pathsim_controls.blocks", + "categories": { + "Controls": ["PIDController", "StateEstimator"] + } +} +``` + +**`scripts/config/pathsim-controls/events.json`:** + +```json +{ + "$schema": "../schemas/events.schema.json", + "toolbox": "pathsim-controls", + "importPath": "pathsim_controls.events", + "events": ["ThresholdEvent"] +} +``` + +### Step 3: Ensure the Python package is installed + +The extraction script needs to import the package. Install it in your development environment: + +```bash +pip install pathsim-controls +``` + +Verify the blocks module is importable and `info()` works: + +```python +from pathsim_controls.blocks import PIDController +print(PIDController.info()) +``` + +### Step 4: Run the extraction + +```bash +npm run extract +``` + +Expected output: + +``` +PathSim Metadata Extractor +======================================== + +Extracting dependencies... + Pinned pathsim to version 0.16.5 + pathsim-controls ... + +Extracting blocks... + Processing toolbox: pathsim + Extracted Constant + ... + Processing toolbox: pathsim-controls + Extracted PIDController + Extracted StateEstimator +Generated: src/lib/nodes/generated/blocks.ts + +Extracting events... + Processing events from: pathsim + Extracted ZeroCrossing + ... + Processing events from: pathsim-controls + Extracted ThresholdEvent +Generated: src/lib/events/generated/events.ts + +Done! +``` + +### Step 5: Verify in dev server + +```bash +npm run dev +``` + +Open the app in a browser. The Block Library panel should now show a "Controls" section containing "PIDController" and "StateEstimator". The Events panel should list "ThresholdEvent". + +### Step 6: Build for production + +```bash +npm run build +``` + +The generated TypeScript files are compiled into the production bundle. The `PYTHON_PACKAGES` constant ensures runtime installation of `pathsim-controls` in both Pyodide and Flask backends. + +--- + +## 12. UI Configuration (Optional) + +After a toolbox's blocks are extracted and appear in the Block Library, you may want to customize their UI behavior. These configurations are in PathView's TypeScript source and are optional. + +### Port synchronization + +For blocks where each input has a corresponding output (parallel processing blocks), add the block class name to the `syncPortBlocks` set. This hides the output port controls and keeps output count in sync with input count. + +**File:** `src/lib/nodes/uiConfig.ts` + +```typescript +export const syncPortBlocks = new Set([ + 'Integrator', + 'Differentiator', + 'Delay', + // ... existing entries ... + 'PIDController', // add your block here +]); +``` + +### Port labels from parameters + +For blocks that derive port names from a parameter value (e.g., a list of channel labels), add an entry to `portLabelParams`. When the user changes the parameter, port names update automatically. + +**File:** `src/lib/nodes/uiConfig.ts` + +```typescript +export const portLabelParams: Record = { + Scope: { param: 'labels', direction: 'input' }, + Spectrum: { param: 'labels', direction: 'input' }, + Adder: { param: 'operations', direction: 'input', parser: parseOperationsString }, + // Add your block: + MyMuxBlock: { param: 'labels', direction: 'input' }, +}; +``` + +The `PortLabelConfig` interface: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `param` | string | Yes | Parameter name whose value determines port labels. | +| `direction` | `"input"` or `"output"` | Yes | Which port direction the labels apply to. | +| `parser` | function | No | Custom parser to convert the param value to a `string[]`. Default uses `parsePythonList`. | + +### Custom shapes + +Map your toolbox's categories to node shapes using the shape registry. Available built-in shapes: `pill`, `rect`, `circle`, `diamond`, `mixed`, `default`. + +**File:** `src/lib/nodes/shapes/registry.ts` + +```typescript +// Add to the categoryShapeMap or call setCategoryShape: +setCategoryShape('Controls', 'rect'); +``` + +The existing category-to-shape mapping: + +| Category | Shape | +|----------|-------| +| Sources | `pill` | +| Dynamic | `rect` | +| Algebraic | `rect` | +| Mixed | `mixed` | +| Recording | `pill` | +| Subsystem | `rect` | + +Categories not in the map use the `default` shape. + +--- + +## Notes for Toolbox Authors + +1. **`info()` is the contract.** Implement it on every block class. The extractor falls back to `__init__` introspection, but `info()` gives you explicit control over port definitions and parameter metadata. + +2. **Port labels must be consistent.** The index values in port label dicts must be zero-based and contiguous. The extractor sorts by index, so `{"y": 1, "x": 0}` correctly produces `["x", "y"]`. + +3. **Parameter defaults drive type inference.** If your parameter should be a `number` in the UI, make sure the default is a Python `float`, not `None`. Use `None` only when the type truly cannot be determined. + +4. **Docstrings are rendered as HTML.** If `docutils` is installed, RST-formatted docstrings are converted to HTML and displayed in the block documentation panel. Write proper RST with `:param:` directives for best results. + +5. **The `# optional` flag is important.** Mark your toolbox as optional in `requirements-pyodide.txt` unless PathView cannot function without it. This ensures the app starts even if your package has installation issues in a user's browser environment. + +6. **Test the extraction locally.** Run `python scripts/extract.py --validate` to check config file validity, then `python scripts/extract.py --blocks` to verify block extraction before committing. From 29a1ee09984f0aeb3016e4b8a282efc6b95e4498 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 12:16:31 +0100 Subject: [PATCH 08/27] Add spec doc references to README Co-Authored-By: KDW1 --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index c9ccad0..34152ff 100644 --- a/README.md +++ b/README.md @@ -325,6 +325,8 @@ npm run build No code changes needed - the extraction script automatically discovers toolbox directories. +For the full toolbox integration reference (Python package contract, config schemas, extraction pipeline, generated output), see [**docs/toolbox-spec.md**](docs/toolbox-spec.md). + --- ## Python Backend System @@ -482,6 +484,8 @@ Then open `http://localhost:5173/?backend=flask`. - **Session TTL** — stale sessions cleaned up after 1 hour of inactivity - **Streaming** — simulations stream via SSE, with the same code injection support as Pyodide +For the full protocol reference (message types, HTTP routes, SSE format, streaming semantics, how to implement a new backend), see [**docs/backend-protocol-spec.md**](docs/backend-protocol-spec.md). + **API routes:** | Route | Method | Action | @@ -599,6 +603,14 @@ PathView uses JSON-based file formats for saving and sharing: The `.pvm` format is fully documented in [**docs/pvm-spec.md**](docs/pvm-spec.md). Use this spec if you are building tools that read or write PathView models (e.g., code generators, importers). A reference Python code generator is available at `scripts/pvm2py.py`. +### Specification Documents + +| Document | Audience | +|----------|----------| +| [**docs/pvm-spec.md**](docs/pvm-spec.md) | Building tools that read/write `.pvm` model files | +| [**docs/backend-protocol-spec.md**](docs/backend-protocol-spec.md) | Implementing a new execution backend (remote server, cloud worker, etc.) | +| [**docs/toolbox-spec.md**](docs/toolbox-spec.md) | Creating a third-party toolbox package for PathView | + ### Export Options - **File > Save** - Save complete model as `.pvm` From 281510e0b45169fd38cc600e60291a0360022576 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 12:29:37 +0100 Subject: [PATCH 09/27] Remove dead code from worker.py and FlaskBackend Co-Authored-By: KDW1 --- server/worker.py | 6 +----- src/lib/pyodide/backend/flask/backend.ts | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/server/worker.py b/server/worker.py index e73ba13..9fc3db0 100644 --- a/server/worker.py +++ b/server/worker.py @@ -26,7 +26,6 @@ # Worker state _namespace = {} -_clean_globals = set() _initialized = False # Streaming state @@ -111,7 +110,7 @@ def _ensure_package(pkg: dict) -> None: def initialize(packages: list[dict] | None = None) -> None: """Initialize the worker: install packages, import standard libs, capture clean globals.""" - global _initialized, _namespace, _clean_globals + global _initialized, _namespace if _initialized: send({"type": "ready"}) @@ -138,9 +137,6 @@ def initialize(packages: list[dict] | None = None) -> None: ) send({"type": "stderr", "value": f"Optional package {pkg.get('import', '?')} failed: {e}\n"}) - # Capture clean state for later cleanup - _clean_globals = set(_namespace.keys()) - _initialized = True send({"type": "ready"}) diff --git a/src/lib/pyodide/backend/flask/backend.ts b/src/lib/pyodide/backend/flask/backend.ts index 7f91c35..2b381d4 100644 --- a/src/lib/pyodide/backend/flask/backend.ts +++ b/src/lib/pyodide/backend/flask/backend.ts @@ -371,7 +371,6 @@ export class FlaskBackend implements Backend { const reader = resp.body.getReader(); const decoder = new TextDecoder(); let buffer = ''; - let currentEvent = ''; while (true) { const { value, done } = await reader.read(); From ceba7110319e6daff83a2d1a577e320a79d48172 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 17:51:00 +0100 Subject: [PATCH 10/27] Add pip-installable package with CLI and auto-detection Co-Authored-By: KDW1 --- .github/workflows/publish-pypi.yml | 77 +++++ .gitignore | 3 + README.md | 65 +++-- package.json | 3 +- pathview_server/__init__.py | 3 + pathview_server/__main__.py | 6 + pathview_server/app.py | 406 ++++++++++++++++++++++++++ pathview_server/cli.py | 55 ++++ {server => pathview_server}/worker.py | 0 pyproject.toml | 34 +++ scripts/build_package.py | 63 ++++ server/app.py | 378 ------------------------ server/requirements.txt | 2 - src/lib/pyodide/backend/index.ts | 29 ++ src/routes/+page.svelte | 6 +- 15 files changed, 727 insertions(+), 403 deletions(-) create mode 100644 .github/workflows/publish-pypi.yml create mode 100644 pathview_server/__init__.py create mode 100644 pathview_server/__main__.py create mode 100644 pathview_server/app.py create mode 100644 pathview_server/cli.py rename {server => pathview_server}/worker.py (100%) create mode 100644 pyproject.toml create mode 100644 scripts/build_package.py delete mode 100644 server/app.py delete mode 100644 server/requirements.txt diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 0000000..0fe59be --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,77 @@ +name: Publish to PyPI + +on: + release: + types: [published] + workflow_dispatch: + inputs: + publish_to: + description: 'Publish target' + required: true + default: 'testpypi' + type: choice + options: + - testpypi + - pypi + +permissions: + contents: read + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Python build tools + run: pip install build twine + + - name: Install Python dependencies + run: | + pip install -r scripts/config/requirements-pyodide.txt + pip install -r scripts/config/requirements-build.txt + + - name: Install Node dependencies + run: npm ci + + - name: Extract blocks from PathSim + run: npm run extract + + - name: Build package (frontend + wheel) + run: python scripts/build_package.py + + - name: Check distribution + run: twine check dist/* + + - name: Publish to Test PyPI + if: github.event_name == 'workflow_dispatch' && inputs.publish_to == 'testpypi' + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} + run: twine upload --repository testpypi dist/* + + - name: Publish to PyPI + if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.publish_to == 'pypi') + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: twine upload dist/* + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ diff --git a/.gitignore b/.gitignore index b4947e2..7ae8466 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,9 @@ tmpclaude-* __pycache__/ +*.egg-info/ +dist/ +pathview_server/static/ # Generated screenshots static/examples/screenshots/ diff --git a/README.md b/README.md index 34152ff..3908794 100644 --- a/README.md +++ b/README.md @@ -17,29 +17,39 @@ A web-based visual node editor for building and simulating dynamic systems with - [Plotly.js](https://plotly.com/javascript/) for interactive plots - [CodeMirror 6](https://codemirror.net/) for code editing -## Getting Started +## Installation + +### pip install (recommended for users) + +```bash +pip install pathview +pathview serve +``` + +This starts the PathView server with a local Python backend and opens your browser. No Node.js required. + +**Options:** +- `--port PORT` — server port (default: 5000) +- `--host HOST` — bind address (default: 127.0.0.1) +- `--no-browser` — don't auto-open the browser +- `--debug` — debug mode with auto-reload + +### Development setup ```bash npm install npm run dev ``` -To use the Flask backend (server-side Python): +To use the Flask backend during development: ```bash -pip install -r server/requirements.txt +pip install flask flask-cors npm run server # Start Flask backend on port 5000 npm run dev # Start Vite dev server (separate terminal) # Open http://localhost:5173/?backend=flask ``` -For production: - -```bash -npm run build -npm run preview -``` - ## Project Structure ``` @@ -82,10 +92,11 @@ src/ ├── routes/ # SvelteKit pages └── app.css # Global styles with CSS variables -server/ +pathview_server/ # Python package (pip install pathview) ├── app.py # Flask server (subprocess management, HTTP routes) ├── worker.py # REPL worker subprocess (Python execution) -└── requirements.txt # Server Python dependencies +├── cli.py # CLI entry point (pathview serve) +└── static/ # Bundled frontend (generated at build time) scripts/ ├── config/ # Configuration files for extraction @@ -467,15 +478,21 @@ Browser Tab Flask Server Worker Subprocess └──────────────┘ └──────────────────┘ └──────────────────┘ ``` -**Setup:** +**Standalone (pip package):** ```bash -pip install -r server/requirements.txt -npm run server # Starts Flask on port 5000 -npm run dev # Starts Vite dev server (separate terminal) +pip install pathview +pathview serve ``` -Then open `http://localhost:5173/?backend=flask`. +**Development (separate servers):** + +```bash +pip install flask flask-cors +npm run server # Starts Flask API on port 5000 +npm run dev # Starts Vite dev server (separate terminal) +# Open http://localhost:5173/?backend=flask +``` **Key properties:** - **Process isolation** — each session gets its own Python subprocess @@ -666,7 +683,8 @@ https://view.pathsim.org/?modelgh=pathsim/pathview/static/examples/feedback-syst |--------|---------| | `npm run dev` | Start Vite development server | | `npm run server` | Start Flask backend server (port 5000) | -| `npm run build` | Production build | +| `npm run build` | Production build (GitHub Pages) | +| `npm run build:package` | Build pip package (frontend + wheel) | | `npm run preview` | Preview production build | | `npm run check` | TypeScript/Svelte type checking | | `npm run lint` | Run ESLint | @@ -783,7 +801,9 @@ Port labels show the name of each input/output port alongside the node. Toggle g ## Deployment -PathView uses a dual deployment strategy with automatic versioning: +PathView has two deployment targets: + +### GitHub Pages (web) | Trigger | What happens | Deployed to | |---------|--------------|-------------| @@ -791,6 +811,13 @@ PathView uses a dual deployment strategy with automatic versioning: | Release published | Bump `package.json`, build, deploy | [view.pathsim.org/](https://view.pathsim.org/) | | Manual dispatch | Choose `dev` or `release` | Respective path | +### PyPI (pip package) + +| Trigger | What happens | Published to | +|---------|--------------|--------------| +| Release published | Build frontend + wheel, publish | [pypi.org/project/pathview](https://pypi.org/project/pathview/) | +| Manual dispatch | Choose `testpypi` or `pypi` | Respective index | + ### How it works 1. Both versions deploy to the `deployment` branch using GitHub Actions diff --git a/package.json b/package.json index a2a30f9..7ea72aa 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "lint": "eslint .", "format": "prettier --write .", - "server": "python server/app.py" + "server": "python -m pathview_server.app", + "build:package": "python scripts/build_package.py" }, "devDependencies": { "@sveltejs/adapter-static": "^3.0.0", diff --git a/pathview_server/__init__.py b/pathview_server/__init__.py new file mode 100644 index 0000000..8808f05 --- /dev/null +++ b/pathview_server/__init__.py @@ -0,0 +1,3 @@ +"""PathView Server — local Flask backend for PathView.""" + +__version__ = "0.5.0" diff --git a/pathview_server/__main__.py b/pathview_server/__main__.py new file mode 100644 index 0000000..4082f5c --- /dev/null +++ b/pathview_server/__main__.py @@ -0,0 +1,6 @@ +"""Entry point for: python -m pathview_server""" + +from pathview_server.cli import main + +if __name__ == "__main__": + main() diff --git a/pathview_server/app.py b/pathview_server/app.py new file mode 100644 index 0000000..7016925 --- /dev/null +++ b/pathview_server/app.py @@ -0,0 +1,406 @@ +""" +Flask server for PathView backend. + +Manages worker subprocesses per session. Routes translate HTTP requests +into subprocess messages and relay responses back. + +Each session gets its own worker subprocess with an isolated Python namespace. +""" + +import os +import sys +import json +import subprocess +import threading +import time +import uuid +import atexit +from pathlib import Path + +from flask import Flask, Response, request, jsonify, send_from_directory +from flask_cors import CORS + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +SESSION_TTL = 3600 # 1 hour of inactivity before cleanup +CLEANUP_INTERVAL = 60 # Check for stale sessions every 60 seconds +REQUEST_TIMEOUT = 300 # 5 minutes default timeout for exec/eval +WORKER_SCRIPT = str(Path(__file__).parent / "worker.py") + +# --------------------------------------------------------------------------- +# Session management +# --------------------------------------------------------------------------- + +class Session: + """A worker subprocess bound to a session.""" + + def __init__(self, session_id: str): + self.session_id = session_id + self.last_active = time.time() + self.lock = threading.Lock() + self.process = subprocess.Popen( + [sys.executable, "-u", WORKER_SCRIPT], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, # line buffered + ) + self._initialized = False + + def send_message(self, msg: dict) -> None: + """Write a JSON message to the subprocess stdin.""" + self.last_active = time.time() + line = json.dumps(msg) + "\n" + self.process.stdin.write(line) + self.process.stdin.flush() + + def read_line(self) -> dict | None: + """Read one JSON line from the subprocess stdout.""" + line = self.process.stdout.readline() + if not line: + return None + return json.loads(line.strip()) + + def ensure_initialized(self, packages: list[dict] | None = None) -> list[dict]: + """Initialize the worker if not already done. Returns any messages received.""" + if self._initialized: + return [] + messages = [] + init_msg = {"type": "init"} + if packages: + init_msg["packages"] = packages + self.send_message(init_msg) + while True: + resp = self.read_line() + if resp is None: + raise RuntimeError("Worker process died during initialization") + messages.append(resp) + if resp.get("type") == "ready": + self._initialized = True + break + if resp.get("type") == "error": + raise RuntimeError(resp.get("error", "Unknown init error")) + return messages + + def is_alive(self) -> bool: + return self.process.poll() is None + + def kill(self) -> None: + """Kill the subprocess.""" + try: + self.process.stdin.close() + except Exception: + pass + try: + self.process.kill() + self.process.wait(timeout=5) + except Exception: + pass + + +# Global session store +_sessions: dict[str, Session] = {} +_sessions_lock = threading.Lock() + + +def get_or_create_session(session_id: str) -> Session: + """Get an existing session or create a new one.""" + with _sessions_lock: + session = _sessions.get(session_id) + if session and not session.is_alive(): + # Dead process, remove stale entry + _sessions.pop(session_id, None) + session = None + if session is None: + session = Session(session_id) + _sessions[session_id] = session + return session + + +def remove_session(session_id: str) -> None: + """Kill and remove a session.""" + with _sessions_lock: + session = _sessions.pop(session_id, None) + if session: + session.kill() + + +def cleanup_stale_sessions() -> None: + """Remove sessions that have been inactive beyond TTL.""" + while True: + time.sleep(CLEANUP_INTERVAL) + now = time.time() + stale = [] + with _sessions_lock: + for sid, session in _sessions.items(): + if now - session.last_active > SESSION_TTL: + stale.append(sid) + for sid in stale: + remove_session(sid) + + +# Start cleanup thread +_cleanup_thread = threading.Thread(target=cleanup_stale_sessions, daemon=True) +_cleanup_thread.start() + + +def _get_session_id() -> str: + """Extract session ID from request headers or generate one.""" + return request.headers.get("X-Session-ID") or str(uuid.uuid4()) + + +# --------------------------------------------------------------------------- +# App factory +# --------------------------------------------------------------------------- + +def create_app(serve_static: bool = False) -> Flask: + """Create the Flask application. + + Args: + serve_static: If True, serve the bundled frontend and skip CORS. + If False, API-only mode with CORS (for dev with Vite). + """ + app = Flask(__name__) + + if not serve_static: + CORS(app) + + # ----------------------------------------------------------------------- + # API routes + # ----------------------------------------------------------------------- + + @app.route("/api/health", methods=["GET"]) + def health(): + return jsonify({"status": "ok"}) + + @app.route("/api/init", methods=["POST"]) + def api_init(): + """Initialize a session's worker with packages from the frontend config.""" + session_id = _get_session_id() + data = request.get_json(force=True) + packages = data.get("packages", []) + + session = get_or_create_session(session_id) + with session.lock: + try: + messages = session.ensure_initialized(packages=packages) + return jsonify({"type": "ready", "messages": messages}) + except Exception as e: + return jsonify({"type": "error", "error": str(e)}), 500 + + @app.route("/api/exec", methods=["POST"]) + def api_exec(): + """Execute Python code in the session's worker.""" + session_id = _get_session_id() + data = request.get_json(force=True) + code = data.get("code", "") + msg_id = data.get("id", str(uuid.uuid4())) + + session = get_or_create_session(session_id) + with session.lock: + try: + session.ensure_initialized() + session.send_message({"type": "exec", "id": msg_id, "code": code}) + + stdout_lines = [] + stderr_lines = [] + while True: + resp = session.read_line() + if resp is None: + return jsonify({"type": "error", "id": msg_id, "error": "Worker process died"}), 500 + resp_type = resp.get("type") + if resp_type == "stdout": + stdout_lines.append(resp.get("value", "")) + elif resp_type == "stderr": + stderr_lines.append(resp.get("value", "")) + elif resp_type == "ok" and resp.get("id") == msg_id: + result = {"type": "ok", "id": msg_id} + if stdout_lines: + result["stdout"] = "".join(stdout_lines) + if stderr_lines: + result["stderr"] = "".join(stderr_lines) + return jsonify(result) + elif resp_type == "error" and resp.get("id") == msg_id: + result = resp + if stdout_lines: + result["stdout"] = "".join(stdout_lines) + if stderr_lines: + result["stderr"] = "".join(stderr_lines) + return jsonify(result), 400 + + except Exception as e: + return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 + + @app.route("/api/eval", methods=["POST"]) + def api_eval(): + """Evaluate a Python expression in the session's worker.""" + session_id = _get_session_id() + data = request.get_json(force=True) + expr = data.get("expr", "") + msg_id = data.get("id", str(uuid.uuid4())) + + session = get_or_create_session(session_id) + with session.lock: + try: + session.ensure_initialized() + session.send_message({"type": "eval", "id": msg_id, "expr": expr}) + + stdout_lines = [] + stderr_lines = [] + while True: + resp = session.read_line() + if resp is None: + return jsonify({"type": "error", "id": msg_id, "error": "Worker process died"}), 500 + resp_type = resp.get("type") + if resp_type == "stdout": + stdout_lines.append(resp.get("value", "")) + elif resp_type == "stderr": + stderr_lines.append(resp.get("value", "")) + elif resp_type == "value" and resp.get("id") == msg_id: + result = resp + if stdout_lines: + result["stdout"] = "".join(stdout_lines) + if stderr_lines: + result["stderr"] = "".join(stderr_lines) + return jsonify(result) + elif resp_type == "error" and resp.get("id") == msg_id: + result = resp + if stdout_lines: + result["stdout"] = "".join(stdout_lines) + if stderr_lines: + result["stderr"] = "".join(stderr_lines) + return jsonify(result), 400 + + except Exception as e: + return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 + + @app.route("/api/stream", methods=["POST"]) + def api_stream(): + """Start a streaming simulation, returning results as SSE.""" + session_id = _get_session_id() + data = request.get_json(force=True) + expr = data.get("expr", "") + msg_id = data.get("id", str(uuid.uuid4())) + + session = get_or_create_session(session_id) + + def generate(): + with session.lock: + try: + session.ensure_initialized() + session.send_message({"type": "stream-start", "id": msg_id, "expr": expr}) + + while True: + resp = session.read_line() + if resp is None: + yield f"event: error\ndata: {json.dumps({'error': 'Worker process died'})}\n\n" + break + resp_type = resp.get("type") + + if resp_type == "stream-data": + yield f"event: data\ndata: {json.dumps({'done': False, 'result': json.loads(resp.get('value', '{}'))})}\n\n" + elif resp_type == "stream-done": + yield f"event: done\ndata: {{}}\n\n" + break + elif resp_type == "stdout": + yield f"event: stdout\ndata: {json.dumps(resp.get('value', ''))}\n\n" + elif resp_type == "stderr": + yield f"event: stderr\ndata: {json.dumps(resp.get('value', ''))}\n\n" + elif resp_type == "error": + yield f"event: error\ndata: {json.dumps({'error': resp.get('error', ''), 'traceback': resp.get('traceback', '')})}\n\n" + break + + except Exception as e: + yield f"event: error\ndata: {json.dumps({'error': str(e)})}\n\n" + + return Response( + generate(), + mimetype="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + }, + ) + + @app.route("/api/stream/exec", methods=["POST"]) + def api_stream_exec(): + """Queue code to execute during an active stream.""" + session_id = _get_session_id() + data = request.get_json(force=True) + code = data.get("code", "") + + session = get_or_create_session(session_id) + try: + session.send_message({"type": "stream-exec", "code": code}) + return jsonify({"status": "queued"}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + @app.route("/api/stream/stop", methods=["POST"]) + def api_stream_stop(): + """Stop an active streaming session.""" + session_id = _get_session_id() + + session = get_or_create_session(session_id) + try: + session.send_message({"type": "stream-stop"}) + return jsonify({"status": "stopped"}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + @app.route("/api/session", methods=["DELETE"]) + def api_session_delete(): + """Kill a session's worker subprocess.""" + session_id = _get_session_id() + remove_session(session_id) + return jsonify({"status": "terminated"}) + + # ----------------------------------------------------------------------- + # Static file serving (pip package mode) + # ----------------------------------------------------------------------- + + if serve_static: + static_dir = Path(__file__).parent / "static" + + @app.route("/", defaults={"path": ""}) + @app.route("/") + def serve_frontend(path): + """Serve the bundled SvelteKit frontend with SPA fallback.""" + if path.startswith("api/"): + return jsonify({"error": "Not found"}), 404 + + file_path = static_dir / path + if path and file_path.is_file(): + return send_from_directory(static_dir, path) + + # SPA fallback + return send_from_directory(static_dir, "index.html") + + return app + + +# --------------------------------------------------------------------------- +# Cleanup on exit +# --------------------------------------------------------------------------- + +@atexit.register +def _cleanup_all_sessions(): + with _sessions_lock: + for session in _sessions.values(): + session.kill() + _sessions.clear() + + +# --------------------------------------------------------------------------- +# Entry point (dev mode: API-only with CORS) +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + port = int(os.environ.get("PORT", 5000)) + debug = os.environ.get("FLASK_DEBUG", "1") == "1" + print(f"PathView Flask backend (API-only) starting on port {port}") + app = create_app(serve_static=False) + app.run(host="0.0.0.0", port=port, debug=debug, threaded=True) diff --git a/pathview_server/cli.py b/pathview_server/cli.py new file mode 100644 index 0000000..d45f35c --- /dev/null +++ b/pathview_server/cli.py @@ -0,0 +1,55 @@ +"""CLI entry point for the pathview command.""" + +import argparse +import sys +import threading +import time +import webbrowser + +from pathview_server import __version__ + + +def main(): + parser = argparse.ArgumentParser( + prog="pathview", + description="PathView — visual node editor for dynamic systems", + ) + parser.add_argument("command", nargs="?", default="serve", choices=["serve"], + help="Command to run (default: serve)") + parser.add_argument("--port", type=int, default=5000, + help="Port to run the server on (default: 5000)") + parser.add_argument("--host", type=str, default="127.0.0.1", + help="Host to bind to (default: 127.0.0.1)") + parser.add_argument("--no-browser", action="store_true", + help="Don't automatically open the browser") + parser.add_argument("--debug", action="store_true", + help="Run in debug mode") + parser.add_argument("--version", action="version", + version=f"pathview {__version__}") + + args = parser.parse_args() + + from pathview_server.app import create_app + + app = create_app(serve_static=True) + + if not args.no_browser: + def open_browser(): + time.sleep(1.5) + webbrowser.open(f"http://{args.host}:{args.port}") + + threading.Thread(target=open_browser, daemon=True).start() + + print(f"PathView v{__version__}") + print(f"Running at http://{args.host}:{args.port}") + print("Press Ctrl+C to stop\n") + + try: + app.run(host=args.host, port=args.port, debug=args.debug, threaded=True) + except KeyboardInterrupt: + print("\nStopping PathView server...") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/server/worker.py b/pathview_server/worker.py similarity index 100% rename from server/worker.py rename to pathview_server/worker.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..167b30e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pathview" +version = "0.5.0" +description = "Visual node editor for building and simulating dynamic systems with PathSim" +readme = "README.md" +license = {text = "MIT"} +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering", +] +dependencies = [ + "flask>=3.0", + "flask-cors>=4.0", +] + +[project.urls] +Homepage = "https://view.pathsim.org" +Repository = "https://github.com/pathsim/pathview" + +[project.scripts] +pathview = "pathview_server.cli:main" + +[tool.setuptools] +packages = ["pathview_server"] + +[tool.setuptools.package-data] +pathview_server = ["static/**/*"] diff --git a/scripts/build_package.py b/scripts/build_package.py new file mode 100644 index 0000000..087c493 --- /dev/null +++ b/scripts/build_package.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +""" +Build script for the PathView PyPI package. + +1. Builds the SvelteKit frontend (vite build) +2. Copies build/ output to pathview_server/static/ +3. Builds the Python wheel +""" + +import os +import sys +import shutil +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).parent.parent +BUILD_DIR = REPO_ROOT / "build" +STATIC_DIR = REPO_ROOT / "pathview_server" / "static" + + +def run(cmd, **kwargs): + print(f" > {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=kwargs.pop("cwd", REPO_ROOT), **kwargs) + if result.returncode != 0: + print(f"ERROR: command failed (exit {result.returncode})") + sys.exit(result.returncode) + + +def main(): + print("[1/4] Cleaning previous builds...") + for d in [BUILD_DIR, STATIC_DIR, REPO_ROOT / "dist"]: + if d.exists(): + shutil.rmtree(d) + + # Remove egg-info + for p in REPO_ROOT.glob("*.egg-info"): + shutil.rmtree(p) + + print("[2/4] Building SvelteKit frontend...") + env = os.environ.copy() + env["BASE_PATH"] = "" + run(["npx", "vite", "build"], env=env) + + if not (BUILD_DIR / "index.html").exists(): + print("ERROR: build/index.html not found") + sys.exit(1) + + print("[3/4] Copying frontend to pathview_server/static/...") + shutil.copytree(BUILD_DIR, STATIC_DIR) + print(f" Copied {sum(1 for _ in STATIC_DIR.rglob('*') if _.is_file())} files") + + print("[4/4] Building Python wheel...") + run([sys.executable, "-m", "build"]) + + print("\nDone! Output:") + dist = REPO_ROOT / "dist" + if dist.exists(): + for f in sorted(dist.iterdir()): + print(f" {f.name}") + + +if __name__ == "__main__": + main() diff --git a/server/app.py b/server/app.py deleted file mode 100644 index 6f8eff4..0000000 --- a/server/app.py +++ /dev/null @@ -1,378 +0,0 @@ -""" -Flask server for PathView backend. - -Manages worker subprocesses per session. Routes translate HTTP requests -into subprocess messages and relay responses back. - -Each session gets its own worker subprocess with an isolated Python namespace. -""" - -import os -import sys -import json -import subprocess -import threading -import time -import uuid -import atexit -from pathlib import Path - -from flask import Flask, Response, request, jsonify -from flask_cors import CORS - -app = Flask(__name__) -CORS(app) - -# --------------------------------------------------------------------------- -# Configuration -# --------------------------------------------------------------------------- - -SESSION_TTL = 3600 # 1 hour of inactivity before cleanup -CLEANUP_INTERVAL = 60 # Check for stale sessions every 60 seconds -REQUEST_TIMEOUT = 300 # 5 minutes default timeout for exec/eval -WORKER_SCRIPT = str(Path(__file__).parent / "worker.py") - -# --------------------------------------------------------------------------- -# Session management -# --------------------------------------------------------------------------- - -class Session: - """A worker subprocess bound to a session.""" - - def __init__(self, session_id: str): - self.session_id = session_id - self.last_active = time.time() - self.lock = threading.Lock() - self.process = subprocess.Popen( - [sys.executable, "-u", WORKER_SCRIPT], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - bufsize=1, # line buffered - ) - self._initialized = False - - def send_message(self, msg: dict) -> None: - """Write a JSON message to the subprocess stdin.""" - self.last_active = time.time() - line = json.dumps(msg) + "\n" - self.process.stdin.write(line) - self.process.stdin.flush() - - def read_line(self) -> dict | None: - """Read one JSON line from the subprocess stdout.""" - line = self.process.stdout.readline() - if not line: - return None - return json.loads(line.strip()) - - def ensure_initialized(self, packages: list[dict] | None = None) -> list[dict]: - """Initialize the worker if not already done. Returns any messages received.""" - if self._initialized: - return [] - messages = [] - init_msg = {"type": "init"} - if packages: - init_msg["packages"] = packages - self.send_message(init_msg) - while True: - resp = self.read_line() - if resp is None: - raise RuntimeError("Worker process died during initialization") - messages.append(resp) - if resp.get("type") == "ready": - self._initialized = True - break - if resp.get("type") == "error": - raise RuntimeError(resp.get("error", "Unknown init error")) - return messages - - def is_alive(self) -> bool: - return self.process.poll() is None - - def kill(self) -> None: - """Kill the subprocess.""" - try: - self.process.stdin.close() - except Exception: - pass - try: - self.process.kill() - self.process.wait(timeout=5) - except Exception: - pass - - -# Global session store -_sessions: dict[str, Session] = {} -_sessions_lock = threading.Lock() - - -def get_or_create_session(session_id: str) -> Session: - """Get an existing session or create a new one.""" - with _sessions_lock: - session = _sessions.get(session_id) - if session and not session.is_alive(): - # Dead process, remove stale entry - _sessions.pop(session_id, None) - session = None - if session is None: - session = Session(session_id) - _sessions[session_id] = session - return session - - -def remove_session(session_id: str) -> None: - """Kill and remove a session.""" - with _sessions_lock: - session = _sessions.pop(session_id, None) - if session: - session.kill() - - -def cleanup_stale_sessions() -> None: - """Remove sessions that have been inactive beyond TTL.""" - while True: - time.sleep(CLEANUP_INTERVAL) - now = time.time() - stale = [] - with _sessions_lock: - for sid, session in _sessions.items(): - if now - session.last_active > SESSION_TTL: - stale.append(sid) - for sid in stale: - remove_session(sid) - - -# Start cleanup thread -_cleanup_thread = threading.Thread(target=cleanup_stale_sessions, daemon=True) -_cleanup_thread.start() - - -def _get_session_id() -> str: - """Extract session ID from request headers or generate one.""" - return request.headers.get("X-Session-ID") or str(uuid.uuid4()) - - -# --------------------------------------------------------------------------- -# Routes -# --------------------------------------------------------------------------- - -@app.route("/api/health", methods=["GET"]) -def health(): - return jsonify({"status": "ok"}) - - -@app.route("/api/init", methods=["POST"]) -def api_init(): - """Initialize a session's worker with packages from the frontend config.""" - session_id = _get_session_id() - data = request.get_json(force=True) - packages = data.get("packages", []) - - session = get_or_create_session(session_id) - with session.lock: - try: - messages = session.ensure_initialized(packages=packages) - # Return all progress/stdout/stderr messages collected during init - return jsonify({"type": "ready", "messages": messages}) - except Exception as e: - return jsonify({"type": "error", "error": str(e)}), 500 - - -@app.route("/api/exec", methods=["POST"]) -def api_exec(): - """Execute Python code in the session's worker.""" - session_id = _get_session_id() - data = request.get_json(force=True) - code = data.get("code", "") - msg_id = data.get("id", str(uuid.uuid4())) - - session = get_or_create_session(session_id) - with session.lock: - try: - session.ensure_initialized() - session.send_message({"type": "exec", "id": msg_id, "code": code}) - - # Collect responses until we get ok/error for this id - stdout_lines = [] - stderr_lines = [] - while True: - resp = session.read_line() - if resp is None: - return jsonify({"type": "error", "id": msg_id, "error": "Worker process died"}), 500 - resp_type = resp.get("type") - if resp_type == "stdout": - stdout_lines.append(resp.get("value", "")) - elif resp_type == "stderr": - stderr_lines.append(resp.get("value", "")) - elif resp_type == "ok" and resp.get("id") == msg_id: - result = {"type": "ok", "id": msg_id} - if stdout_lines: - result["stdout"] = "".join(stdout_lines) - if stderr_lines: - result["stderr"] = "".join(stderr_lines) - return jsonify(result) - elif resp_type == "error" and resp.get("id") == msg_id: - result = resp - if stdout_lines: - result["stdout"] = "".join(stdout_lines) - if stderr_lines: - result["stderr"] = "".join(stderr_lines) - return jsonify(result), 400 - - except Exception as e: - return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 - - -@app.route("/api/eval", methods=["POST"]) -def api_eval(): - """Evaluate a Python expression in the session's worker.""" - session_id = _get_session_id() - data = request.get_json(force=True) - expr = data.get("expr", "") - msg_id = data.get("id", str(uuid.uuid4())) - - session = get_or_create_session(session_id) - with session.lock: - try: - session.ensure_initialized() - session.send_message({"type": "eval", "id": msg_id, "expr": expr}) - - stdout_lines = [] - stderr_lines = [] - while True: - resp = session.read_line() - if resp is None: - return jsonify({"type": "error", "id": msg_id, "error": "Worker process died"}), 500 - resp_type = resp.get("type") - if resp_type == "stdout": - stdout_lines.append(resp.get("value", "")) - elif resp_type == "stderr": - stderr_lines.append(resp.get("value", "")) - elif resp_type == "value" and resp.get("id") == msg_id: - result = resp - if stdout_lines: - result["stdout"] = "".join(stdout_lines) - if stderr_lines: - result["stderr"] = "".join(stderr_lines) - return jsonify(result) - elif resp_type == "error" and resp.get("id") == msg_id: - result = resp - if stdout_lines: - result["stdout"] = "".join(stdout_lines) - if stderr_lines: - result["stderr"] = "".join(stderr_lines) - return jsonify(result), 400 - - except Exception as e: - return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 - - -@app.route("/api/stream", methods=["POST"]) -def api_stream(): - """Start a streaming simulation, returning results as SSE.""" - session_id = _get_session_id() - data = request.get_json(force=True) - expr = data.get("expr", "") - msg_id = data.get("id", str(uuid.uuid4())) - - session = get_or_create_session(session_id) - - def generate(): - with session.lock: - try: - session.ensure_initialized() - session.send_message({"type": "stream-start", "id": msg_id, "expr": expr}) - - while True: - resp = session.read_line() - if resp is None: - yield f"event: error\ndata: {json.dumps({'error': 'Worker process died'})}\n\n" - break - resp_type = resp.get("type") - - if resp_type == "stream-data": - yield f"event: data\ndata: {json.dumps({'done': False, 'result': json.loads(resp.get('value', '{}'))})}\n\n" - elif resp_type == "stream-done": - yield f"event: done\ndata: {{}}\n\n" - break - elif resp_type == "stdout": - yield f"event: stdout\ndata: {json.dumps(resp.get('value', ''))}\n\n" - elif resp_type == "stderr": - yield f"event: stderr\ndata: {json.dumps(resp.get('value', ''))}\n\n" - elif resp_type == "error": - yield f"event: error\ndata: {json.dumps({'error': resp.get('error', ''), 'traceback': resp.get('traceback', '')})}\n\n" - break - - except Exception as e: - yield f"event: error\ndata: {json.dumps({'error': str(e)})}\n\n" - - return Response( - generate(), - mimetype="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "X-Accel-Buffering": "no", - }, - ) - - -@app.route("/api/stream/exec", methods=["POST"]) -def api_stream_exec(): - """Queue code to execute during an active stream.""" - session_id = _get_session_id() - data = request.get_json(force=True) - code = data.get("code", "") - - session = get_or_create_session(session_id) - try: - session.send_message({"type": "stream-exec", "code": code}) - return jsonify({"status": "queued"}) - except Exception as e: - return jsonify({"error": str(e)}), 500 - - -@app.route("/api/stream/stop", methods=["POST"]) -def api_stream_stop(): - """Stop an active streaming session.""" - session_id = _get_session_id() - - session = get_or_create_session(session_id) - try: - session.send_message({"type": "stream-stop"}) - return jsonify({"status": "stopped"}) - except Exception as e: - return jsonify({"error": str(e)}), 500 - - -@app.route("/api/session", methods=["DELETE"]) -def api_session_delete(): - """Kill a session's worker subprocess.""" - session_id = _get_session_id() - remove_session(session_id) - return jsonify({"status": "terminated"}) - - -# --------------------------------------------------------------------------- -# Cleanup on exit -# --------------------------------------------------------------------------- - -@atexit.register -def _cleanup_all_sessions(): - with _sessions_lock: - for session in _sessions.values(): - session.kill() - _sessions.clear() - - -# --------------------------------------------------------------------------- -# Entry point -# --------------------------------------------------------------------------- - -if __name__ == "__main__": - port = int(os.environ.get("PORT", 5000)) - debug = os.environ.get("FLASK_DEBUG", "1") == "1" - print(f"PathView Flask backend starting on port {port}") - app.run(host="0.0.0.0", port=port, debug=debug, threaded=True) diff --git a/server/requirements.txt b/server/requirements.txt deleted file mode 100644 index 3535b45..0000000 --- a/server/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -flask>=3.0 -flask-cors>=4.0 diff --git a/src/lib/pyodide/backend/index.ts b/src/lib/pyodide/backend/index.ts index 5a227a1..86c6aae 100644 --- a/src/lib/pyodide/backend/index.ts +++ b/src/lib/pyodide/backend/index.ts @@ -56,6 +56,35 @@ export function initBackendFromUrl(): void { } } +/** + * Auto-detect if a Flask backend is available at the same origin. + * Used when the frontend is served by the Flask server (pip package mode). + * URL parameters take precedence — if `?backend=` is set, auto-detection is skipped. + */ +export async function autoDetectBackend(): Promise { + if (typeof window === 'undefined') return; + + // URL params override auto-detection + const params = new URLSearchParams(window.location.search); + if (params.has('backend')) return; + + try { + const response = await fetch('/api/health', { + method: 'GET', + signal: AbortSignal.timeout(2000) + }); + if (response.ok) { + const data = await response.json(); + if (data.status === 'ok') { + setFlaskHost(window.location.origin); + switchBackend('flask'); + } + } + } catch { + // No Flask backend at same origin — will use Pyodide + } +} + // Alias for backward compatibility export const replState = { subscribe: backendState.subscribe diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 268e53f..d6d5c1f 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -40,7 +40,7 @@ import { openEventDialog } from '$lib/stores/eventDialog'; import type { MenuItemType } from '$lib/components/ContextMenu.svelte'; import { pyodideState, simulationState, initPyodide, stopSimulation, continueStreamingSimulation } from '$lib/pyodide/bridge'; - import { initBackendFromUrl } from '$lib/pyodide/backend'; + import { initBackendFromUrl, autoDetectBackend } from '$lib/pyodide/backend'; import { runGraphStreamingSimulation, validateGraphSimulation } from '$lib/pyodide/pathsimRunner'; import { consoleStore } from '$lib/stores/console'; import { newGraph, saveFile, saveAsFile, setupAutoSave, clearAutoSave, debouncedAutoSave, openImportDialog, importFromUrl, currentFileName } from '$lib/schema/fileOps'; @@ -382,8 +382,8 @@ const continueTooltip = { text: "Continue", shortcut: "Shift+Enter" }; onMount(() => { - // Check URL params for backend selection (e.g. ?backend=flask&host=http://localhost:5000) - initBackendFromUrl(); + // Auto-detect same-origin Flask backend (pip package mode), then check URL params + autoDetectBackend().then(() => initBackendFromUrl()); // Subscribe to stores (with cleanup) const unsubPinnedPreviews = pinnedPreviewsStore.subscribe((pinned) => { From 8c042c0b1922158ddc23136f0a29355cb29ee9d8 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:05:06 +0100 Subject: [PATCH 11/27] Fix streaming, logging, and review issues for Flask backend - Migrate streaming from SSE to polling (start+poll pattern) - Fix stdin race condition with noop flush and leftover queue - Fix logging capture with permanent _ProtocolWriter on sys.stdout/stderr - Add recursion guard to _ProtocolWriter to prevent infinite loops - Fix ensureServerInit to retry on failure instead of caching rejection - Fix initBackendFromUrl to call init() for proper callback setup - Add resp.ok checks and poll response validation in FlaskBackend - Align stopStreaming to wait for server confirmation like Pyodide - Validate X-Session-ID header, return 400 when missing - Prevent orphan sessions in poll/exec/stop by using lookup instead of create - Move cleanup thread start into create_app factory - Fix requires-python to >=3.10 and add numpy dependency - Remove dead code: _capture_output, REQUEST_TIMEOUT, unused imports Co-Authored-By: KDW1 --- pathview_server/app.py | 178 +++++++--- pathview_server/worker.py | 182 ++++++----- pyproject.toml | 3 +- scripts/build_package.py | 4 +- src/lib/pyodide/backend/flask/backend.ts | 392 +++++++++++------------ src/lib/pyodide/backend/index.ts | 7 +- 6 files changed, 426 insertions(+), 340 deletions(-) diff --git a/pathview_server/app.py b/pathview_server/app.py index 7016925..a909e4e 100644 --- a/pathview_server/app.py +++ b/pathview_server/app.py @@ -10,6 +10,7 @@ import os import sys import json +import queue import subprocess import threading import time @@ -17,7 +18,7 @@ import atexit from pathlib import Path -from flask import Flask, Response, request, jsonify, send_from_directory +from flask import Flask, request, jsonify, send_from_directory from flask_cors import CORS # --------------------------------------------------------------------------- @@ -26,7 +27,6 @@ SESSION_TTL = 3600 # 1 hour of inactivity before cleanup CLEANUP_INTERVAL = 60 # Check for stale sessions every 60 seconds -REQUEST_TIMEOUT = 300 # 5 minutes default timeout for exec/eval WORKER_SCRIPT = str(Path(__file__).parent / "worker.py") # --------------------------------------------------------------------------- @@ -49,6 +49,10 @@ def __init__(self, session_id: str): bufsize=1, # line buffered ) self._initialized = False + # Streaming state: background thread reads worker stdout into a queue + self._stream_queue: queue.Queue[dict] = queue.Queue() + self._stream_reader: threading.Thread | None = None + self._streaming = False def send_message(self, msg: dict) -> None: """Write a JSON message to the subprocess stdin.""" @@ -85,11 +89,63 @@ def ensure_initialized(self, packages: list[dict] | None = None) -> list[dict]: raise RuntimeError(resp.get("error", "Unknown init error")) return messages + def start_stream_reader(self) -> None: + """Start a background thread that reads worker stdout into the stream queue.""" + self._streaming = True + # Clear any stale messages + while not self._stream_queue.empty(): + try: + self._stream_queue.get_nowait() + except queue.Empty: + break + + def reader(): + while self._streaming: + resp = self.read_line() + if resp is None: + self._stream_queue.put({"type": "error", "error": "Worker process died"}) + self._streaming = False + break + self._stream_queue.put(resp) + if resp.get("type") in ("stream-done", "error"): + self._streaming = False + break + + self._stream_reader = threading.Thread(target=reader, daemon=True) + self._stream_reader.start() + + def stop_stream_reader(self) -> None: + """Signal the stream reader to stop.""" + self._streaming = False + + def flush_worker_reader(self) -> None: + """Send a noop message to unblock the worker's stdin reader thread. + + After streaming ends, the worker's reader thread is blocked on stdin.readline(). + This sends a harmless message so the reader thread wakes up, checks stop_event, + and exits — allowing the main thread to resume reading stdin safely. + """ + try: + self.send_message({"type": "noop"}) + except Exception: + pass + + def drain_stream_queue(self) -> list[dict]: + """Drain all messages currently in the stream queue.""" + messages = [] + while True: + try: + messages.append(self._stream_queue.get_nowait()) + except queue.Empty: + break + return messages + def is_alive(self) -> bool: return self.process.poll() is None def kill(self) -> None: """Kill the subprocess.""" + self._streaming = False try: self.process.stdin.close() except Exception: @@ -142,14 +198,22 @@ def cleanup_stale_sessions() -> None: remove_session(sid) -# Start cleanup thread -_cleanup_thread = threading.Thread(target=cleanup_stale_sessions, daemon=True) -_cleanup_thread.start() +_cleanup_started = False + + +def _start_cleanup_thread() -> None: + """Start the cleanup thread once (idempotent).""" + global _cleanup_started + if _cleanup_started: + return + _cleanup_started = True + t = threading.Thread(target=cleanup_stale_sessions, daemon=True) + t.start() -def _get_session_id() -> str: - """Extract session ID from request headers or generate one.""" - return request.headers.get("X-Session-ID") or str(uuid.uuid4()) +def _get_session_id() -> str | None: + """Extract session ID from request headers. Returns None if missing.""" + return request.headers.get("X-Session-ID") # --------------------------------------------------------------------------- @@ -164,6 +228,7 @@ def create_app(serve_static: bool = False) -> Flask: If False, API-only mode with CORS (for dev with Vite). """ app = Flask(__name__) + _start_cleanup_thread() if not serve_static: CORS(app) @@ -180,6 +245,8 @@ def health(): def api_init(): """Initialize a session's worker with packages from the frontend config.""" session_id = _get_session_id() + if not session_id: + return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 data = request.get_json(force=True) packages = data.get("packages", []) @@ -195,6 +262,8 @@ def api_init(): def api_exec(): """Execute Python code in the session's worker.""" session_id = _get_session_id() + if not session_id: + return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 data = request.get_json(force=True) code = data.get("code", "") msg_id = data.get("id", str(uuid.uuid4())) @@ -238,6 +307,8 @@ def api_exec(): def api_eval(): """Evaluate a Python expression in the session's worker.""" session_id = _get_session_id() + if not session_id: + return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 data = request.get_json(force=True) expr = data.get("expr", "") msg_id = data.get("id", str(uuid.uuid4())) @@ -277,62 +348,62 @@ def api_eval(): except Exception as e: return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 - @app.route("/api/stream", methods=["POST"]) - def api_stream(): - """Start a streaming simulation, returning results as SSE.""" + @app.route("/api/stream/start", methods=["POST"]) + def api_stream_start(): + """Start streaming — sends stream-start to worker and returns immediately. + + A background thread reads worker stdout into a queue. The frontend + polls /api/stream/poll to drain that queue, mirroring how the Pyodide + worker sends stream-data / stream-done messages via postMessage. + """ session_id = _get_session_id() + if not session_id: + return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 data = request.get_json(force=True) expr = data.get("expr", "") msg_id = data.get("id", str(uuid.uuid4())) session = get_or_create_session(session_id) + with session.lock: + try: + session.ensure_initialized() + session.send_message({"type": "stream-start", "id": msg_id, "expr": expr}) + session.start_stream_reader() + return jsonify({"status": "started", "id": msg_id}) + except Exception as e: + return jsonify({"type": "error", "error": str(e)}), 500 - def generate(): - with session.lock: - try: - session.ensure_initialized() - session.send_message({"type": "stream-start", "id": msg_id, "expr": expr}) - - while True: - resp = session.read_line() - if resp is None: - yield f"event: error\ndata: {json.dumps({'error': 'Worker process died'})}\n\n" - break - resp_type = resp.get("type") - - if resp_type == "stream-data": - yield f"event: data\ndata: {json.dumps({'done': False, 'result': json.loads(resp.get('value', '{}'))})}\n\n" - elif resp_type == "stream-done": - yield f"event: done\ndata: {{}}\n\n" - break - elif resp_type == "stdout": - yield f"event: stdout\ndata: {json.dumps(resp.get('value', ''))}\n\n" - elif resp_type == "stderr": - yield f"event: stderr\ndata: {json.dumps(resp.get('value', ''))}\n\n" - elif resp_type == "error": - yield f"event: error\ndata: {json.dumps({'error': resp.get('error', ''), 'traceback': resp.get('traceback', '')})}\n\n" - break - - except Exception as e: - yield f"event: error\ndata: {json.dumps({'error': str(e)})}\n\n" - - return Response( - generate(), - mimetype="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "X-Accel-Buffering": "no", - }, - ) + @app.route("/api/stream/poll", methods=["POST"]) + def api_stream_poll(): + """Poll for stream messages — returns all queued messages since last poll.""" + session_id = _get_session_id() + if not session_id: + return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 + with _sessions_lock: + session = _sessions.get(session_id) + if not session: + return jsonify({"messages": [], "done": True}) + messages = session.drain_stream_queue() + done = any(m.get("type") in ("stream-done", "error") for m in messages) + if done: + # Send noop to unblock the worker's stdin reader thread + # so the main thread can resume processing exec/eval + session.flush_worker_reader() + return jsonify({"messages": messages, "done": done}) @app.route("/api/stream/exec", methods=["POST"]) def api_stream_exec(): """Queue code to execute during an active stream.""" session_id = _get_session_id() + if not session_id: + return jsonify({"error": "Missing X-Session-ID header"}), 400 data = request.get_json(force=True) code = data.get("code", "") - session = get_or_create_session(session_id) + with _sessions_lock: + session = _sessions.get(session_id) + if not session: + return jsonify({"error": "No active session"}), 404 try: session.send_message({"type": "stream-exec", "code": code}) return jsonify({"status": "queued"}) @@ -343,8 +414,13 @@ def api_stream_exec(): def api_stream_stop(): """Stop an active streaming session.""" session_id = _get_session_id() + if not session_id: + return jsonify({"error": "Missing X-Session-ID header"}), 400 - session = get_or_create_session(session_id) + with _sessions_lock: + session = _sessions.get(session_id) + if not session: + return jsonify({"status": "stopped"}) try: session.send_message({"type": "stream-stop"}) return jsonify({"status": "stopped"}) @@ -355,6 +431,8 @@ def api_stream_stop(): def api_session_delete(): """Kill a session's worker subprocess.""" session_id = _get_session_id() + if not session_id: + return jsonify({"error": "Missing X-Session-ID header"}), 400 remove_session(session_id) return jsonify({"status": "terminated"}) diff --git a/pathview_server/worker.py b/pathview_server/worker.py index 9fc3db0..558a270 100644 --- a/pathview_server/worker.py +++ b/pathview_server/worker.py @@ -8,13 +8,13 @@ Threading model: - Main thread: reads stdin, processes init/exec/eval synchronously -- During streaming: a reader thread handles stream-stop and stream-exec - while the main thread runs the streaming loop +- During streaming: a reader thread handles stream-stop and stream-exec, + puts non-streaming messages into a leftover queue +- After streaming: main thread drains leftovers before resuming stdin reads - Stdout lock: thread-safe writing to stdout (protocol messages only) """ import sys -import io import json import subprocess import threading @@ -24,6 +24,9 @@ # Lock for thread-safe stdout writing (protocol messages only) _stdout_lock = threading.Lock() +# Keep a reference to the real stdout pipe — protocol messages go here. +_real_stdout = sys.stdout + # Worker state _namespace = {} _initialized = False @@ -31,46 +34,56 @@ # Streaming state _streaming_active = False _streaming_code_queue = queue.Queue() +_leftover_queue: queue.Queue[dict | None] = queue.Queue() def send(response: dict) -> None: """Send a JSON response to the parent process via stdout.""" with _stdout_lock: - sys.stdout.write(json.dumps(response) + "\n") - sys.stdout.flush() - - -def _capture_output(func): - """Run func with stdout/stderr captured, sending output as messages.""" - old_stdout = sys.stdout - old_stderr = sys.stderr - captured_out = io.StringIO() - captured_err = io.StringIO() - sys.stdout = captured_out - sys.stderr = captured_err - try: - result = func() - finally: - sys.stdout = old_stdout - sys.stderr = old_stderr - out = captured_out.getvalue() - err = captured_err.getvalue() - if out: - send({"type": "stdout", "value": out}) - if err: - send({"type": "stderr", "value": err}) - return result + _real_stdout.write(json.dumps(response) + "\n") + _real_stdout.flush() + + +class _ProtocolWriter: + """File-like object that routes writes through the worker protocol. + + Installed as sys.stdout/sys.stderr permanently so that ALL output + (print, logging handlers, third-party libraries) is captured and + forwarded to the frontend console. This avoids the stale-reference + bug where StreamHandler(sys.stdout) captures a temporary StringIO. + """ + + def __init__(self, msg_type: str): + self.msg_type = msg_type + self._in_write = False + + def write(self, text: str) -> int: + if text and not self._in_write: + self._in_write = True + try: + send({"type": self.msg_type, "value": text}) + except Exception: + pass + finally: + self._in_write = False + return len(text) if text else 0 + + def flush(self) -> None: + pass + + def isatty(self) -> bool: + return False def read_message(): """Read one JSON message from stdin. Returns None on EOF.""" - line = sys.stdin.readline() - if not line: - return None - line = line.strip() - if not line: - return read_message() # skip blank lines - return json.loads(line) + while True: + line = sys.stdin.readline() + if not line: + return None + line = line.strip() + if line: + return json.loads(line) def _install_package(pip_spec: str, pre: bool = False) -> None: @@ -118,6 +131,12 @@ def initialize(packages: list[dict] | None = None) -> None: send({"type": "progress", "value": "Initializing Python worker..."}) + # Replace sys.stdout/stderr with protocol writers BEFORE importing packages. + # Any StreamHandler created later (e.g. by pathsim's LoggerManager singleton) + # will capture these persistent objects, so logging always routes through send(). + sys.stdout = _ProtocolWriter("stdout") + sys.stderr = _ProtocolWriter("stderr") + # Set up the namespace with common imports _namespace = {"__builtins__": __builtins__} exec("import numpy as np", _namespace) @@ -148,9 +167,7 @@ def exec_code(msg_id: str, code: str) -> None: return try: - def run(): - exec(code, _namespace) - _capture_output(run) + exec(code, _namespace) send({"type": "ok", "id": msg_id}) except Exception as e: tb = traceback.format_exc() @@ -164,13 +181,10 @@ def eval_expr(msg_id: str, expr: str) -> None: return try: - def run(): - # Mirror worker.ts: store result, then JSON-serialize - exec_code_str = f"_eval_result = {expr}" - exec(exec_code_str, _namespace) - to_json = _namespace.get("_to_json", str) - return json.dumps(_namespace["_eval_result"], default=to_json) - result = _capture_output(run) + exec_code_str = f"_eval_result = {expr}" + exec(exec_code_str, _namespace) + to_json = _namespace.get("_to_json", str) + result = json.dumps(_namespace["_eval_result"], default=to_json) send({"type": "value", "id": msg_id, "value": result}) except Exception as e: tb = traceback.format_exc() @@ -178,13 +192,19 @@ def run(): def _streaming_reader_thread(stop_event: threading.Event) -> None: - """Read stdin during streaming, handling stream-stop and stream-exec.""" + """Read stdin during streaming, handling stream-stop and stream-exec. + + Non-streaming messages are saved to _leftover_queue for the main loop. + After stop_event is set, the thread will exit once it reads one more line + (the flush message sent by the server). + """ global _streaming_active - while not stop_event.is_set(): + while True: line = sys.stdin.readline() if not line: - # EOF — stop streaming and signal main loop to exit + # EOF — stop streaming _streaming_active = False + _leftover_queue.put(None) break line = line.strip() if not line: @@ -195,6 +215,11 @@ def _streaming_reader_thread(stop_event: threading.Event) -> None: send({"type": "error", "error": f"Invalid JSON: {line}"}) continue + # If stop_event is set, we're just flushing — put message in leftovers + if stop_event.is_set(): + _leftover_queue.put(msg) + break + msg_type = msg.get("type") if msg_type == "stream-stop": _streaming_active = False @@ -202,7 +227,9 @@ def _streaming_reader_thread(stop_event: threading.Event) -> None: code = msg.get("code") if code and _streaming_active: _streaming_code_queue.put(code) - # Other messages during streaming are ignored (shouldn't happen) + else: + # Non-streaming message arrived — save for main loop + _leftover_queue.put(msg) def run_streaming_loop(msg_id: str, expr: str) -> None: @@ -236,35 +263,30 @@ def run_streaming_loop(msg_id: str, expr: str) -> None: except queue.Empty: break try: - def run_queued(c=code): - exec(c, _namespace) - _capture_output(run_queued) + exec(code, _namespace) except Exception as e: send({"type": "stderr", "value": f"Stream exec error: {e}"}) # Step the generator - def run_step(): - exec_code_str = f"_eval_result = {expr}" - exec(exec_code_str, _namespace) - to_json = _namespace.get("_to_json", str) - return json.dumps(_namespace["_eval_result"], default=to_json) - result = _capture_output(run_step) - - # Parse result - parsed = json.loads(result) + exec_code_str = f"_eval_result = {expr}" + exec(exec_code_str, _namespace) + raw_result = _namespace["_eval_result"] + done = raw_result.get("done", False) if isinstance(raw_result, dict) else False # Check if stopped during Python execution - still send final data if not _streaming_active: - if not parsed.get("done") and parsed.get("result"): - send({"type": "stream-data", "id": msg_id, "value": result}) + if not done and (isinstance(raw_result, dict) and raw_result.get("result")): + to_json = _namespace.get("_to_json", str) + send({"type": "stream-data", "id": msg_id, "value": json.dumps(raw_result, default=to_json)}) break # Check if simulation completed - if parsed.get("done"): + if done: break # Send result and continue - send({"type": "stream-data", "id": msg_id, "value": result}) + to_json = _namespace.get("_to_json", str) + send({"type": "stream-data", "id": msg_id, "value": json.dumps(raw_result, default=to_json)}) except Exception as e: tb = traceback.format_exc() @@ -274,18 +296,25 @@ def run_step(): stop_event.set() # Always send done when loop ends send({"type": "stream-done", "id": msg_id}) - - -def stop_streaming() -> None: - """Stop the streaming loop.""" - global _streaming_active - _streaming_active = False + # Reader thread will exit on the next stdin read (flush message from server) def main() -> None: """Main loop: read messages from stdin and process them.""" while True: - msg = read_message() + # First drain any leftover messages from the streaming reader thread + msg = None + while not _leftover_queue.empty(): + try: + msg = _leftover_queue.get_nowait() + if msg is not None: + break + except queue.Empty: + break + + # If no leftover, read from stdin directly + if msg is None: + msg = read_message() if msg is None: # stdin closed, exit break @@ -317,11 +346,16 @@ def main() -> None: run_streaming_loop(msg_id, expr) elif msg_type == "stream-stop": - stop_streaming() + # Only during streaming (handled by reader thread) + pass elif msg_type == "stream-exec": - if isinstance(code, str) and _streaming_active: - _streaming_code_queue.put(code) + # Only during streaming (handled by reader thread) + pass + + elif msg_type == "noop": + # Flush message from server — ignore + pass else: raise ValueError(f"Unknown message type: {msg_type}") diff --git a/pyproject.toml b/pyproject.toml index 167b30e..1e0518b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "0.5.0" description = "Visual node editor for building and simulating dynamic systems with PathSim" readme = "README.md" license = {text = "MIT"} -requires-python = ">=3.9" +requires-python = ">=3.10" classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Science/Research", @@ -18,6 +18,7 @@ classifiers = [ dependencies = [ "flask>=3.0", "flask-cors>=4.0", + "numpy", ] [project.urls] diff --git a/scripts/build_package.py b/scripts/build_package.py index 087c493..72841df 100644 --- a/scripts/build_package.py +++ b/scripts/build_package.py @@ -20,7 +20,9 @@ def run(cmd, **kwargs): print(f" > {' '.join(cmd)}") - result = subprocess.run(cmd, cwd=kwargs.pop("cwd", REPO_ROOT), **kwargs) + # shell=True needed on Windows for npx/npm resolution + result = subprocess.run(cmd, cwd=kwargs.pop("cwd", REPO_ROOT), + shell=(sys.platform == "win32"), **kwargs) if result.returncode != 0: print(f"ERROR: command failed (exit {result.returncode})") sys.exit(result.returncode) diff --git a/src/lib/pyodide/backend/flask/backend.ts b/src/lib/pyodide/backend/flask/backend.ts index 2b381d4..8d7f695 100644 --- a/src/lib/pyodide/backend/flask/backend.ts +++ b/src/lib/pyodide/backend/flask/backend.ts @@ -1,6 +1,10 @@ /** * Flask Backend - * Implements the Backend interface using a Flask server with subprocess workers + * Implements the Backend interface using a Flask server with subprocess workers. + * + * Mirrors the Pyodide worker's message-passing pattern: + * - exec/eval use simple request/response + * - streaming uses start + poll (like postMessage with stream-data/stream-done) */ import type { Backend, BackendState } from '../types'; @@ -9,21 +13,18 @@ import { TIMEOUTS } from '$lib/constants/python'; import { STATUS_MESSAGES } from '$lib/constants/messages'; import { PYTHON_PACKAGES } from '$lib/constants/dependencies'; -/** - * Flask Backend Implementation - * - * Communicates with a Flask server that manages Python subprocess workers. - * Each browser session gets its own isolated Python process on the server. - * Supports streaming via Server-Sent Events (SSE). - */ +/** Polling interval for stream results (ms) */ +const STREAM_POLL_INTERVAL = 30; + export class FlaskBackend implements Backend { private host: string; private sessionId: string; private messageId = 0; private _isStreaming = false; - private streamAbortController: AbortController | null = null; + private streamPollTimer: ReturnType | null = null; + private serverInitPromise: Promise | null = null; - // Stream state + // Stream callbacks — same shape as PyodideBackend's streamState private streamState: { onData: ((data: unknown) => void) | null; onDone: (() => void) | null; @@ -35,8 +36,7 @@ export class FlaskBackend implements Backend { private stderrCallback: ((value: string) => void) | null = null; constructor(host: string) { - this.host = host.replace(/\/$/, ''); // strip trailing slash - // Get or create session ID from sessionStorage + this.host = host.replace(/\/$/, ''); const stored = typeof sessionStorage !== 'undefined' ? sessionStorage.getItem('flask-session-id') : null; if (stored) { this.sessionId = stored; @@ -64,25 +64,13 @@ export class FlaskBackend implements Backend { })); try { - // Health check - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), TIMEOUTS.INIT); - const resp = await fetch(`${this.host}/api/health`, { - signal: controller.signal + signal: AbortSignal.timeout(TIMEOUTS.INIT) }); - clearTimeout(timeout); - - if (!resp.ok) { - throw new Error(`Server health check failed: ${resp.status}`); - } + if (!resp.ok) throw new Error(`Server health check failed: ${resp.status}`); - // Initialize worker with packages from the shared config (single source of truth) backendState.update((s) => ({ ...s, progress: 'Initializing Python worker...' })); - const initController = new AbortController(); - const initTimeout = setTimeout(() => initController.abort(), TIMEOUTS.INIT); - const initResp = await fetch(`${this.host}/api/init`, { method: 'POST', headers: { @@ -90,17 +78,12 @@ export class FlaskBackend implements Backend { 'X-Session-ID': this.sessionId }, body: JSON.stringify({ packages: PYTHON_PACKAGES }), - signal: initController.signal + signal: AbortSignal.timeout(TIMEOUTS.INIT) }); - clearTimeout(initTimeout); - const initData = await initResp.json(); - if (initData.type === 'error') { - throw new Error(initData.error); - } + if (initData.type === 'error') throw new Error(initData.error); - // Forward any stdout/stderr messages from init if (initData.messages) { for (const msg of initData.messages) { if (msg.type === 'stdout' && this.stdoutCallback) this.stdoutCallback(msg.value); @@ -111,6 +94,8 @@ export class FlaskBackend implements Backend { } } + this.serverInitPromise = Promise.resolve(); + backendState.update((s) => ({ ...s, initialized: true, @@ -119,33 +104,22 @@ export class FlaskBackend implements Backend { })); } catch (error) { const msg = error instanceof Error ? error.message : String(error); - backendState.update((s) => ({ - ...s, - loading: false, - error: `Flask backend error: ${msg}` - })); + backendState.update((s) => ({ ...s, loading: false, error: `Flask backend error: ${msg}` })); throw error; } } terminate(): void { - // Abort any active stream - if (this.streamAbortController) { - this.streamAbortController.abort(); - this.streamAbortController = null; - } - - // Clear stream state + this.stopStreaming(); this._isStreaming = false; this.streamState = { onData: null, onDone: null, onError: null }; - // Kill server-side session (fire and forget) fetch(`${this.host}/api/session`, { method: 'DELETE', headers: { 'X-Session-ID': this.sessionId } }).catch(() => {}); - // Reset state + this.serverInitPromise = null; backendState.reset(); } @@ -174,89 +148,105 @@ export class FlaskBackend implements Backend { } // ------------------------------------------------------------------------- - // Execution + // Lazy server init // ------------------------------------------------------------------------- - async exec(code: string, timeout: number = TIMEOUTS.SIMULATION): Promise { - const id = this.generateId(); - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout); + private ensureServerInit(): Promise { + if (this.serverInitPromise) return this.serverInitPromise; - try { - const resp = await fetch(`${this.host}/api/exec`, { + this.serverInitPromise = (async () => { + const resp = await fetch(`${this.host}/api/init`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Session-ID': this.sessionId }, - body: JSON.stringify({ id, code }), - signal: controller.signal + body: JSON.stringify({ packages: PYTHON_PACKAGES }), + signal: AbortSignal.timeout(TIMEOUTS.INIT) }); - const data = await resp.json(); + if (data.type === 'error') throw new Error(data.error); + if (data.messages) { + for (const msg of data.messages) { + if (msg.type === 'stdout' && this.stdoutCallback) this.stdoutCallback(msg.value); + if (msg.type === 'stderr' && this.stderrCallback) this.stderrCallback(msg.value); + } + } + })(); - // Forward stdout/stderr from response - if (data.stdout && this.stdoutCallback) this.stdoutCallback(data.stdout); - if (data.stderr && this.stderrCallback) this.stderrCallback(data.stderr); + // Clear on failure so subsequent calls retry instead of returning the rejected promise + this.serverInitPromise.catch(() => { + this.serverInitPromise = null; + }); - if (data.type === 'error') { - const errorMsg = data.traceback ? `${data.error}\n${data.traceback}` : data.error; - throw new Error(errorMsg); - } - } catch (error) { - if (error instanceof DOMException && error.name === 'AbortError') { - throw new Error('Execution timeout'); - } - throw error; - } finally { - clearTimeout(timeoutId); - } + return this.serverInitPromise; } - async evaluate(expr: string, timeout: number = TIMEOUTS.SIMULATION): Promise { + // ------------------------------------------------------------------------- + // Execution + // ------------------------------------------------------------------------- + + async exec(code: string, timeout: number = TIMEOUTS.SIMULATION): Promise { + await this.ensureServerInit(); const id = this.generateId(); - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout); - try { - const resp = await fetch(`${this.host}/api/eval`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Session-ID': this.sessionId - }, - body: JSON.stringify({ id, expr }), - signal: controller.signal - }); + const resp = await fetch(`${this.host}/api/exec`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ id, code }), + signal: AbortSignal.timeout(timeout) + }); - const data = await resp.json(); + if (!resp.ok && resp.status >= 500) { + throw new Error(`Server error: ${resp.status}`); + } - // Forward stdout/stderr from response - if (data.stdout && this.stdoutCallback) this.stdoutCallback(data.stdout); - if (data.stderr && this.stderrCallback) this.stderrCallback(data.stderr); + const data = await resp.json(); + if (data.stdout && this.stdoutCallback) this.stdoutCallback(data.stdout); + if (data.stderr && this.stderrCallback) this.stderrCallback(data.stderr); - if (data.type === 'error') { - const errorMsg = data.traceback ? `${data.error}\n${data.traceback}` : data.error; - throw new Error(errorMsg); - } + if (data.type === 'error') { + const errorMsg = data.traceback ? `${data.error}\n${data.traceback}` : data.error; + throw new Error(errorMsg); + } + } - if (data.value === undefined) { - throw new Error('No value returned from eval'); - } + async evaluate(expr: string, timeout: number = TIMEOUTS.SIMULATION): Promise { + await this.ensureServerInit(); + const id = this.generateId(); - return JSON.parse(data.value) as T; - } catch (error) { - if (error instanceof DOMException && error.name === 'AbortError') { - throw new Error('Evaluation timeout'); - } - throw error; - } finally { - clearTimeout(timeoutId); + const resp = await fetch(`${this.host}/api/eval`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ id, expr }), + signal: AbortSignal.timeout(timeout) + }); + + if (!resp.ok && resp.status >= 500) { + throw new Error(`Server error: ${resp.status}`); } + + const data = await resp.json(); + if (data.stdout && this.stdoutCallback) this.stdoutCallback(data.stdout); + if (data.stderr && this.stderrCallback) this.stderrCallback(data.stderr); + + if (data.type === 'error') { + const errorMsg = data.traceback ? `${data.error}\n${data.traceback}` : data.error; + throw new Error(errorMsg); + } + + if (data.value === undefined) throw new Error('No value returned from eval'); + return JSON.parse(data.value) as T; } // ------------------------------------------------------------------------- - // Streaming + // Streaming — mirrors Pyodide worker's postMessage pattern via polling // ------------------------------------------------------------------------- startStreaming( @@ -265,12 +255,6 @@ export class FlaskBackend implements Backend { onDone: () => void, onError: (error: Error) => void ): void { - if (!this.isReady()) { - onError(new Error('Backend not initialized')); - return; - } - - // Stop any existing stream if (this._isStreaming) { this.stopStreaming(); } @@ -283,29 +267,59 @@ export class FlaskBackend implements Backend { onError }; - this.streamAbortController = new AbortController(); - - // Start SSE stream - this.consumeSSEStream(id, expr, this.streamAbortController.signal); + // Start stream on server, then poll for results + this.ensureServerInit() + .then(() => + fetch(`${this.host}/api/stream/start`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ id, expr }) + }) + ) + .then((resp) => resp.json()) + .then((data) => { + if (data.type === 'error') { + throw new Error(data.error); + } + // Start polling loop — same as Pyodide worker's onmessage dispatching + this.pollStreamResults(); + }) + .catch((error) => { + this._isStreaming = false; + onError(error instanceof Error ? error : new Error(String(error))); + this.streamState = { onData: null, onDone: null, onError: null }; + }); } stopStreaming(): void { if (!this._isStreaming) return; - // Send stop to server + // Stop polling timer — the server will send stream-done which triggers onDone + if (this.streamPollTimer) { + clearTimeout(this.streamPollTimer); + this.streamPollTimer = null; + } + + // Tell server to stop, then do one final poll to get the stream-done message fetch(`${this.host}/api/stream/stop`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Session-ID': this.sessionId } - }).catch(() => {}); - - // Abort the SSE connection - if (this.streamAbortController) { - this.streamAbortController.abort(); - this.streamAbortController = null; - } + }) + .then(() => this.pollStreamResults()) + .catch(() => { + // If final poll fails, clean up locally + this._isStreaming = false; + if (this.streamState.onDone) { + this.streamState.onDone(); + } + this.streamState = { onData: null, onDone: null, onError: null }; + }); } isStreaming(): boolean { @@ -318,7 +332,6 @@ export class FlaskBackend implements Backend { return; } - // Fire and forget fetch(`${this.host}/api/stream/exec`, { method: 'POST', headers: { @@ -350,72 +363,37 @@ export class FlaskBackend implements Backend { } /** - * Consume an SSE stream from /api/stream, dispatching events to callbacks. + * Poll the server for stream messages and dispatch them to callbacks. + * This mirrors the Pyodide backend's handleResponse for stream-data/stream-done. */ - private async consumeSSEStream(id: string, expr: string, signal: AbortSignal): Promise { + private async pollStreamResults(): Promise { + if (!this._isStreaming) return; + try { - const resp = await fetch(`${this.host}/api/stream`, { + const resp = await fetch(`${this.host}/api/stream/poll`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Session-ID': this.sessionId - }, - body: JSON.stringify({ id, expr }), - signal + } }); - if (!resp.ok || !resp.body) { - throw new Error(`Stream request failed: ${resp.status}`); - } - - const reader = resp.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; - - while (true) { - const { value, done } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - - // Process complete SSE messages (separated by double newlines) - const parts = buffer.split('\n\n'); - buffer = parts.pop() || ''; // Keep incomplete last part - - for (const part of parts) { - if (!part.trim()) continue; - - // Parse SSE fields - let eventType = ''; - let eventData = ''; - - for (const line of part.split('\n')) { - if (line.startsWith('event: ')) { - eventType = line.slice(7); - } else if (line.startsWith('data: ')) { - eventData = line.slice(6); - } - } + const data = await resp.json(); - if (!eventType) continue; + if (!Array.isArray(data?.messages)) { + throw new Error(data?.error || 'Invalid poll response'); + } - this.handleSSEEvent(eventType, eventData); + for (const msg of data.messages) { + this.handleStreamMessage(msg); + if (!this._isStreaming) return; // done or error stopped streaming + } - if (eventType === 'done' || eventType === 'error') { - return; - } - } + // Schedule next poll if still streaming + if (this._isStreaming) { + this.streamPollTimer = setTimeout(() => this.pollStreamResults(), STREAM_POLL_INTERVAL); } } catch (error) { - if (signal.aborted) { - // Aborted by stopStreaming — call onDone - this._isStreaming = false; - if (this.streamState.onDone) { - this.streamState.onDone(); - } - this.streamState = { onData: null, onDone: null, onError: null }; - return; - } this._isStreaming = false; if (this.streamState.onError) { this.streamState.onError(error instanceof Error ? error : new Error(String(error))); @@ -424,62 +402,52 @@ export class FlaskBackend implements Backend { } } - private handleSSEEvent(eventType: string, data: string): void { - switch (eventType) { - case 'data': { - if (this.streamState.onData) { + /** + * Handle a single message from the worker — same dispatch as PyodideBackend.handleResponse + */ + private handleStreamMessage(msg: Record): void { + const type = msg.type as string; + + switch (type) { + case 'stream-data': { + if (this.streamState.onData && msg.value) { try { - const parsed = JSON.parse(data); - this.streamState.onData(parsed); + this.streamState.onData(JSON.parse(msg.value as string)); } catch { // Ignore parse errors } } break; } - case 'stdout': { - if (this.stdoutCallback) { - try { - this.stdoutCallback(JSON.parse(data)); - } catch { - this.stdoutCallback(data); - } + case 'stream-done': { + this._isStreaming = false; + if (this.streamState.onDone) { + this.streamState.onDone(); } + this.streamState = { onData: null, onDone: null, onError: null }; break; } - case 'stderr': { - if (this.stderrCallback) { - try { - this.stderrCallback(JSON.parse(data)); - } catch { - this.stderrCallback(data); - } + case 'stdout': { + if (this.stdoutCallback && msg.value) { + this.stdoutCallback(msg.value as string); } break; } - case 'done': { - this._isStreaming = false; - if (this.streamState.onDone) { - this.streamState.onDone(); + case 'stderr': { + if (this.stderrCallback && msg.value) { + this.stderrCallback(msg.value as string); } - this.streamState = { onData: null, onDone: null, onError: null }; break; } case 'error': { this._isStreaming = false; if (this.streamState.onError) { - try { - const parsed = JSON.parse(data); - const msg = parsed.traceback - ? `${parsed.error}\n${parsed.traceback}` - : parsed.error || 'Unknown error'; - this.streamState.onError(new Error(msg)); - } catch { - this.streamState.onError(new Error(data || 'Stream error')); - } + const errorMsg = msg.traceback + ? `${msg.error}\n${msg.traceback}` + : (msg.error as string) || 'Unknown error'; + this.streamState.onError(new Error(errorMsg)); } this.streamState = { onData: null, onDone: null, onError: null }; - backendState.update((s) => ({ ...s, error: 'Stream error' })); break; } } diff --git a/src/lib/pyodide/backend/index.ts b/src/lib/pyodide/backend/index.ts index 86c6aae..ef04b5c 100644 --- a/src/lib/pyodide/backend/index.ts +++ b/src/lib/pyodide/backend/index.ts @@ -33,7 +33,7 @@ export { FlaskBackend } from './flask/backend'; // These delegate to the current backend and maintain API compatibility // ============================================================================ -import { getBackend, switchBackend, setFlaskHost } from './registry'; +import { getBackend, switchBackend, setFlaskHost, getBackendType } from './registry'; import { backendState } from './state'; import { consoleStore } from '$lib/stores/console'; @@ -42,7 +42,7 @@ import { consoleStore } from '$lib/stores/console'; * Reads `?backend=flask` and `?host=...` from the current URL. * Call this early in page mount, before any backend usage. */ -export function initBackendFromUrl(): void { +export async function initBackendFromUrl(): Promise { if (typeof window === 'undefined') return; const params = new URLSearchParams(window.location.search); const backendParam = params.get('backend'); @@ -53,6 +53,7 @@ export function initBackendFromUrl(): void { setFlaskHost(hostParam); } switchBackend('flask'); + await init(); } } @@ -78,6 +79,8 @@ export async function autoDetectBackend(): Promise { if (data.status === 'ok') { setFlaskHost(window.location.origin); switchBackend('flask'); + // Run full init — sets up callbacks, logs progress, initializes worker + await init(); } } } catch { From 0cb9ecbefec3bba0a2ae3567921b0789b49a6f94 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:25:20 +0100 Subject: [PATCH 12/27] Add exec/eval timeout watchdog (30s worker, 35s server) --- pathview_server/app.py | 43 ++++++++++++++++++++--- pathview_server/worker.py | 72 ++++++++++++++++++++++++++++++++++----- 2 files changed, 102 insertions(+), 13 deletions(-) diff --git a/pathview_server/app.py b/pathview_server/app.py index a909e4e..d4397d8 100644 --- a/pathview_server/app.py +++ b/pathview_server/app.py @@ -27,6 +27,7 @@ SESSION_TTL = 3600 # 1 hour of inactivity before cleanup CLEANUP_INTERVAL = 60 # Check for stale sessions every 60 seconds +EXEC_TIMEOUT = 35 # Server-side timeout for exec/eval (slightly > worker's 30s) WORKER_SCRIPT = str(Path(__file__).parent / "worker.py") # --------------------------------------------------------------------------- @@ -68,6 +69,32 @@ def read_line(self) -> dict | None: return None return json.loads(line.strip()) + def read_line_timeout(self, timeout: float = EXEC_TIMEOUT) -> dict | None: + """Read one JSON line with a timeout. Returns None on EOF or timeout. + + Raises TimeoutError if no response within the timeout period. + """ + result = [None] + error = [None] + + def reader(): + try: + result[0] = self.read_line() + except Exception as e: + error[0] = e + + t = threading.Thread(target=reader, daemon=True) + t.start() + t.join(timeout) + + if t.is_alive(): + raise TimeoutError(f"Worker unresponsive after {timeout}s") + + if error[0]: + raise error[0] + + return result[0] + def ensure_initialized(self, packages: list[dict] | None = None) -> list[dict]: """Initialize the worker if not already done. Returns any messages received.""" if self._initialized: @@ -277,9 +304,10 @@ def api_exec(): stdout_lines = [] stderr_lines = [] while True: - resp = session.read_line() + resp = session.read_line_timeout() if resp is None: - return jsonify({"type": "error", "id": msg_id, "error": "Worker process died"}), 500 + remove_session(session_id) + return jsonify({"type": "error", "errorType": "worker-crashed", "id": msg_id, "error": "Worker process died"}), 500 resp_type = resp.get("type") if resp_type == "stdout": stdout_lines.append(resp.get("value", "")) @@ -300,6 +328,9 @@ def api_exec(): result["stderr"] = "".join(stderr_lines) return jsonify(result), 400 + except TimeoutError: + remove_session(session_id) + return jsonify({"type": "error", "errorType": "timeout", "id": msg_id, "error": "Execution timed out"}), 504 except Exception as e: return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 @@ -322,9 +353,10 @@ def api_eval(): stdout_lines = [] stderr_lines = [] while True: - resp = session.read_line() + resp = session.read_line_timeout() if resp is None: - return jsonify({"type": "error", "id": msg_id, "error": "Worker process died"}), 500 + remove_session(session_id) + return jsonify({"type": "error", "errorType": "worker-crashed", "id": msg_id, "error": "Worker process died"}), 500 resp_type = resp.get("type") if resp_type == "stdout": stdout_lines.append(resp.get("value", "")) @@ -345,6 +377,9 @@ def api_eval(): result["stderr"] = "".join(stderr_lines) return jsonify(result), 400 + except TimeoutError: + remove_session(session_id) + return jsonify({"type": "error", "errorType": "timeout", "id": msg_id, "error": "Execution timed out"}), 504 except Exception as e: return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 diff --git a/pathview_server/worker.py b/pathview_server/worker.py index 558a270..c31322c 100644 --- a/pathview_server/worker.py +++ b/pathview_server/worker.py @@ -20,6 +20,7 @@ import threading import traceback import queue +import ctypes # Lock for thread-safe stdout writing (protocol messages only) _stdout_lock = threading.Lock() @@ -31,6 +32,9 @@ _namespace = {} _initialized = False +# Default timeout for exec/eval (seconds) +EXEC_TIMEOUT = 30 + # Streaming state _streaming_active = False _streaming_code_queue = queue.Queue() @@ -160,6 +164,51 @@ def initialize(packages: list[dict] | None = None) -> None: send({"type": "ready"}) +def _raise_in_thread(thread_id: int, exc_type: type) -> None: + """Raise an exception in the given thread (best-effort interrupt).""" + ctypes.pythonapi.PyThreadState_SetAsyncExc( + ctypes.c_ulong(thread_id), ctypes.py_object(exc_type) + ) + + +def _run_with_timeout(func, timeout: float = EXEC_TIMEOUT): + """Run func() in a daemon thread with a timeout. + + Returns (result, error_string, traceback_string). + If timeout fires, raises TimeoutError. + """ + result_holder = [None, None, None] # result, error, traceback + + def target(): + try: + result_holder[0] = func() + except Exception as e: + result_holder[1] = str(e) + result_holder[2] = traceback.format_exc() + + t = threading.Thread(target=target, daemon=True) + t.start() + t.join(timeout) + + if t.is_alive(): + # Try to interrupt the stuck thread + _raise_in_thread(t.ident, KeyboardInterrupt) + t.join(2) # Give it 2s to handle the interrupt + raise TimeoutError(f"Execution timed out after {timeout}s") + + if result_holder[1] is not None: + raise _ExecError(result_holder[1], result_holder[2]) + + return result_holder[0] + + +class _ExecError(Exception): + """Wraps an error from user code execution with its traceback.""" + def __init__(self, message: str, tb: str | None = None): + super().__init__(message) + self.tb = tb + + def exec_code(msg_id: str, code: str) -> None: """Execute Python code (no return value).""" if not _initialized: @@ -167,11 +216,12 @@ def exec_code(msg_id: str, code: str) -> None: return try: - exec(code, _namespace) + _run_with_timeout(lambda: exec(code, _namespace)) send({"type": "ok", "id": msg_id}) - except Exception as e: - tb = traceback.format_exc() - send({"type": "error", "id": msg_id, "error": str(e), "traceback": tb}) + except TimeoutError as e: + send({"type": "error", "id": msg_id, "error": str(e)}) + except _ExecError as e: + send({"type": "error", "id": msg_id, "error": str(e), "traceback": e.tb}) def eval_expr(msg_id: str, expr: str) -> None: @@ -181,14 +231,18 @@ def eval_expr(msg_id: str, expr: str) -> None: return try: - exec_code_str = f"_eval_result = {expr}" - exec(exec_code_str, _namespace) + def do_eval(): + exec_code_str = f"_eval_result = {expr}" + exec(exec_code_str, _namespace) + + _run_with_timeout(do_eval) to_json = _namespace.get("_to_json", str) result = json.dumps(_namespace["_eval_result"], default=to_json) send({"type": "value", "id": msg_id, "value": result}) - except Exception as e: - tb = traceback.format_exc() - send({"type": "error", "id": msg_id, "error": str(e), "traceback": tb}) + except TimeoutError as e: + send({"type": "error", "id": msg_id, "error": str(e)}) + except _ExecError as e: + send({"type": "error", "id": msg_id, "error": str(e), "traceback": e.tb}) def _streaming_reader_thread(stop_event: threading.Event) -> None: From 4938d118f91b660748d70c0292210c39406d1e15 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:25:38 +0100 Subject: [PATCH 13/27] Use waitress as production WSGI server, keep Flask dev server for --debug --- pathview_server/cli.py | 6 +++++- pyproject.toml | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pathview_server/cli.py b/pathview_server/cli.py index d45f35c..39e6171 100644 --- a/pathview_server/cli.py +++ b/pathview_server/cli.py @@ -45,7 +45,11 @@ def open_browser(): print("Press Ctrl+C to stop\n") try: - app.run(host=args.host, port=args.port, debug=args.debug, threaded=True) + if args.debug: + app.run(host=args.host, port=args.port, debug=True, threaded=True) + else: + from waitress import serve + serve(app, host=args.host, port=args.port, threads=4) except KeyboardInterrupt: print("\nStopping PathView server...") sys.exit(0) diff --git a/pyproject.toml b/pyproject.toml index 1e0518b..315966e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "flask>=3.0", "flask-cors>=4.0", "numpy", + "waitress>=3.0", ] [project.urls] From 4ee85e20200c0e67f311ac1282bb2799934f10cb Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:25:55 +0100 Subject: [PATCH 14/27] Replace sleep-based browser open with health check polling --- pathview_server/cli.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pathview_server/cli.py b/pathview_server/cli.py index 39e6171..63340ce 100644 --- a/pathview_server/cli.py +++ b/pathview_server/cli.py @@ -34,11 +34,19 @@ def main(): app = create_app(serve_static=True) if not args.no_browser: - def open_browser(): - time.sleep(1.5) - webbrowser.open(f"http://{args.host}:{args.port}") + def open_browser_when_ready(): + import urllib.request + health_url = f"http://{args.host}:{args.port}/api/health" + deadline = time.time() + 10 + while time.time() < deadline: + try: + urllib.request.urlopen(health_url, timeout=1) + webbrowser.open(f"http://{args.host}:{args.port}") + return + except Exception: + time.sleep(0.2) - threading.Thread(target=open_browser, daemon=True).start() + threading.Thread(target=open_browser_when_ready, daemon=True).start() print(f"PathView v{__version__}") print(f"Running at http://{args.host}:{args.port}") From 4d18231a992200977141acf8c79774e41d7727b1 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:26:33 +0100 Subject: [PATCH 15/27] Add worker crash recovery with auto-reinit on next request --- src/lib/pyodide/backend/flask/backend.ts | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/lib/pyodide/backend/flask/backend.ts b/src/lib/pyodide/backend/flask/backend.ts index 8d7f695..dcdb37e 100644 --- a/src/lib/pyodide/backend/flask/backend.ts +++ b/src/lib/pyodide/backend/flask/backend.ts @@ -200,15 +200,12 @@ export class FlaskBackend implements Backend { signal: AbortSignal.timeout(timeout) }); - if (!resp.ok && resp.status >= 500) { - throw new Error(`Server error: ${resp.status}`); - } - const data = await resp.json(); if (data.stdout && this.stdoutCallback) this.stdoutCallback(data.stdout); if (data.stderr && this.stderrCallback) this.stderrCallback(data.stderr); if (data.type === 'error') { + this.handleWorkerError(data); const errorMsg = data.traceback ? `${data.error}\n${data.traceback}` : data.error; throw new Error(errorMsg); } @@ -228,15 +225,12 @@ export class FlaskBackend implements Backend { signal: AbortSignal.timeout(timeout) }); - if (!resp.ok && resp.status >= 500) { - throw new Error(`Server error: ${resp.status}`); - } - const data = await resp.json(); if (data.stdout && this.stdoutCallback) this.stdoutCallback(data.stdout); if (data.stderr && this.stderrCallback) this.stderrCallback(data.stderr); if (data.type === 'error') { + this.handleWorkerError(data); const errorMsg = data.traceback ? `${data.error}\n${data.traceback}` : data.error; throw new Error(errorMsg); } @@ -362,6 +356,20 @@ export class FlaskBackend implements Backend { return `repl_${++this.messageId}`; } + /** + * Check if a response indicates the worker crashed or timed out. + * If so, clear serverInitPromise so the next request triggers re-init. + */ + private handleWorkerError(data: Record): void { + const errorType = data.errorType as string | undefined; + if (errorType === 'worker-crashed' || errorType === 'timeout') { + this.serverInitPromise = null; + if (this.stderrCallback) { + this.stderrCallback('Python worker crashed, restarting on next request...\n'); + } + } + } + /** * Poll the server for stream messages and dispatch them to callbacks. * This mirrors the Pyodide backend's handleResponse for stream-data/stream-done. From 4bdaa0665adcf1d7ca4228c25486b1f7cd82571b Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:28:14 +0100 Subject: [PATCH 16/27] Add pytest test suite and CI workflow (22 tests) --- .github/workflows/test.yml | 31 +++++++ pyproject.toml | 5 ++ tests/__init__.py | 0 tests/conftest.py | 39 +++++++++ tests/test_app.py | 157 ++++++++++++++++++++++++++++++++++++ tests/test_worker.py | 160 +++++++++++++++++++++++++++++++++++++ 6 files changed, 392 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_app.py create mode 100644 tests/test_worker.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..517f313 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,31 @@ +name: Tests + +on: + push: + branches: [main, feature/flask-backend] + pull_request: + branches: [main, feature/flask-backend] + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[test]" + + - name: Run tests + run: pytest tests/ -v diff --git a/pyproject.toml b/pyproject.toml index 315966e..23493c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,11 @@ dependencies = [ "waitress>=3.0", ] +[project.optional-dependencies] +test = [ + "pytest>=8.0", +] + [project.urls] Homepage = "https://view.pathsim.org" Repository = "https://github.com/pathsim/pathview" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e5a0b75 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,39 @@ +"""Shared pytest fixtures for PathView server tests.""" + +import pytest + +from pathview_server.app import create_app, _sessions, _sessions_lock + + +@pytest.fixture() +def app(): + """Create a Flask test app (API-only, no static serving).""" + application = create_app(serve_static=False) + application.config["TESTING"] = True + yield application + # Clean up all sessions after each test + with _sessions_lock: + for session in _sessions.values(): + session.kill() + _sessions.clear() + + +@pytest.fixture() +def client(app): + """Flask test client.""" + return app.test_client() + + +@pytest.fixture() +def session_id(): + """A stable session ID for tests.""" + return "test-session-001" + + +@pytest.fixture() +def session_headers(session_id): + """Headers with session ID and content type.""" + return { + "X-Session-ID": session_id, + "Content-Type": "application/json", + } diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..354a6d5 --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,157 @@ +"""Integration tests for Flask routes.""" + +import json + + +def test_health(client): + resp = client.get("/api/health") + assert resp.status_code == 200 + assert resp.get_json()["status"] == "ok" + + +def test_init_missing_session_id(client): + resp = client.post("/api/init", json={}) + assert resp.status_code == 400 + assert "Missing X-Session-ID" in resp.get_json()["error"] + + +def test_init_creates_session(client, session_headers): + resp = client.post("/api/init", json={"packages": []}, headers=session_headers) + assert resp.status_code == 200 + data = resp.get_json() + assert data["type"] == "ready" + + +def test_exec_simple(client, session_headers): + # Init first + client.post("/api/init", json={"packages": []}, headers=session_headers) + + # Execute code + resp = client.post("/api/exec", json={"code": "x = 42"}, headers=session_headers) + assert resp.status_code == 200 + assert resp.get_json()["type"] == "ok" + + +def test_exec_with_print(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + resp = client.post( + "/api/exec", json={"code": "print('hello world')"}, headers=session_headers + ) + assert resp.status_code == 200 + data = resp.get_json() + assert data["type"] == "ok" + assert "hello world" in data.get("stdout", "") + + +def test_exec_error(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + resp = client.post( + "/api/exec", json={"code": "raise ValueError('test error')"}, headers=session_headers + ) + assert resp.status_code == 400 + data = resp.get_json() + assert data["type"] == "error" + assert "test error" in data["error"] + + +def test_eval_simple(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + # Set a variable, then eval it + client.post("/api/exec", json={"code": "y = 123"}, headers=session_headers) + + resp = client.post("/api/eval", json={"expr": "y"}, headers=session_headers) + assert resp.status_code == 200 + data = resp.get_json() + assert data["type"] == "value" + assert json.loads(data["value"]) == 123 + + +def test_eval_expression(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + resp = client.post("/api/eval", json={"expr": "2 + 3"}, headers=session_headers) + assert resp.status_code == 200 + data = resp.get_json() + assert json.loads(data["value"]) == 5 + + +def test_eval_error(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + resp = client.post( + "/api/eval", json={"expr": "undefined_var"}, headers=session_headers + ) + assert resp.status_code == 400 + data = resp.get_json() + assert data["type"] == "error" + + +def test_session_delete(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + resp = client.delete("/api/session", headers=session_headers) + assert resp.status_code == 200 + assert resp.get_json()["status"] == "terminated" + + +def test_session_persistence(client, session_headers): + """Variables set in one exec should be available in the next.""" + client.post("/api/init", json={"packages": []}, headers=session_headers) + + client.post("/api/exec", json={"code": "my_var = 'persistent'"}, headers=session_headers) + + resp = client.post("/api/eval", json={"expr": "my_var"}, headers=session_headers) + data = resp.get_json() + assert json.loads(data["value"]) == "persistent" + + +def test_streaming_lifecycle(client, session_headers): + """Test stream start → poll → done cycle.""" + client.post("/api/init", json={"packages": []}, headers=session_headers) + + # Set up a generator that yields one value then is done + client.post( + "/api/exec", + json={"code": "_step = 0\ndef _gen():\n global _step\n _step += 1\n return {'result': _step, 'done': _step >= 2}"}, + headers=session_headers, + ) + + # Start streaming + resp = client.post( + "/api/stream/start", + json={"expr": "_gen()"}, + headers=session_headers, + ) + assert resp.status_code == 200 + assert resp.get_json()["status"] == "started" + + # Poll until done + import time + done = False + messages = [] + deadline = time.time() + 10 + while not done and time.time() < deadline: + resp = client.post("/api/stream/poll", headers=session_headers) + data = resp.get_json() + messages.extend(data.get("messages", [])) + done = data.get("done", False) + if not done: + time.sleep(0.1) + + assert done + types = [m["type"] for m in messages] + assert "stream-data" in types + assert "stream-done" in types + + +def test_exec_missing_session_id(client): + resp = client.post("/api/exec", json={"code": "pass"}) + assert resp.status_code == 400 + + +def test_eval_missing_session_id(client): + resp = client.post("/api/eval", json={"expr": "1"}) + assert resp.status_code == 400 diff --git a/tests/test_worker.py b/tests/test_worker.py new file mode 100644 index 0000000..8397720 --- /dev/null +++ b/tests/test_worker.py @@ -0,0 +1,160 @@ +"""Unit tests for the worker subprocess message protocol.""" + +import json +import subprocess +import sys +from pathlib import Path + +WORKER_SCRIPT = str(Path(__file__).parent.parent / "pathview_server" / "worker.py") + + +def _start_worker(): + """Start a worker subprocess and return it.""" + return subprocess.Popen( + [sys.executable, "-u", WORKER_SCRIPT], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + +def _send(proc, msg): + """Send a JSON message to the worker.""" + proc.stdin.write(json.dumps(msg) + "\n") + proc.stdin.flush() + + +def _recv(proc): + """Read one JSON response from the worker.""" + line = proc.stdout.readline() + if not line: + return None + return json.loads(line.strip()) + + +def _recv_until(proc, target_type, target_id=None): + """Read messages until we get the target type. Returns (target_msg, other_msgs).""" + others = [] + while True: + msg = _recv(proc) + if msg is None: + raise RuntimeError("Worker died unexpectedly") + if msg.get("type") == target_type and (target_id is None or msg.get("id") == target_id): + return msg, others + others.append(msg) + + +def test_init_ready(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + msg, _ = _recv_until(proc, "ready") + assert msg["type"] == "ready" + finally: + proc.kill() + proc.wait() + + +def test_exec_ok(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + _send(proc, {"type": "exec", "id": "e1", "code": "x = 10"}) + msg, _ = _recv_until(proc, "ok", "e1") + assert msg["type"] == "ok" + assert msg["id"] == "e1" + finally: + proc.kill() + proc.wait() + + +def test_exec_error(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + _send(proc, {"type": "exec", "id": "e2", "code": "1/0"}) + msg, _ = _recv_until(proc, "error", "e2") + assert msg["type"] == "error" + assert "ZeroDivisionError" in msg.get("traceback", "") + finally: + proc.kill() + proc.wait() + + +def test_eval_value(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + _send(proc, {"type": "exec", "id": "e3", "code": "z = 42"}) + _recv_until(proc, "ok", "e3") + + _send(proc, {"type": "eval", "id": "v1", "expr": "z"}) + msg, _ = _recv_until(proc, "value", "v1") + assert json.loads(msg["value"]) == 42 + finally: + proc.kill() + proc.wait() + + +def test_eval_error(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + _send(proc, {"type": "eval", "id": "v2", "expr": "undefined_variable"}) + msg, _ = _recv_until(proc, "error", "v2") + assert msg["type"] == "error" + assert "NameError" in msg.get("traceback", "") or "undefined_variable" in msg.get("error", "") + finally: + proc.kill() + proc.wait() + + +def test_exec_print_captured(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + _send(proc, {"type": "exec", "id": "e4", "code": "print('captured')"}) + msg, others = _recv_until(proc, "ok", "e4") + stdout_msgs = [m for m in others if m.get("type") == "stdout"] + assert any("captured" in m.get("value", "") for m in stdout_msgs) + finally: + proc.kill() + proc.wait() + + +def test_exec_before_init(): + proc = _start_worker() + try: + _send(proc, {"type": "exec", "id": "e5", "code": "x = 1"}) + msg, _ = _recv_until(proc, "error", "e5") + assert "not initialized" in msg["error"].lower() + finally: + proc.kill() + proc.wait() + + +def test_double_init(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + # Second init should just return ready immediately + _send(proc, {"type": "init"}) + msg, _ = _recv_until(proc, "ready") + assert msg["type"] == "ready" + finally: + proc.kill() + proc.wait() From 67ec646815df2561be1feeb45571daa0c4b1f91c Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:28:40 +0100 Subject: [PATCH 17/27] Use pyproject.toml as single source of truth for version --- pathview_server/__init__.py | 6 +++++- scripts/build_package.py | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/pathview_server/__init__.py b/pathview_server/__init__.py index 8808f05..1431ca5 100644 --- a/pathview_server/__init__.py +++ b/pathview_server/__init__.py @@ -1,3 +1,7 @@ """PathView Server — local Flask backend for PathView.""" -__version__ = "0.5.0" +try: + from importlib.metadata import version + __version__ = version("pathview") +except Exception: + __version__ = "0.5.0" # fallback for editable installs / dev diff --git a/scripts/build_package.py b/scripts/build_package.py index 72841df..537ad31 100644 --- a/scripts/build_package.py +++ b/scripts/build_package.py @@ -7,7 +7,9 @@ 3. Builds the Python wheel """ +import json import os +import re import sys import shutil import subprocess @@ -28,7 +30,30 @@ def run(cmd, **kwargs): sys.exit(result.returncode) +def _sync_version(): + """Read version from pyproject.toml and sync to package.json.""" + pyproject = REPO_ROOT / "pyproject.toml" + text = pyproject.read_text() + match = re.search(r'^version\s*=\s*"([^"]+)"', text, re.MULTILINE) + if not match: + print("ERROR: could not find version in pyproject.toml") + sys.exit(1) + version = match.group(1) + + pkg_json_path = REPO_ROOT / "package.json" + pkg = json.loads(pkg_json_path.read_text()) + if pkg.get("version") != version: + print(f" Syncing version {pkg.get('version')} → {version} in package.json") + pkg["version"] = version + pkg_json_path.write_text(json.dumps(pkg, indent=2) + "\n") + return version + + def main(): + print("[0/4] Syncing version...") + version = _sync_version() + print(f" Version: {version}") + print("[1/4] Cleaning previous builds...") for d in [BUILD_DIR, STATIC_DIR, REPO_ROOT / "dist"]: if d.exists(): From 8c4aa8f94c76e64b04c6a0c903594a5c509c462c Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:28:55 +0100 Subject: [PATCH 18/27] Add network security warning when binding to 0.0.0.0 --- pathview_server/cli.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pathview_server/cli.py b/pathview_server/cli.py index 63340ce..df29875 100644 --- a/pathview_server/cli.py +++ b/pathview_server/cli.py @@ -50,7 +50,13 @@ def open_browser_when_ready(): print(f"PathView v{__version__}") print(f"Running at http://{args.host}:{args.port}") - print("Press Ctrl+C to stop\n") + + if args.host == "0.0.0.0": + print("\nWARNING: Binding to 0.0.0.0 makes the server accessible on your network.") + print(" There is no authentication — anyone on your network can execute Python code.") + print(" Only use this on trusted networks.") + + print("\nPress Ctrl+C to stop\n") try: if args.debug: From 71e37fdd8a7bf53661eaa6d07f3f860fadbe7f40 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:29:27 +0100 Subject: [PATCH 19/27] Share session across tabs via localStorage + BroadcastChannel --- src/lib/pyodide/backend/flask/backend.ts | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/lib/pyodide/backend/flask/backend.ts b/src/lib/pyodide/backend/flask/backend.ts index dcdb37e..63038eb 100644 --- a/src/lib/pyodide/backend/flask/backend.ts +++ b/src/lib/pyodide/backend/flask/backend.ts @@ -16,6 +16,9 @@ import { PYTHON_PACKAGES } from '$lib/constants/dependencies'; /** Polling interval for stream results (ms) */ const STREAM_POLL_INTERVAL = 30; +/** BroadcastChannel name for cross-tab session coordination */ +const SESSION_CHANNEL = 'flask-session'; + export class FlaskBackend implements Backend { private host: string; private sessionId: string; @@ -23,6 +26,7 @@ export class FlaskBackend implements Backend { private _isStreaming = false; private streamPollTimer: ReturnType | null = null; private serverInitPromise: Promise | null = null; + private broadcastChannel: BroadcastChannel | null = null; // Stream callbacks — same shape as PyodideBackend's streamState private streamState: { @@ -37,15 +41,26 @@ export class FlaskBackend implements Backend { constructor(host: string) { this.host = host.replace(/\/$/, ''); - const stored = typeof sessionStorage !== 'undefined' ? sessionStorage.getItem('flask-session-id') : null; + const stored = typeof localStorage !== 'undefined' ? localStorage.getItem('flask-session-id') : null; if (stored) { this.sessionId = stored; } else { this.sessionId = crypto.randomUUID(); - if (typeof sessionStorage !== 'undefined') { - sessionStorage.setItem('flask-session-id', this.sessionId); + if (typeof localStorage !== 'undefined') { + localStorage.setItem('flask-session-id', this.sessionId); } } + + // Listen for session termination from other tabs + if (typeof BroadcastChannel !== 'undefined') { + this.broadcastChannel = new BroadcastChannel(SESSION_CHANNEL); + this.broadcastChannel.onmessage = (event) => { + if (event.data?.type === 'session-terminated') { + this.serverInitPromise = null; + backendState.reset(); + } + }; + } } // ------------------------------------------------------------------------- @@ -119,6 +134,9 @@ export class FlaskBackend implements Backend { headers: { 'X-Session-ID': this.sessionId } }).catch(() => {}); + // Notify other tabs that the session was terminated + this.broadcastChannel?.postMessage({ type: 'session-terminated' }); + this.serverInitPromise = null; backendState.reset(); } From 24bf65f14acfb2677062cc398e58e2c998c88f62 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:29:46 +0100 Subject: [PATCH 20/27] Remove shell=True from build script, use explicit npx binary resolution --- scripts/build_package.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/build_package.py b/scripts/build_package.py index 537ad31..68eb4ad 100644 --- a/scripts/build_package.py +++ b/scripts/build_package.py @@ -20,11 +20,19 @@ STATIC_DIR = REPO_ROOT / "pathview_server" / "static" +def _find_npx(): + """Find the npx binary. On Windows, use npx.cmd.""" + name = "npx.cmd" if sys.platform == "win32" else "npx" + path = shutil.which(name) + if not path: + print(f"ERROR: {name} not found on PATH") + sys.exit(1) + return path + + def run(cmd, **kwargs): print(f" > {' '.join(cmd)}") - # shell=True needed on Windows for npx/npm resolution - result = subprocess.run(cmd, cwd=kwargs.pop("cwd", REPO_ROOT), - shell=(sys.platform == "win32"), **kwargs) + result = subprocess.run(cmd, cwd=kwargs.pop("cwd", REPO_ROOT), **kwargs) if result.returncode != 0: print(f"ERROR: command failed (exit {result.returncode})") sys.exit(result.returncode) @@ -66,7 +74,8 @@ def main(): print("[2/4] Building SvelteKit frontend...") env = os.environ.copy() env["BASE_PATH"] = "" - run(["npx", "vite", "build"], env=env) + npx = _find_npx() + run([npx, "vite", "build"], env=env) if not (BUILD_DIR / "index.html").exists(): print("ERROR: build/index.html not found") From c2c81627efdf63fe2becbadb8aca8d385445286b Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 21:42:51 +0100 Subject: [PATCH 21/27] =?UTF-8?q?Fix=20stream=20stop=E2=86=92exec=20crash:?= =?UTF-8?q?=20wait=20for=20reader=20thread=20before=20stdout=20reads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pathview_server/app.py | 18 +++++++++++++ tests/test_app.py | 60 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/pathview_server/app.py b/pathview_server/app.py index d4397d8..3bfb43e 100644 --- a/pathview_server/app.py +++ b/pathview_server/app.py @@ -145,6 +145,18 @@ def stop_stream_reader(self) -> None: """Signal the stream reader to stop.""" self._streaming = False + def wait_for_stream_reader(self, timeout: float = 5) -> None: + """Wait for the stream reader thread to fully exit. + + Must be called before any direct stdout reads (exec/eval) to prevent + concurrent reads on the same pipe which cause JSONDecodeError. + """ + reader = self._stream_reader + if reader is not None and reader.is_alive(): + self._streaming = False + reader.join(timeout) + self._stream_reader = None + def flush_worker_reader(self) -> None: """Send a noop message to unblock the worker's stdin reader thread. @@ -298,6 +310,9 @@ def api_exec(): session = get_or_create_session(session_id) with session.lock: try: + # Wait for any lingering stream reader thread to exit before + # reading stdout — prevents concurrent pipe reads / JSONDecodeError + session.wait_for_stream_reader() session.ensure_initialized() session.send_message({"type": "exec", "id": msg_id, "code": code}) @@ -347,6 +362,9 @@ def api_eval(): session = get_or_create_session(session_id) with session.lock: try: + # Wait for any lingering stream reader thread to exit before + # reading stdout — prevents concurrent pipe reads / JSONDecodeError + session.wait_for_stream_reader() session.ensure_initialized() session.send_message({"type": "eval", "id": msg_id, "expr": expr}) diff --git a/tests/test_app.py b/tests/test_app.py index 354a6d5..6215521 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -147,6 +147,66 @@ def test_streaming_lifecycle(client, session_headers): assert "stream-done" in types +def test_stream_stop_then_exec(client, session_headers): + """Regression: exec after stream/stop must not crash with JSONDecodeError. + + The stream reader thread must fully exit before exec reads stdout. + """ + client.post("/api/init", json={"packages": []}, headers=session_headers) + + # Set up a generator that never finishes on its own (needs manual stop) + client.post( + "/api/exec", + json={"code": "_counter = 0\ndef _infinite():\n global _counter\n _counter += 1\n return {'result': _counter, 'done': False}"}, + headers=session_headers, + ) + + # Start streaming + resp = client.post( + "/api/stream/start", + json={"expr": "_infinite()"}, + headers=session_headers, + ) + assert resp.status_code == 200 + + # Let a few polls go through + import time + for _ in range(5): + client.post("/api/stream/poll", headers=session_headers) + time.sleep(0.05) + + # Stop streaming + client.post("/api/stream/stop", headers=session_headers) + + # Poll until done + done = False + deadline = time.time() + 10 + while not done and time.time() < deadline: + resp = client.post("/api/stream/poll", headers=session_headers) + data = resp.get_json() + done = data.get("done", False) + if not done: + time.sleep(0.1) + + # Now exec should work without crashing + resp = client.post( + "/api/exec", + json={"code": "result_after_stop = 'ok'"}, + headers=session_headers, + ) + assert resp.status_code == 200 + assert resp.get_json()["type"] == "ok" + + # Verify the namespace is intact + resp = client.post( + "/api/eval", + json={"expr": "result_after_stop"}, + headers=session_headers, + ) + assert resp.status_code == 200 + assert json.loads(resp.get_json()["value"]) == "ok" + + def test_exec_missing_session_id(client): resp = client.post("/api/exec", json={"code": "pass"}) assert resp.status_code == 400 From 66061f0cbad823aa8d2ef76266b1a705b727545f Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 22:03:07 +0100 Subject: [PATCH 22/27] =?UTF-8?q?Fix=20stop=E2=86=92restart=20hang:=20acti?= =?UTF-8?q?vely=20stop=20streaming=20in=20wait=5Ffor=5Fstream=5Freader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pathview_server/app.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/pathview_server/app.py b/pathview_server/app.py index 3bfb43e..1b0c91b 100644 --- a/pathview_server/app.py +++ b/pathview_server/app.py @@ -150,13 +150,38 @@ def wait_for_stream_reader(self, timeout: float = 5) -> None: Must be called before any direct stdout reads (exec/eval) to prevent concurrent reads on the same pipe which cause JSONDecodeError. + + Actively stops streaming if needed: sends stream-stop to the worker + so it sends stream-done, which unblocks the reader thread. """ reader = self._stream_reader - if reader is not None and reader.is_alive(): + if reader is None: + return + + if reader.is_alive(): self._streaming = False + # Ensure the worker stops streaming — stream-stop might not have + # been sent yet if api_stream_stop races with api_exec. + try: + self.send_message({"type": "stream-stop"}) + except Exception: + pass reader.join(timeout) + + # Send noop to unblock the worker's stdin reader thread so the + # worker main loop can resume processing exec/eval messages. + self.flush_worker_reader() + self._stream_reader = None + # Drain stale stream messages (stream-data, stream-done) from the + # queue so they don't leak into subsequent poll responses. + while not self._stream_queue.empty(): + try: + self._stream_queue.get_nowait() + except queue.Empty: + break + def flush_worker_reader(self) -> None: """Send a noop message to unblock the worker's stdin reader thread. From 06227776084727f739d824b1f0e7a9234a432238 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Tue, 10 Feb 2026 22:10:41 +0100 Subject: [PATCH 23/27] Keep final stream data in queue so frontend can poll it before restart --- pathview_server/app.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/pathview_server/app.py b/pathview_server/app.py index 1b0c91b..d36358c 100644 --- a/pathview_server/app.py +++ b/pathview_server/app.py @@ -173,14 +173,9 @@ def wait_for_stream_reader(self, timeout: float = 5) -> None: self.flush_worker_reader() self._stream_reader = None - - # Drain stale stream messages (stream-data, stream-done) from the - # queue so they don't leak into subsequent poll responses. - while not self._stream_queue.empty(): - try: - self._stream_queue.get_nowait() - except queue.Empty: - break + # Don't drain the stream queue here — the frontend's poll chain + # still needs the final stream-data/stream-done messages. + # start_stream_reader() clears stale messages when a new stream begins. def flush_worker_reader(self) -> None: """Send a noop message to unblock the worker's stdin reader thread. From 412d7deb96d9e293c3db71a28c48a25b33ea52c3 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Wed, 11 Feb 2026 10:05:22 +0100 Subject: [PATCH 24/27] Add virtualenv isolation for Flask backend workers --- pathview_server/app.py | 52 ++++++++++++++++++++++++++++----------- pathview_server/cli.py | 6 ++++- pathview_server/venv.py | 34 +++++++++++++++++++++++++ pathview_server/worker.py | 6 +++-- 4 files changed, 81 insertions(+), 17 deletions(-) create mode 100644 pathview_server/venv.py diff --git a/pathview_server/app.py b/pathview_server/app.py index d36358c..bf717bc 100644 --- a/pathview_server/app.py +++ b/pathview_server/app.py @@ -21,6 +21,8 @@ from flask import Flask, request, jsonify, send_from_directory from flask_cors import CORS +from pathview_server.venv import get_venv_python + # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- @@ -42,7 +44,7 @@ def __init__(self, session_id: str): self.last_active = time.time() self.lock = threading.Lock() self.process = subprocess.Popen( - [sys.executable, "-u", WORKER_SCRIPT], + [get_venv_python(), "-u", WORKER_SCRIPT], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -152,21 +154,29 @@ def wait_for_stream_reader(self, timeout: float = 5) -> None: concurrent reads on the same pipe which cause JSONDecodeError. Actively stops streaming if needed: sends stream-stop to the worker - so it sends stream-done, which unblocks the reader thread. + so it sends stream-done, which the reader thread reads naturally and + exits via its own loop termination. This ensures all final stream-data + messages are read into the queue before the reader exits. """ reader = self._stream_reader if reader is None: return if reader.is_alive(): - self._streaming = False - # Ensure the worker stops streaming — stream-stop might not have - # been sent yet if api_stream_stop races with api_exec. + # Do NOT set self._streaming = False here — that would cause the + # reader thread to exit early on its next loop check, missing the + # final stream-data and stream-done messages. Instead, send + # stream-stop so the worker sends stream-done, which the reader + # reads and exits naturally (line 137-139 in start_stream_reader). try: self.send_message({"type": "stream-stop"}) except Exception: pass reader.join(timeout) + # If reader is still alive after timeout, force-stop it + if reader.is_alive(): + self._streaming = False + reader.join(1) # Send noop to unblock the worker's stdin reader thread so the # worker main loop can resume processing exec/eval messages. @@ -189,9 +199,19 @@ def flush_worker_reader(self) -> None: except Exception: pass - def drain_stream_queue(self) -> list[dict]: - """Drain all messages currently in the stream queue.""" - messages = [] + def drain_stream_queue(self, timeout: float = 0) -> list[dict]: + """Drain all messages from the stream queue. + + If timeout > 0, blocks until at least one message arrives or + the timeout expires. This turns polling into long-polling, + eliminating empty responses and reducing HTTP overhead. + """ + messages: list[dict] = [] + if timeout > 0 and self._stream_queue.empty(): + try: + messages.append(self._stream_queue.get(timeout=timeout)) + except queue.Empty: + return messages while True: try: messages.append(self._stream_queue.get_nowait()) @@ -290,7 +310,7 @@ def create_app(serve_static: bool = False) -> Flask: _start_cleanup_thread() if not serve_static: - CORS(app) + CORS(app, max_age=3600) # ----------------------------------------------------------------------- # API routes @@ -425,9 +445,8 @@ def api_eval(): def api_stream_start(): """Start streaming — sends stream-start to worker and returns immediately. - A background thread reads worker stdout into a queue. The frontend - polls /api/stream/poll to drain that queue, mirroring how the Pyodide - worker sends stream-data / stream-done messages via postMessage. + A background thread reads worker stdout into a queue. The + frontend long-polls /api/stream/poll to drain that queue. """ session_id = _get_session_id() if not session_id: @@ -448,7 +467,12 @@ def api_stream_start(): @app.route("/api/stream/poll", methods=["POST"]) def api_stream_poll(): - """Poll for stream messages — returns all queued messages since last poll.""" + """Long-poll for stream messages. + + Blocks up to 100 ms waiting for data before returning. + This eliminates empty responses and reduces HTTP overhead + compared to blind 30 ms polling. + """ session_id = _get_session_id() if not session_id: return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 @@ -456,7 +480,7 @@ def api_stream_poll(): session = _sessions.get(session_id) if not session: return jsonify({"messages": [], "done": True}) - messages = session.drain_stream_queue() + messages = session.drain_stream_queue(timeout=0.1) done = any(m.get("type") in ("stream-done", "error") for m in messages) if done: # Send noop to unblock the worker's stdin reader thread diff --git a/pathview_server/cli.py b/pathview_server/cli.py index df29875..e82b7d1 100644 --- a/pathview_server/cli.py +++ b/pathview_server/cli.py @@ -7,6 +7,7 @@ import webbrowser from pathview_server import __version__ +from pathview_server.venv import VENV_DIR, ensure_venv def main(): @@ -29,9 +30,11 @@ def main(): args = parser.parse_args() + ensure_venv() + from pathview_server.app import create_app - app = create_app(serve_static=True) + app = create_app(serve_static=not args.debug) if not args.no_browser: def open_browser_when_ready(): @@ -49,6 +52,7 @@ def open_browser_when_ready(): threading.Thread(target=open_browser_when_ready, daemon=True).start() print(f"PathView v{__version__}") + print(f" Python venv: {VENV_DIR}") print(f"Running at http://{args.host}:{args.port}") if args.host == "0.0.0.0": diff --git a/pathview_server/venv.py b/pathview_server/venv.py new file mode 100644 index 0000000..df8e28a --- /dev/null +++ b/pathview_server/venv.py @@ -0,0 +1,34 @@ +"""Virtual environment management for PathView worker subprocesses. + +Creates and manages a dedicated venv at ~/.pathview/venv so that simulation +dependencies (pathsim, pathsim-chem, numpy, etc.) are installed in isolation +rather than polluting the user's global/active environment. +""" + +import subprocess +import sys +from pathlib import Path + +VENV_DIR = Path.home() / ".pathview" / "venv" + + +def get_venv_python() -> str: + """Return path to the venv's Python executable.""" + if sys.platform == "win32": + return str(VENV_DIR / "Scripts" / "python.exe") + return str(VENV_DIR / "bin" / "python") + + +def ensure_venv() -> str: + """Create the venv if it doesn't exist. Returns venv Python path.""" + python = get_venv_python() + if Path(python).exists(): + return python + + print("Creating PathView virtual environment...") + VENV_DIR.parent.mkdir(parents=True, exist_ok=True) + subprocess.run([sys.executable, "-m", "venv", str(VENV_DIR)], check=True) + # Upgrade pip in the venv + subprocess.run([python, "-m", "pip", "install", "--upgrade", "pip", "--quiet"], check=True) + print(f" Virtual environment created at {VENV_DIR}") + return python diff --git a/pathview_server/worker.py b/pathview_server/worker.py index c31322c..04b0b7e 100644 --- a/pathview_server/worker.py +++ b/pathview_server/worker.py @@ -143,11 +143,10 @@ def initialize(packages: list[dict] | None = None) -> None: # Set up the namespace with common imports _namespace = {"__builtins__": __builtins__} - exec("import numpy as np", _namespace) exec("import gc", _namespace) exec("import json", _namespace) - # Install and import packages from the frontend config (single source of truth) + # Install packages FIRST (pathsim brings numpy as a dependency) if packages: send({"type": "progress", "value": "Installing dependencies..."}) for pkg in packages: @@ -160,6 +159,9 @@ def initialize(packages: list[dict] | None = None) -> None: ) send({"type": "stderr", "value": f"Optional package {pkg.get('import', '?')} failed: {e}\n"}) + # Import numpy AFTER packages are installed (numpy comes with pathsim) + exec("import numpy as np", _namespace) + _initialized = True send({"type": "ready"}) From 5a315c03ffd39434c5e4580680386096425894f5 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Wed, 11 Feb 2026 10:16:10 +0100 Subject: [PATCH 25/27] Fix CI: ensure venv exists before app tests spawn workers --- tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index e5a0b75..d3ceb8f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,11 +3,13 @@ import pytest from pathview_server.app import create_app, _sessions, _sessions_lock +from pathview_server.venv import ensure_venv @pytest.fixture() def app(): """Create a Flask test app (API-only, no static serving).""" + ensure_venv() application = create_app(serve_static=False) application.config["TESTING"] = True yield application From f4cfa69171e54bd82e2c079c99a8b1057a726333 Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Wed, 11 Feb 2026 10:17:34 +0100 Subject: [PATCH 26/27] Fix CI: handle missing numpy in fresh venv without simulation packages --- pathview_server/worker.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pathview_server/worker.py b/pathview_server/worker.py index 04b0b7e..2546e74 100644 --- a/pathview_server/worker.py +++ b/pathview_server/worker.py @@ -159,8 +159,12 @@ def initialize(packages: list[dict] | None = None) -> None: ) send({"type": "stderr", "value": f"Optional package {pkg.get('import', '?')} failed: {e}\n"}) - # Import numpy AFTER packages are installed (numpy comes with pathsim) - exec("import numpy as np", _namespace) + # Import numpy AFTER packages are installed (numpy comes with pathsim). + # In a fresh venv without simulation packages, numpy won't be available. + try: + exec("import numpy as np", _namespace) + except Exception: + pass _initialized = True send({"type": "ready"}) From 796a2468eab8b534fe0ab53d99bec2bddfcfcf3f Mon Sep 17 00:00:00 2001 From: Milan Rother Date: Wed, 11 Feb 2026 10:45:48 +0100 Subject: [PATCH 27/27] Consolidate duplicated exec/eval routes and init logic --- pathview_server/app.py | 79 +++++----------- src/lib/pyodide/backend/flask/backend.ts | 112 +++++++++++------------ 2 files changed, 75 insertions(+), 116 deletions(-) diff --git a/pathview_server/app.py b/pathview_server/app.py index bf717bc..5bf56ba 100644 --- a/pathview_server/app.py +++ b/pathview_server/app.py @@ -337,15 +337,16 @@ def api_init(): except Exception as e: return jsonify({"type": "error", "error": str(e)}), 500 - @app.route("/api/exec", methods=["POST"]) - def api_exec(): - """Execute Python code in the session's worker.""" + def _handle_worker_request(msg: dict, success_type: str) -> tuple: + """Send a message to the worker and collect the response. + + Shared by api_exec (success_type="ok") and api_eval (success_type="value"). + Returns a Flask response tuple. + """ session_id = _get_session_id() if not session_id: return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 - data = request.get_json(force=True) - code = data.get("code", "") - msg_id = data.get("id", str(uuid.uuid4())) + msg_id = msg.get("id", str(uuid.uuid4())) session = get_or_create_session(session_id) with session.lock: @@ -354,7 +355,7 @@ def api_exec(): # reading stdout — prevents concurrent pipe reads / JSONDecodeError session.wait_for_stream_reader() session.ensure_initialized() - session.send_message({"type": "exec", "id": msg_id, "code": code}) + session.send_message(msg) stdout_lines = [] stderr_lines = [] @@ -368,8 +369,8 @@ def api_exec(): stdout_lines.append(resp.get("value", "")) elif resp_type == "stderr": stderr_lines.append(resp.get("value", "")) - elif resp_type == "ok" and resp.get("id") == msg_id: - result = {"type": "ok", "id": msg_id} + elif resp_type == success_type and resp.get("id") == msg_id: + result = resp if stdout_lines: result["stdout"] = "".join(stdout_lines) if stderr_lines: @@ -389,57 +390,25 @@ def api_exec(): except Exception as e: return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 + @app.route("/api/exec", methods=["POST"]) + def api_exec(): + """Execute Python code in the session's worker.""" + data = request.get_json(force=True) + msg_id = data.get("id", str(uuid.uuid4())) + return _handle_worker_request( + {"type": "exec", "id": msg_id, "code": data.get("code", "")}, + success_type="ok", + ) + @app.route("/api/eval", methods=["POST"]) def api_eval(): """Evaluate a Python expression in the session's worker.""" - session_id = _get_session_id() - if not session_id: - return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 data = request.get_json(force=True) - expr = data.get("expr", "") msg_id = data.get("id", str(uuid.uuid4())) - - session = get_or_create_session(session_id) - with session.lock: - try: - # Wait for any lingering stream reader thread to exit before - # reading stdout — prevents concurrent pipe reads / JSONDecodeError - session.wait_for_stream_reader() - session.ensure_initialized() - session.send_message({"type": "eval", "id": msg_id, "expr": expr}) - - stdout_lines = [] - stderr_lines = [] - while True: - resp = session.read_line_timeout() - if resp is None: - remove_session(session_id) - return jsonify({"type": "error", "errorType": "worker-crashed", "id": msg_id, "error": "Worker process died"}), 500 - resp_type = resp.get("type") - if resp_type == "stdout": - stdout_lines.append(resp.get("value", "")) - elif resp_type == "stderr": - stderr_lines.append(resp.get("value", "")) - elif resp_type == "value" and resp.get("id") == msg_id: - result = resp - if stdout_lines: - result["stdout"] = "".join(stdout_lines) - if stderr_lines: - result["stderr"] = "".join(stderr_lines) - return jsonify(result) - elif resp_type == "error" and resp.get("id") == msg_id: - result = resp - if stdout_lines: - result["stdout"] = "".join(stdout_lines) - if stderr_lines: - result["stderr"] = "".join(stderr_lines) - return jsonify(result), 400 - - except TimeoutError: - remove_session(session_id) - return jsonify({"type": "error", "errorType": "timeout", "id": msg_id, "error": "Execution timed out"}), 504 - except Exception as e: - return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 + return _handle_worker_request( + {"type": "eval", "id": msg_id, "expr": data.get("expr", "")}, + success_type="value", + ) @app.route("/api/stream/start", methods=["POST"]) def api_stream_start(): diff --git a/src/lib/pyodide/backend/flask/backend.ts b/src/lib/pyodide/backend/flask/backend.ts index 63038eb..cdb19ca 100644 --- a/src/lib/pyodide/backend/flask/backend.ts +++ b/src/lib/pyodide/backend/flask/backend.ts @@ -13,8 +13,10 @@ import { TIMEOUTS } from '$lib/constants/python'; import { STATUS_MESSAGES } from '$lib/constants/messages'; import { PYTHON_PACKAGES } from '$lib/constants/dependencies'; -/** Polling interval for stream results (ms) */ -const STREAM_POLL_INTERVAL = 30; +/** Delay between polls (ms). The server uses long-polling (blocks up + * to 100 ms until data arrives), so data delivery is near-instant. + * This interval is just a safety gap between consecutive requests. */ +const STREAM_POLL_INTERVAL = 5; /** BroadcastChannel name for cross-tab session coordination */ const SESSION_CHANNEL = 'flask-session'; @@ -86,28 +88,7 @@ export class FlaskBackend implements Backend { backendState.update((s) => ({ ...s, progress: 'Initializing Python worker...' })); - const initResp = await fetch(`${this.host}/api/init`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Session-ID': this.sessionId - }, - body: JSON.stringify({ packages: PYTHON_PACKAGES }), - signal: AbortSignal.timeout(TIMEOUTS.INIT) - }); - const initData = await initResp.json(); - - if (initData.type === 'error') throw new Error(initData.error); - - if (initData.messages) { - for (const msg of initData.messages) { - if (msg.type === 'stdout' && this.stdoutCallback) this.stdoutCallback(msg.value); - if (msg.type === 'stderr' && this.stderrCallback) this.stderrCallback(msg.value); - if (msg.type === 'progress') { - backendState.update((s) => ({ ...s, progress: msg.value })); - } - } - } + await this.postInit({ updateProgress: true }); this.serverInitPromise = Promise.resolve(); @@ -126,6 +107,10 @@ export class FlaskBackend implements Backend { terminate(): void { this.stopStreaming(); + if (this.streamPollTimer) { + clearTimeout(this.streamPollTimer); + this.streamPollTimer = null; + } this._isStreaming = false; this.streamState = { onData: null, onDone: null, onError: null }; @@ -172,25 +157,7 @@ export class FlaskBackend implements Backend { private ensureServerInit(): Promise { if (this.serverInitPromise) return this.serverInitPromise; - this.serverInitPromise = (async () => { - const resp = await fetch(`${this.host}/api/init`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Session-ID': this.sessionId - }, - body: JSON.stringify({ packages: PYTHON_PACKAGES }), - signal: AbortSignal.timeout(TIMEOUTS.INIT) - }); - const data = await resp.json(); - if (data.type === 'error') throw new Error(data.error); - if (data.messages) { - for (const msg of data.messages) { - if (msg.type === 'stdout' && this.stdoutCallback) this.stdoutCallback(msg.value); - if (msg.type === 'stderr' && this.stderrCallback) this.stderrCallback(msg.value); - } - } - })(); + this.serverInitPromise = this.postInit({ updateProgress: false }); // Clear on failure so subsequent calls retry instead of returning the rejected promise this.serverInitPromise.catch(() => { @@ -271,6 +238,13 @@ export class FlaskBackend implements Backend { this.stopStreaming(); } + // Clear any lingering poll timer from a previous stream so it + // doesn't pick up a stale stream-done and fire the NEW onDone. + if (this.streamPollTimer) { + clearTimeout(this.streamPollTimer); + this.streamPollTimer = null; + } + const id = this.generateId(); this._isStreaming = true; this.streamState = { @@ -296,7 +270,6 @@ export class FlaskBackend implements Backend { if (data.type === 'error') { throw new Error(data.error); } - // Start polling loop — same as Pyodide worker's onmessage dispatching this.pollStreamResults(); }) .catch((error) => { @@ -309,29 +282,19 @@ export class FlaskBackend implements Backend { stopStreaming(): void { if (!this._isStreaming) return; - // Stop polling timer — the server will send stream-done which triggers onDone - if (this.streamPollTimer) { - clearTimeout(this.streamPollTimer); - this.streamPollTimer = null; - } - - // Tell server to stop, then do one final poll to get the stream-done message + // Just send the stop signal — don't disrupt the polling loop. + // The poll loop will naturally pick up stream-done and clean up, + // matching how Pyodide's stopStreaming just sends stream-stop and + // lets worker.onmessage handle the rest. fetch(`${this.host}/api/stream/stop`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Session-ID': this.sessionId } - }) - .then(() => this.pollStreamResults()) - .catch(() => { - // If final poll fails, clean up locally - this._isStreaming = false; - if (this.streamState.onDone) { - this.streamState.onDone(); - } - this.streamState = { onData: null, onDone: null, onError: null }; - }); + }).catch(() => { + // Network failure — poll loop will also fail and clean up + }); } isStreaming(): boolean { @@ -374,6 +337,33 @@ export class FlaskBackend implements Backend { return `repl_${++this.messageId}`; } + /** + * POST /api/init with packages and forward worker messages to callbacks. + * Shared by init() (first load with progress UI) and ensureServerInit() (lazy re-init). + */ + private async postInit(opts: { updateProgress: boolean }): Promise { + const resp = await fetch(`${this.host}/api/init`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ packages: PYTHON_PACKAGES }), + signal: AbortSignal.timeout(TIMEOUTS.INIT) + }); + const data = await resp.json(); + if (data.type === 'error') throw new Error(data.error); + if (data.messages) { + for (const msg of data.messages) { + if (msg.type === 'stdout' && this.stdoutCallback) this.stdoutCallback(msg.value); + if (msg.type === 'stderr' && this.stderrCallback) this.stderrCallback(msg.value); + if (msg.type === 'progress' && opts.updateProgress) { + backendState.update((s) => ({ ...s, progress: msg.value })); + } + } + } + } + /** * Check if a response indicates the worker crashed or timed out. * If so, clear serverInitPromise so the next request triggers re-init.