diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 00000000..0fe59be7 --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,77 @@ +name: Publish to PyPI + +on: + release: + types: [published] + workflow_dispatch: + inputs: + publish_to: + description: 'Publish target' + required: true + default: 'testpypi' + type: choice + options: + - testpypi + - pypi + +permissions: + contents: read + +jobs: + build-and-publish: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Python build tools + run: pip install build twine + + - name: Install Python dependencies + run: | + pip install -r scripts/config/requirements-pyodide.txt + pip install -r scripts/config/requirements-build.txt + + - name: Install Node dependencies + run: npm ci + + - name: Extract blocks from PathSim + run: npm run extract + + - name: Build package (frontend + wheel) + run: python scripts/build_package.py + + - name: Check distribution + run: twine check dist/* + + - name: Publish to Test PyPI + if: github.event_name == 'workflow_dispatch' && inputs.publish_to == 'testpypi' + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} + run: twine upload --repository testpypi dist/* + + - name: Publish to PyPI + if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.publish_to == 'pypi') + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: twine upload dist/* + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..517f3136 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,31 @@ +name: Tests + +on: + push: + branches: [main, feature/flask-backend] + pull_request: + branches: [main, feature/flask-backend] + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[test]" + + - name: Run tests + run: pytest tests/ -v diff --git a/.gitignore b/.gitignore index b4947e25..7ae8466c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,9 @@ tmpclaude-* __pycache__/ +*.egg-info/ +dist/ +pathview_server/static/ # Generated screenshots static/examples/screenshots/ diff --git a/README.md b/README.md index b7f7f82c..39087945 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ # PathView - System Modeling in the Browser -A web-based visual node editor for building and simulating dynamic systems with [PathSim](https://github.com/pathsim/pathsim) as the backend. Runs entirely in the browser via Pyodide - no server required. The UI is hosted at [view.pathsim.org](https://view.pathsim.org), free to use for everyone. +A web-based visual node editor for building and simulating dynamic systems with [PathSim](https://github.com/pathsim/pathsim) as the backend. Runs entirely in the browser via Pyodide by default — no server required. Optionally, a Flask backend enables server-side Python execution with any packages (including those with native dependencies that Pyodide can't run). The UI is hosted at [view.pathsim.org](https://view.pathsim.org), free to use for everyone. ## Tech Stack @@ -17,18 +17,37 @@ A web-based visual node editor for building and simulating dynamic systems with - [Plotly.js](https://plotly.com/javascript/) for interactive plots - [CodeMirror 6](https://codemirror.net/) for code editing -## Getting Started +## Installation + +### pip install (recommended for users) + +```bash +pip install pathview +pathview serve +``` + +This starts the PathView server with a local Python backend and opens your browser. No Node.js required. + +**Options:** +- `--port PORT` — server port (default: 5000) +- `--host HOST` — bind address (default: 127.0.0.1) +- `--no-browser` — don't auto-open the browser +- `--debug` — debug mode with auto-reload + +### Development setup ```bash npm install npm run dev ``` -For production: +To use the Flask backend during development: ```bash -npm run build -npm run preview +pip install flask flask-cors +npm run server # Start Flask backend on port 5000 +npm run dev # Start Vite dev server (separate terminal) +# Open http://localhost:5173/?backend=flask ``` ## Project Structure @@ -61,7 +80,8 @@ src/ │ ├── routing/ # Orthogonal wire routing (A* pathfinding) │ ├── pyodide/ # Python runtime (backend, bridge) │ │ └── backend/ # Modular backend system (registry, state, types) -│ │ └── pyodide/ # Pyodide Web Worker implementation +│ │ ├── pyodide/ # Pyodide Web Worker implementation +│ │ └── flask/ # Flask HTTP/SSE backend implementation │ ├── schema/ # File I/O (save/load, component export) │ ├── simulation/ # Simulation metadata │ │ └── generated/ # Auto-generated defaults @@ -72,6 +92,12 @@ src/ ├── routes/ # SvelteKit pages └── app.css # Global styles with CSS variables +pathview_server/ # Python package (pip install pathview) +├── app.py # Flask server (subprocess management, HTTP routes) +├── worker.py # REPL worker subprocess (Python execution) +├── cli.py # CLI entry point (pathview serve) +└── static/ # Bundled frontend (generated at build time) + scripts/ ├── config/ # Configuration files for extraction │ ├── schemas/ # JSON schemas for validation @@ -100,8 +126,8 @@ scripts/ │ v ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ Plot/Console │<────│ bridge.ts │<────│ REPL Worker │ -│ (results) │ │ (queue + rAF) │ │ (Pyodide) │ +│ Plot/Console │<────│ bridge.ts │<────│ Backend │ +│ (results) │ │ (queue + rAF) │ │ (Pyodide/Flask) │ └─────────────────┘ └─────────────────┘ └─────────────────┘ ``` @@ -153,6 +179,7 @@ Key files: `src/lib/routing/` (pathfinder, grid builder, route calculator) | **Backend** | Modular Python execution interface | `pyodide/backend/` | | **Backend Registry** | Factory for swappable backends | `pyodide/backend/registry.ts` | | **PyodideBackend** | Web Worker Pyodide implementation | `pyodide/backend/pyodide/` | +| **FlaskBackend** | HTTP/SSE Flask server implementation | `pyodide/backend/flask/` | | **Simulation Bridge** | High-level simulation API | `pyodide/bridge.ts` | | **Schema** | File/component save/load operations | `schema/fileOps.ts`, `schema/componentOps.ts` | | **Export Utils** | SVG/CSV/Python file downloads | `utils/download.ts`, `export/svg/`, `utils/csvExport.ts` | @@ -309,6 +336,8 @@ npm run build No code changes needed - the extraction script automatically discovers toolbox directories. +For the full toolbox integration reference (Python package contract, config schemas, extraction pipeline, generated output), see [**docs/toolbox-spec.md**](docs/toolbox-spec.md). + --- ## Python Backend System @@ -326,29 +355,36 @@ The Python runtime uses a modular backend architecture, allowing different execu ┌──────────────┼──────────────┐ ▼ ▼ ▼ ┌───────────┐ ┌───────────┐ ┌───────────┐ - │ Pyodide │ │ Local │ │ Remote │ + │ Pyodide │ │ Flask │ │ Remote │ │ Backend │ │ Backend │ │ Backend │ - │ (Worker) │ │ (Flask) │ │ (Server) │ + │ (default) │ │ (HTTP) │ │ (future) │ └───────────┘ └───────────┘ └───────────┘ - │ (future) (future) - ▼ - ┌───────────┐ - │ Web Worker│ - │ (Pyodide) │ - └───────────┘ + │ │ + ▼ ▼ + ┌───────────┐ ┌───────────┐ + │ Web Worker│ │ Flask │──> Python subprocess + │ (Pyodide) │ │ Server │ (one per session) + └───────────┘ └───────────┘ ``` ### Backend Registry ```typescript -import { getBackend, switchBackend } from '$lib/pyodide/backend'; +import { getBackend, switchBackend, setFlaskHost } from '$lib/pyodide/backend'; // Get current backend (defaults to Pyodide) const backend = getBackend(); -// Switch to a different backend type (future) -// switchBackend('local'); // Use local Python via Flask -// switchBackend('remote'); // Use remote server +// Switch to Flask backend +setFlaskHost('http://localhost:5000'); +switchBackend('flask'); +``` + +Backend selection can also be controlled via URL parameters: + +``` +http://localhost:5173/?backend=flask # Flask on default port +http://localhost:5173/?backend=flask&host=http://myserver:5000 # Custom host ``` ### REPL Protocol @@ -426,6 +462,60 @@ await stopSimulation(); execDuringStreaming('source.amplitude = 2.0'); ``` +### Flask Backend + +The Flask backend enables server-side Python execution for packages that Pyodide can't run (e.g., FESTIM or other packages with native C/Fortran dependencies). It mirrors the Web Worker architecture: one subprocess per session with the same REPL protocol. + +``` +Browser Tab Flask Server Worker Subprocess +┌──────────────┐ ┌──────────────────┐ ┌──────────────────┐ +│ FlaskBackend │ HTTP/SSE │ app.py │ stdin │ worker.py │ +│ exec() │──POST────────→│ route → session │──JSON───→│ exec(code, ns) │ +│ eval() │──POST────────→│ subprocess mgr │──JSON───→│ eval(expr, ns) │ +│ stream() │──POST (SSE)──→│ pipe SSE relay │←─JSON────│ streaming loop │ +│ inject() │──POST────────→│ → code queue │──JSON───→│ queue drain │ +│ stop() │──POST────────→│ → stop flag │──JSON───→│ stop check │ +└──────────────┘ └──────────────────┘ └──────────────────┘ +``` + +**Standalone (pip package):** + +```bash +pip install pathview +pathview serve +``` + +**Development (separate servers):** + +```bash +pip install flask flask-cors +npm run server # Starts Flask API on port 5000 +npm run dev # Starts Vite dev server (separate terminal) +# Open http://localhost:5173/?backend=flask +``` + +**Key properties:** +- **Process isolation** — each session gets its own Python subprocess +- **Namespace persistence** — variables persist across exec/eval calls within a session +- **Dynamic packages** — packages from `PYTHON_PACKAGES` (the same config used by Pyodide) are pip-installed on first init +- **Session TTL** — stale sessions cleaned up after 1 hour of inactivity +- **Streaming** — simulations stream via SSE, with the same code injection support as Pyodide + +For the full protocol reference (message types, HTTP routes, SSE format, streaming semantics, how to implement a new backend), see [**docs/backend-protocol-spec.md**](docs/backend-protocol-spec.md). + +**API routes:** + +| Route | Method | Action | +|-------|--------|--------| +| `/api/health` | GET | Health check | +| `/api/init` | POST | Initialize worker with packages | +| `/api/exec` | POST | Execute Python code | +| `/api/eval` | POST | Evaluate expression, return JSON | +| `/api/stream` | POST | Start streaming simulation (SSE) | +| `/api/stream/exec` | POST | Inject code during streaming | +| `/api/stream/stop` | POST | Stop streaming | +| `/api/session` | DELETE | Kill session subprocess | + --- ## State Management @@ -530,6 +620,14 @@ PathView uses JSON-based file formats for saving and sharing: The `.pvm` format is fully documented in [**docs/pvm-spec.md**](docs/pvm-spec.md). Use this spec if you are building tools that read or write PathView models (e.g., code generators, importers). A reference Python code generator is available at `scripts/pvm2py.py`. +### Specification Documents + +| Document | Audience | +|----------|----------| +| [**docs/pvm-spec.md**](docs/pvm-spec.md) | Building tools that read/write `.pvm` model files | +| [**docs/backend-protocol-spec.md**](docs/backend-protocol-spec.md) | Implementing a new execution backend (remote server, cloud worker, etc.) | +| [**docs/toolbox-spec.md**](docs/toolbox-spec.md) | Creating a third-party toolbox package for PathView | + ### Export Options - **File > Save** - Save complete model as `.pvm` @@ -583,8 +681,10 @@ https://view.pathsim.org/?modelgh=pathsim/pathview/static/examples/feedback-syst | Script | Purpose | |--------|---------| -| `npm run dev` | Start development server | -| `npm run build` | Production build | +| `npm run dev` | Start Vite development server | +| `npm run server` | Start Flask backend server (port 5000) | +| `npm run build` | Production build (GitHub Pages) | +| `npm run build:package` | Build pip package (frontend + wheel) | | `npm run preview` | Preview production build | | `npm run check` | TypeScript/Svelte type checking | | `npm run lint` | Run ESLint | @@ -665,7 +765,7 @@ Port labels show the name of each input/output port alongside the node. Toggle g 2. **Subsystems are nested graphs** - The Interface node inside a subsystem mirrors its parent's ports (inverted direction). -3. **No server required** - Everything runs client-side via Pyodide WebAssembly. +3. **No server required by default** - Everything runs client-side via Pyodide. The optional Flask backend enables server-side execution for packages with native dependencies. 4. **Registry pattern** - Nodes and events are registered centrally for extensibility. @@ -701,7 +801,9 @@ Port labels show the name of each input/output port alongside the node. Toggle g ## Deployment -PathView uses a dual deployment strategy with automatic versioning: +PathView has two deployment targets: + +### GitHub Pages (web) | Trigger | What happens | Deployed to | |---------|--------------|-------------| @@ -709,6 +811,13 @@ PathView uses a dual deployment strategy with automatic versioning: | Release published | Bump `package.json`, build, deploy | [view.pathsim.org/](https://view.pathsim.org/) | | Manual dispatch | Choose `dev` or `release` | Respective path | +### PyPI (pip package) + +| Trigger | What happens | Published to | +|---------|--------------|--------------| +| Release published | Build frontend + wheel, publish | [pypi.org/project/pathview](https://pypi.org/project/pathview/) | +| Manual dispatch | Choose `testpypi` or `pypi` | Respective index | + ### How it works 1. Both versions deploy to the `deployment` branch using GitHub Actions diff --git a/docs/backend-protocol-spec.md b/docs/backend-protocol-spec.md new file mode 100644 index 00000000..6a4df23e --- /dev/null +++ b/docs/backend-protocol-spec.md @@ -0,0 +1,1151 @@ +# PathView REPL Backend Protocol Specification + +**Version:** 1.0.0 +**Status:** Normative + +The PathView REPL backend protocol defines the contract that any Python execution backend must implement. It covers the TypeScript interface, the wire-level message protocol between main thread and worker, the HTTP API used by the Flask reference implementation, and the streaming semantics that enable live simulation. + +This document is the authoritative reference for anyone building a new backend (remote server, cloud worker, WebSocket bridge, etc.). + +--- + +## Table of Contents + +- [1. Overview](#1-overview) +- [2. Backend Interface](#2-backend-interface) + - [2.1 Lifecycle Methods](#21-lifecycle-methods) + - [2.2 State Methods](#22-state-methods) + - [2.3 Execution Methods](#23-execution-methods) + - [2.4 Streaming Methods](#24-streaming-methods) + - [2.5 Output Callbacks](#25-output-callbacks) +- [3. REPL Message Protocol](#3-repl-message-protocol) + - [3.1 Request Messages (main -> worker)](#31-request-messages-main---worker) + - [3.2 Response Messages (worker -> main)](#32-response-messages-worker---main) +- [4. Message Flows](#4-message-flows) + - [4.1 Init Flow](#41-init-flow) + - [4.2 Exec Flow](#42-exec-flow) + - [4.3 Eval Flow](#43-eval-flow) + - [4.4 Streaming Flow](#44-streaming-flow) + - [4.5 Code Injection During Streaming](#45-code-injection-during-streaming) + - [4.6 Stop Streaming](#46-stop-streaming) +- [5. Streaming Semantics](#5-streaming-semantics) +- [6. HTTP API (Flask Reference Implementation)](#6-http-api-flask-reference-implementation) + - [6.1 Route Summary](#61-route-summary) + - [6.2 Route Details](#62-route-details) +- [7. SSE Event Format](#7-sse-event-format) +- [8. State Management](#8-state-management) +- [9. Backend Registration](#9-backend-registration) +- [10. Worked Example](#10-worked-example) + - [10.1 Abstract Message Flow](#101-abstract-message-flow) + - [10.2 HTTP Equivalents](#102-http-equivalents) + +--- + +## 1. Overview + +PathView uses a modular backend system for Python execution. The `Backend` interface defines a transport-agnostic contract that decouples the UI from the execution environment. Two implementations currently exist: + +| Backend | Transport | Environment | +|---------|-----------|-------------| +| `PyodideBackend` | Web Worker `postMessage` | Browser (Pyodide WASM) | +| `FlaskBackend` | HTTP + SSE | Local/remote Flask server | + +The `Backend` interface is defined in `src/lib/pyodide/backend/types.ts`. Implementations are registered in `src/lib/pyodide/backend/registry.ts` and re-exported from `src/lib/pyodide/backend/index.ts`. + +This spec defines the protocol at two levels: + +1. **Abstract level** -- the TypeScript `Backend` interface and the `REPLRequest`/`REPLResponse` message types. +2. **Wire level** -- the HTTP routes and SSE event format used by HTTP-based backends. + +Any new backend must implement the abstract interface. HTTP-based backends should additionally follow the wire-level conventions documented in sections 6 and 7. + +--- + +## 2. Backend Interface + +```typescript +interface Backend { + // Lifecycle + init(): Promise; + terminate(): void; + + // State + getState(): BackendState; + subscribe(callback: (state: BackendState) => void): () => void; + isReady(): boolean; + isLoading(): boolean; + getError(): string | null; + + // Execution + exec(code: string, timeout?: number): Promise; + evaluate(expr: string, timeout?: number): Promise; + + // Streaming + startStreaming( + expr: string, + onData: (data: T) => void, + onDone: () => void, + onError: (error: Error) => void + ): void; + stopStreaming(): void; + isStreaming(): boolean; + execDuringStreaming(code: string): void; + + // Output + onStdout(callback: (value: string) => void): void; + onStderr(callback: (value: string) => void): void; +} + +interface BackendState { + initialized: boolean; + loading: boolean; + error: string | null; + progress: string; +} +``` + +### 2.1 Lifecycle Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `init` | `init(): Promise` | Initialize the backend. Load the runtime, connect to the server, install packages, etc. The promise resolves when the backend is ready to execute code. Must set `BackendState.loading = true` at the start and `initialized = true` on success. Called once at application startup. Idempotent -- calling it when already initialized or loading is a no-op. | +| `terminate` | `terminate(): void` | Tear down the backend and release all resources. Reject pending requests, abort active streams, destroy workers or connections. Must call `backendState.reset()` to return state to initial values. | + +### 2.2 State Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `getState` | `getState(): BackendState` | Return a snapshot of the current backend state. | +| `subscribe` | `subscribe(callback: (state: BackendState) => void): () => void` | Subscribe to state changes. Returns an unsubscribe function. Delegates to the shared `backendState` Svelte store. | +| `isReady` | `isReady(): boolean` | Return `true` if `initialized` is `true`. Shorthand for `getState().initialized`. | +| `isLoading` | `isLoading(): boolean` | Return `true` if the backend is currently initializing. Shorthand for `getState().loading`. | +| `getError` | `getError(): string \| null` | Return the current error message, or `null` if no error. Shorthand for `getState().error`. | + +### 2.3 Execution Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `exec` | `exec(code: string, timeout?: number): Promise` | Execute Python code with no return value. The promise resolves on success and rejects on error. The `timeout` parameter (milliseconds) is optional; implementations should default to a reasonable value. Called for code setup, imports, variable definitions, and simulation construction. | +| `evaluate` | `evaluate(expr: string, timeout?: number): Promise` | Evaluate a Python expression and return the result as a parsed JSON value. The backend must serialize the result to JSON on the Python side and deserialize it on the TypeScript side. Rejects if the expression errors or the result is not JSON-serializable. | + +### 2.4 Streaming Methods + +| Method | Signature | Description | +|--------|-----------|-------------| +| `startStreaming` | `startStreaming(expr: string, onData: (data: T) => void, onDone: () => void, onError: (error: Error) => void): void` | Begin an autonomous streaming loop. The backend repeatedly evaluates `expr`, calling `onData` with each parsed JSON result. The loop continues until the expression returns `{done: true}`, `stopStreaming()` is called, or an error occurs. `onDone` is always called when the loop exits (regardless of reason). `onError` is called if the main expression throws. If a stream is already active, it is stopped before the new one begins. | +| `stopStreaming` | `stopStreaming(): void` | Request the streaming loop to stop. The backend finishes the current step and then sends `stream-done`. This is a request, not an immediate abort -- `onDone` will still fire. | +| `isStreaming` | `isStreaming(): boolean` | Return `true` if a streaming loop is currently active. | +| `execDuringStreaming` | `execDuringStreaming(code: string): void` | Queue Python code to be executed between streaming steps. The code is drained and executed before the next evaluation of the stream expression. Used for runtime parameter changes and event injection. Errors in queued code are reported via stderr but do not stop the stream. No-op if no stream is active. | + +### 2.5 Output Callbacks + +| Method | Signature | Description | +|--------|-----------|-------------| +| `onStdout` | `onStdout(callback: (value: string) => void): void` | Register a callback for captured stdout output. Only one callback is active at a time (last registration wins). Called during `exec`, `evaluate`, and streaming whenever Python code writes to stdout. | +| `onStderr` | `onStderr(callback: (value: string) => void): void` | Register a callback for captured stderr output. Same semantics as `onStdout`. | + +--- + +## 3. REPL Message Protocol + +The REPL message protocol defines the typed messages exchanged between the main thread and the execution worker. For the `PyodideBackend`, these are `postMessage` payloads. For the `FlaskBackend`, they are mapped onto HTTP request/response bodies and SSE events. The types are defined in `src/lib/pyodide/backend/types.ts`. + +### 3.1 Request Messages (main -> worker) + +#### `init` + +Initialize the Python runtime and install packages. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"init"` | yes | Message type discriminant. | + +The Pyodide worker reads its package list from the compiled-in `PYTHON_PACKAGES` constant. HTTP-based backends receive the package list in the request body (see [Section 6](#6-http-api-flask-reference-implementation)). + +#### `exec` + +Execute Python code (no return value expected). + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"exec"` | yes | Message type discriminant. | +| `id` | `string` | yes | Unique request ID for correlating the response. Convention: `"repl_N"`. | +| `code` | `string` | yes | Python code to execute. | + +#### `eval` + +Evaluate a Python expression and return the JSON-serialized result. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"eval"` | yes | Message type discriminant. | +| `id` | `string` | yes | Unique request ID. | +| `expr` | `string` | yes | Python expression to evaluate. The result must be JSON-serializable. | + +#### `stream-start` + +Begin an autonomous streaming loop. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stream-start"` | yes | Message type discriminant. | +| `id` | `string` | yes | Unique stream ID. All `stream-data` and `stream-done` responses reference this ID. | +| `expr` | `string` | yes | Python expression to evaluate each iteration. Should return `{done: bool, result: any}`. | + +#### `stream-stop` + +Request the streaming loop to stop. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stream-stop"` | yes | Message type discriminant. | + +No additional fields. The worker finishes the current iteration and then sends `stream-done`. + +#### `stream-exec` + +Queue code for execution between streaming steps. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stream-exec"` | yes | Message type discriminant. | +| `code` | `string` | yes | Python code to queue. Executed before the next evaluation of the stream expression. | + +### 3.2 Response Messages (worker -> main) + +#### `ready` + +Sent after successful initialization. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"ready"` | yes | Message type discriminant. | + +#### `ok` + +Sent after successful `exec` completion. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"ok"` | yes | Message type discriminant. | +| `id` | `string` | yes | The request ID from the originating `exec` request. | + +#### `value` + +Sent after successful `eval` completion. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"value"` | yes | Message type discriminant. | +| `id` | `string` | yes | The request ID from the originating `eval` request. | +| `value` | `string` | yes | JSON-serialized result of the evaluated expression. The main thread parses this with `JSON.parse()`. | + +#### `error` + +Sent when any operation fails. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"error"` | yes | Message type discriminant. | +| `id` | `string` | no | The request ID of the failed operation. May be absent for global errors (e.g., init failures). | +| `error` | `string` | yes | Human-readable error message. | +| `traceback` | `string` | no | Python traceback string, if available. | + +#### `stdout` + +Captured standard output from Python execution. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stdout"` | yes | Message type discriminant. | +| `value` | `string` | yes | The captured output text. | + +#### `stderr` + +Captured standard error from Python execution. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stderr"` | yes | Message type discriminant. | +| `value` | `string` | yes | The captured error text. | + +#### `progress` + +Loading progress updates during initialization. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"progress"` | yes | Message type discriminant. | +| `value` | `string` | yes | Human-readable progress message (e.g., `"Installing pathsim..."`). | + +#### `stream-data` + +A single result from the streaming loop. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stream-data"` | yes | Message type discriminant. | +| `id` | `string` | yes | The stream ID from the originating `stream-start` request. | +| `value` | `string` | yes | JSON-serialized step result. Convention: `{"done": false, "result": {...}}`. | + +#### `stream-done` + +The streaming loop has exited. + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"stream-done"` | yes | Message type discriminant. | +| `id` | `string` | yes | The stream ID from the originating `stream-start` request. | + +This message is **always** sent when the loop exits, whether it completed naturally (`done: true`), was stopped via `stream-stop`, or errored. + +--- + +## 4. Message Flows + +### 4.1 Init Flow + +``` +main worker + | | + |──── init ──────────────────────>| + | | (load Pyodide / connect) + |<──── progress("Loading...") ────| + |<──── progress("Installing...") ─| + |<──── stdout("pathsim loaded") ──| + |<──── progress("Installing...") ─| + |<──── ready ─────────────────────| + | | +``` + +The worker may send zero or more `progress`, `stdout`, and `stderr` messages during initialization. The sequence ends with exactly one `ready` or `error`. + +### 4.2 Exec Flow + +``` +main worker + | | + |──── exec {id, code} ──────────>| + | | (execute Python code) + |<──── stdout("...") ────────────| (zero or more) + |<──── stderr("...") ────────────| (zero or more) + |<──── ok {id} ──────────────────| (success) + | | + | ── OR on failure ── | + | | + |<──── error {id, error, tb?} ───| + | | +``` + +### 4.3 Eval Flow + +``` +main worker + | | + |──── eval {id, expr} ──────────>| + | | (evaluate expression) + |<──── stdout("...") ────────────| (zero or more) + |<──── value {id, value} ────────| (success, value is JSON string) + | | + | ── OR on failure ── | + | | + |<──── error {id, error, tb?} ───| + | | +``` + +### 4.4 Streaming Flow + +``` +main worker + | | + |── stream-start {id, expr} ────>| + | | (enter streaming loop) + |<── stream-data {id, value} ────| \ + |<── stream-data {id, value} ────| } repeated until done + |<── stream-data {id, value} ────| / + | | (expr returns {done: true}) + |<── stream-done {id} ───────────| + | | +``` + +### 4.5 Code Injection During Streaming + +``` +main worker + | | + |── stream-start {id, expr} ────>| + |<── stream-data {id, value} ────| + |<── stream-data {id, value} ────| + | | + |── stream-exec {code} ─────────>| (queued) + | | + | | [drain queue: exec code] + | | [eval expr] + |<── stream-data {id, value} ────| + |<── stream-data {id, value} ────| + |<── stream-done {id} ───────────| + | | +``` + +If the queued code errors, the worker sends a `stderr` message but continues the streaming loop: + +``` + |── stream-exec {code} ─────────>| + | | [exec code → error] + |<── stderr("Stream exec error") | + | | [eval expr → continues] + |<── stream-data {id, value} ────| +``` + +### 4.6 Stop Streaming + +``` +main worker + | | + |── stream-start {id, expr} ────>| + |<── stream-data {id, value} ────| + |<── stream-data {id, value} ────| + | | + |── stream-stop ─────────────────>| + | | (finish current step) + |<── stream-data {id, value} ────| (optional: final partial result) + |<── stream-done {id} ───────────| (always sent) + | | +``` + +The worker does not abort the currently executing Python step. It sets a flag and exits the loop after the current step completes. + +--- + +## 5. Streaming Semantics + +The streaming loop is the core mechanism for live simulation. It runs autonomously on the worker side after receiving `stream-start`. + +### Loop Algorithm + +``` +function runStreamingLoop(id, expr): + active = true + codeQueue = [] + + try: + while active: + // 1. Drain and execute queued code + while codeQueue is not empty: + code = codeQueue.dequeue() + try: + exec(code) + except error: + send stderr("Stream exec error: " + error) + + // 2. Evaluate the stream expression + result = eval(expr) // JSON: {done: bool, result: any} + + // 3. Check for stop (set by stream-stop handler) + if not active: + if result is not done and has data: + send stream-data(id, result) + break + + // 4. Check for natural completion + if result.done: + break + + // 5. Send result and continue + send stream-data(id, result) + + except error: + send error(id, error, traceback?) + finally: + active = false + send stream-done(id) // ALWAYS sent +``` + +### Key Rules + +1. **Code queue drain.** On each iteration, ALL queued code snippets are drained and executed before the expression is evaluated. This ensures parameter changes take effect on the next step. + +2. **Queue isolation.** Errors in queued code are reported via `stderr` but do **not** stop the streaming loop. The loop continues with the next evaluation. + +3. **Expression errors are fatal.** If the main stream expression throws, the loop exits and sends `error` followed by `stream-done`. + +4. **`stream-done` is always sent.** Regardless of whether the loop completed naturally (`done: true`), was stopped (`stream-stop`), or errored, the `stream-done` message is always the last message for that stream ID. + +5. **Result convention.** The stream expression should return a JSON-serializable object with the shape `{done: boolean, result: any}`. When `done` is `true`, the loop exits without sending the final result as `stream-data`. + +6. **Stop semantics.** `stream-stop` sets a flag. The worker does not preempt a running Python evaluation. The flag is checked after the current step completes. + +7. **Single stream.** Only one stream can be active at a time. Starting a new stream while one is active will stop the existing stream first. + +--- + +## 6. HTTP API (Flask Reference Implementation) + +The Flask backend (`src/lib/pyodide/backend/flask/backend.ts`) maps the abstract protocol onto HTTP requests. All routes except `/api/health` require the `X-Session-ID` header to identify the browser session. Each session gets an isolated Python process on the server. + +### 6.1 Route Summary + +| Route | Method | Description | +|-------|--------|-------------| +| `/api/health` | GET | Server health check | +| `/api/init` | POST | Initialize Python worker with packages | +| `/api/exec` | POST | Execute Python code | +| `/api/eval` | POST | Evaluate Python expression | +| `/api/stream` | POST | Start streaming (returns SSE stream) | +| `/api/stream/exec` | POST | Queue code during streaming | +| `/api/stream/stop` | POST | Stop active stream | +| `/api/session` | DELETE | Terminate session and destroy worker | + +### 6.2 Route Details + +#### GET /api/health + +Server health check. No authentication required. + +**Request:** +``` +GET /api/health +``` + +**Response:** +```json +{ + "status": "ok" +} +``` + +--- + +#### POST /api/init + +Initialize the Python runtime and install packages. + +**Request:** +``` +POST /api/init +Content-Type: application/json +X-Session-ID: +``` + +```json +{ + "packages": [ + { + "pip": "pathsim==0.16.5", + "import": "pathsim", + "required": true, + "pre": true + }, + { + "pip": "pathsim-chem>=0.2rc2", + "import": "pathsim_chem", + "required": false, + "pre": true + } + ] +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `packages` | `PackageConfig[]` | no | Packages to install. If omitted, only the base runtime is initialized. | +| `packages[].pip` | `string` | yes | pip install specifier (e.g., `"pathsim==0.16.5"`). | +| `packages[].import` | `string` | yes | Python import name for verification (e.g., `"pathsim"`). | +| `packages[].required` | `boolean` | yes | If `true`, failure to install this package is a fatal error. | +| `packages[].pre` | `boolean` | yes | If `true`, pass `pre=True` to pip (allow pre-release versions). | + +**Success Response:** +```json +{ + "type": "ready", + "messages": [ + { "type": "progress", "value": "Installing pathsim..." }, + { "type": "stdout", "value": "pathsim 0.16.5 loaded successfully" }, + { "type": "progress", "value": "Installing pathsim_chem..." }, + { "type": "stdout", "value": "pathsim_chem 0.2rc3.dev1 loaded successfully" } + ] +} +``` + +The `messages` array contains all `progress`, `stdout`, and `stderr` messages that were generated during initialization, in order. + +**Error Response:** +```json +{ + "type": "error", + "error": "Failed to install required package pathsim==0.16.5: ..." +} +``` + +--- + +#### POST /api/exec + +Execute Python code with no return value. + +**Request:** +``` +POST /api/exec +Content-Type: application/json +X-Session-ID: +``` + +```json +{ + "id": "repl_1", + "code": "x = 42\nprint('hello')" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `id` | `string` | yes | Request ID for correlation. | +| `code` | `string` | yes | Python code to execute. | + +**Success Response:** +```json +{ + "type": "ok", + "id": "repl_1", + "stdout": "hello", + "stderr": "" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"ok"` | yes | Success discriminant. | +| `id` | `string` | yes | Echoed request ID. | +| `stdout` | `string` | no | Captured stdout output during execution. | +| `stderr` | `string` | no | Captured stderr output during execution. | + +**Error Response:** +```json +{ + "type": "error", + "id": "repl_1", + "error": "NameError: name 'y' is not defined", + "traceback": "Traceback (most recent call last):\n ...", + "stdout": "", + "stderr": "" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"error"` | yes | Error discriminant. | +| `id` | `string` | yes | Echoed request ID. | +| `error` | `string` | yes | Error message. | +| `traceback` | `string` | no | Python traceback. | +| `stdout` | `string` | no | Any stdout captured before the error. | +| `stderr` | `string` | no | Any stderr captured before the error. | + +--- + +#### POST /api/eval + +Evaluate a Python expression and return the JSON-serialized result. + +**Request:** +``` +POST /api/eval +Content-Type: application/json +X-Session-ID: +``` + +```json +{ + "id": "repl_2", + "expr": "json.dumps({'x': x, 'y': [1,2,3]})" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `id` | `string` | yes | Request ID. | +| `expr` | `string` | yes | Python expression to evaluate. Result must be JSON-serializable. | + +**Success Response:** +```json +{ + "type": "value", + "id": "repl_2", + "value": "{\"x\": 42, \"y\": [1, 2, 3]}", + "stdout": "", + "stderr": "" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `type` | `"value"` | yes | Success discriminant. | +| `id` | `string` | yes | Echoed request ID. | +| `value` | `string` | yes | JSON-serialized result. The client parses this with `JSON.parse()`. | +| `stdout` | `string` | no | Captured stdout. | +| `stderr` | `string` | no | Captured stderr. | + +**Error Response:** Same shape as exec error response. + +--- + +#### POST /api/stream + +Start a streaming loop. Returns a Server-Sent Events stream. + +**Request:** +``` +POST /api/stream +Content-Type: application/json +X-Session-ID: +``` + +```json +{ + "id": "repl_3", + "expr": "step_simulation()" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `id` | `string` | yes | Stream ID. | +| `expr` | `string` | yes | Python expression to evaluate each iteration. Should return `{done: bool, result: any}`. | + +**Response:** `Content-Type: text/event-stream` (SSE). See [Section 7](#7-sse-event-format). + +--- + +#### POST /api/stream/exec + +Queue code for execution between streaming steps. + +**Request:** +``` +POST /api/stream/exec +Content-Type: application/json +X-Session-ID: +``` + +```json +{ + "code": "controller.set_gain(2.0)" +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `code` | `string` | yes | Python code to queue. | + +**Response:** +```json +{ + "status": "queued" +} +``` + +--- + +#### POST /api/stream/stop + +Stop the active streaming loop. + +**Request:** +``` +POST /api/stream/stop +Content-Type: application/json +X-Session-ID: +``` + +```json +{} +``` + +**Response:** +```json +{ + "status": "stopped" +} +``` + +--- + +#### DELETE /api/session + +Terminate the session and destroy the server-side Python worker process. + +**Request:** +``` +DELETE /api/session +X-Session-ID: +``` + +**Response:** +```json +{ + "status": "terminated" +} +``` + +--- + +## 7. SSE Event Format + +The `/api/stream` endpoint returns a standard Server-Sent Events stream. Each event has an `event` field (the event type) and a `data` field (JSON payload). Events are separated by double newlines. + +### Event Types + +#### `data` -- Streaming step result + +``` +event: data +data: {"done":false,"result":{"t":0.5,"values":[1.2,3.4]}} + +``` + +The `data` field is a JSON string matching the return value of the stream expression. + +#### `stdout` -- Captured standard output + +``` +event: stdout +data: "Step 50 complete" + +``` + +The `data` field is a JSON-encoded string. + +#### `stderr` -- Captured standard error + +``` +event: stderr +data: "Stream exec error: NameError: name 'foo' is not defined" + +``` + +The `data` field is a JSON-encoded string. + +#### `done` -- Stream completed + +``` +event: done +data: {} + +``` + +Sent when the streaming loop exits (any reason). This is always the last event in the stream. The SSE connection closes after this event. + +#### `error` -- Stream error + +``` +event: error +data: {"error":"ZeroDivisionError: division by zero","traceback":"Traceback ..."} + +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `error` | `string` | yes | Error message. | +| `traceback` | `string` | no | Python traceback. | + +Sent when the main stream expression throws. The `done` event is NOT sent after `error` in the SSE stream (the error event terminates the stream). The client-side `FlaskBackend` maps this to an `onError` callback. + +### Complete SSE Example + +``` +event: data +data: {"done":false,"result":{"t":0.0,"values":[0.0]}} + +event: stdout +data: "Simulation step 1" + +event: data +data: {"done":false,"result":{"t":0.5,"values":[0.25]}} + +event: data +data: {"done":false,"result":{"t":1.0,"values":[1.0]}} + +event: done +data: {} + +``` + +--- + +## 8. State Management + +All backends share a single `backendState` Svelte writable store defined in `src/lib/pyodide/backend/state.ts`. This store drives the UI loading indicators, error displays, and ready checks. + +```typescript +const initialState: BackendState = { + initialized: false, + loading: false, + error: null, + progress: '' +}; +``` + +### State Transitions + +Backends must follow this state machine: + +``` + init() called + [idle] ──────────────────────────> [loading] + initialized: false initialized: false + loading: false loading: true + error: null error: null + progress: '' progress: "Loading..." + | + ┌────────────────┼────────────────┐ + | | | + (progress) (success) (failure) + | | | + [loading] [ready] [error] + progress: initialized: true loading: false + "Installing..." loading: false error: "msg" + progress: "Ready" + | + terminate() + | + [idle] (backendState.reset()) +``` + +### Rules for Implementations + +1. **Set `loading = true` and `error = null`** at the start of `init()`. +2. **Update `progress`** during loading to give the user feedback (e.g., `"Loading Pyodide..."`, `"Installing pathsim..."`). +3. **Set `initialized = true` and `loading = false`** on successful initialization. +4. **Set `loading = false` and `error = `** on initialization failure. +5. **Call `backendState.reset()`** in `terminate()` to return to the idle state. +6. **Set `error`** when execution errors occur that affect the overall backend health (not for per-request errors, which are returned via the promise rejection or error callback). + +--- + +## 9. Backend Registration + +To add a new backend type to PathView: + +### Step 1: Implement the Backend Interface + +Create a new file (e.g., `src/lib/pyodide/backend/remote/backend.ts`): + +```typescript +import type { Backend, BackendState } from '../types'; +import { backendState } from '../state'; + +export class RemoteBackend implements Backend { + // ... implement all methods from the Backend interface +} +``` + +### Step 2: Add Type to BackendType Union + +In `src/lib/pyodide/backend/registry.ts`: + +```typescript +export type BackendType = 'pyodide' | 'flask' | 'remote'; +``` + +### Step 3: Add Case to createBackend() + +In `src/lib/pyodide/backend/registry.ts`: + +```typescript +export function createBackend(type: BackendType): Backend { + switch (type) { + case 'pyodide': + return new PyodideBackend(); + case 'flask': + return new FlaskBackend(flaskHost); + case 'remote': + return new RemoteBackend(remoteConfig); + default: + throw new Error(`Unknown backend type: ${type}`); + } +} +``` + +### Step 4: Re-export from index.ts + +In `src/lib/pyodide/backend/index.ts`: + +```typescript +export { RemoteBackend } from './remote/backend'; +``` + +The backend is now available via `switchBackend('remote')` or the `?backend=remote` URL parameter (if `initBackendFromUrl()` is updated to handle it). + +--- + +## 10. Worked Example + +This section traces a complete session: initialization with packages, code execution, expression evaluation, then a streaming simulation with code injection and stop. Both the abstract message protocol and the HTTP equivalents are shown. + +### 10.1 Abstract Message Flow + +``` +=== INITIALIZATION === + +main → worker: { type: "init" } +worker → main: { type: "progress", value: "Loading Pyodide..." } +worker → main: { type: "progress", value: "Installing dependencies..." } +worker → main: { type: "progress", value: "Installing pathsim..." } +worker → main: { type: "stdout", value: "pathsim 0.16.5 loaded successfully" } +worker → main: { type: "progress", value: "Installing pathsim_chem..." } +worker → main: { type: "stdout", value: "pathsim_chem 0.2rc3.dev1 loaded successfully" } +worker → main: { type: "ready" } + +=== EXEC: Set up simulation === + +main → worker: { type: "exec", id: "repl_1", code: "import json\nfrom pathsim import Simulation, Connection\nfrom pathsim.blocks import Integrator, Constant, Scope\nfrom pathsim.solvers import SSPRK22\n\nconstant = Constant(value=1.0)\nintegrator = Integrator(initial_value=0.0)\nscope = Scope()\nsim = Simulation(\n [constant, integrator, scope],\n [Connection(constant[0], integrator[0]),\n Connection(integrator[0], scope[0])],\n Solver=SSPRK22, dt=0.01\n)" } +worker → main: { type: "ok", id: "repl_1" } + +=== EXEC: Set up streaming generator === + +main → worker: { type: "exec", id: "repl_2", code: "sim_iter = sim.run_generator(duration=10, steps_per_yield=100)\ndef step_simulation():\n try:\n result = next(sim_iter)\n return {'done': False, 'result': result}\n except StopIteration:\n return {'done': True, 'result': None}" } +worker → main: { type: "ok", id: "repl_2" } + +=== EVAL: Check initial state === + +main → worker: { type: "eval", id: "repl_3", expr: "json.dumps({'t': 0, 'ready': True})" } +worker → main: { type: "value", id: "repl_3", value: "{\"t\": 0, \"ready\": true}" } + +=== STREAMING: Run simulation with live updates === + +main → worker: { type: "stream-start", id: "repl_4", expr: "json.dumps(step_simulation(), default=str)" } +worker → main: { type: "stream-data", id: "repl_4", value: "{\"done\":false,\"result\":{\"t\":1.0}}" } +worker → main: { type: "stream-data", id: "repl_4", value: "{\"done\":false,\"result\":{\"t\":2.0}}" } + +--- User changes a parameter at t=2.0 --- + +main → worker: { type: "stream-exec", code: "constant.set(value=2.0)" } + +--- Worker drains queue, applies change, then continues --- + +worker → main: { type: "stream-data", id: "repl_4", value: "{\"done\":false,\"result\":{\"t\":3.0}}" } +worker → main: { type: "stream-data", id: "repl_4", value: "{\"done\":false,\"result\":{\"t\":4.0}}" } + +--- User stops the simulation at t=4.0 --- + +main → worker: { type: "stream-stop" } + +--- Worker finishes current step --- + +worker → main: { type: "stream-data", id: "repl_4", value: "{\"done\":false,\"result\":{\"t\":5.0}}" } +worker → main: { type: "stream-done", id: "repl_4" } +``` + +### 10.2 HTTP Equivalents + +The same session expressed as HTTP requests (Flask backend): + +``` +=== INITIALIZATION === + +GET /api/health +→ 200 {"status": "ok"} + +POST /api/init +Headers: X-Session-ID: a1b2c3d4-... +Body: { + "packages": [ + {"pip": "pathsim==0.16.5", "import": "pathsim", "required": true, "pre": true}, + {"pip": "pathsim-chem>=0.2rc2", "import": "pathsim_chem", "required": false, "pre": true} + ] +} +→ 200 { + "type": "ready", + "messages": [ + {"type": "progress", "value": "Installing pathsim..."}, + {"type": "stdout", "value": "pathsim 0.16.5 loaded successfully"}, + {"type": "progress", "value": "Installing pathsim_chem..."}, + {"type": "stdout", "value": "pathsim_chem 0.2rc3.dev1 loaded successfully"} + ] +} + +=== EXEC === + +POST /api/exec +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {"id": "repl_1", "code": "import json\nfrom pathsim import ..."} +→ 200 {"type": "ok", "id": "repl_1"} + +POST /api/exec +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {"id": "repl_2", "code": "sim_iter = sim.run_generator(...)..."} +→ 200 {"type": "ok", "id": "repl_2"} + +=== EVAL === + +POST /api/eval +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {"id": "repl_3", "expr": "json.dumps({'t': 0, 'ready': True})"} +→ 200 {"type": "value", "id": "repl_3", "value": "{\"t\": 0, \"ready\": true}"} + +=== STREAMING === + +POST /api/stream +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {"id": "repl_4", "expr": "json.dumps(step_simulation(), default=str)"} +→ 200 (SSE stream) + + event: data + data: {"done":false,"result":{"t":1.0}} + + event: data + data: {"done":false,"result":{"t":2.0}} + +--- concurrent request --- + +POST /api/stream/exec +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {"code": "constant.set(value=2.0)"} +→ 200 {"status": "queued"} + +--- SSE continues --- + + event: data + data: {"done":false,"result":{"t":3.0}} + + event: data + data: {"done":false,"result":{"t":4.0}} + +--- concurrent request --- + +POST /api/stream/stop +Headers: Content-Type: application/json, X-Session-ID: a1b2c3d4-... +Body: {} +→ 200 {"status": "stopped"} + +--- SSE concludes --- + + event: data + data: {"done":false,"result":{"t":5.0}} + + event: done + data: {} + +=== CLEANUP === + +DELETE /api/session +Headers: X-Session-ID: a1b2c3d4-... +→ 200 {"status": "terminated"} +``` + +--- + +## Notes for Backend Authors + +1. **JSON serialization.** All values crossing the boundary (eval results, stream data) must be valid JSON strings. The Python side should use `json.dumps()` with a `default` handler for non-serializable types. + +2. **ID correlation.** The `id` field in requests is echoed in responses. Clients use it to match responses to pending promises. Generate unique IDs (the convention is `"repl_N"` with an incrementing counter). + +3. **Timeout handling.** Clients set their own timeouts. Backends should not enforce timeouts unless they need to protect server resources. If a backend does enforce timeouts, it should send an `error` response with a clear timeout message. + +4. **Stdout/stderr capture.** Backends must capture Python's stdout and stderr and forward them as `stdout`/`stderr` messages. For Web Worker backends, this is done via Pyodide's `setStdout`/`setStderr`. For HTTP backends, captured output is included in the response body or sent as SSE events during streaming. + +5. **Session isolation.** HTTP-based backends must isolate sessions. Each `X-Session-ID` maps to a separate Python process or namespace. Leaking state between sessions is a correctness and security issue. + +6. **Idempotent init.** Calling `init()` when already initialized or loading should be a no-op, not an error. + +7. **Graceful terminate.** `terminate()` must clean up all resources: reject pending promises, abort streams, kill workers/processes, and reset state. It should not throw. diff --git a/docs/toolbox-spec.md b/docs/toolbox-spec.md new file mode 100644 index 00000000..891da7e4 --- /dev/null +++ b/docs/toolbox-spec.md @@ -0,0 +1,699 @@ +# PathView Toolbox Integration Specification + +**Version:** 1.0.0 + +A toolbox is a Python package that provides computational blocks (and optionally events) for the PathView visual simulation environment. Toolboxes extend PathView with new block types that appear in the Block Library panel. + +This document is the authoritative reference for third-party developers creating toolbox packages and integrating them into PathView. + +Existing toolboxes: `pathsim` (core simulation blocks), `pathsim-chem` (chemical engineering blocks). + +--- + +## Table of Contents + +- [1. Overview](#1-overview) +- [2. Python Package Requirements](#2-python-package-requirements) + - [2.1 Block Classes](#21-block-classes) + - [2.2 Event Classes](#22-event-classes) +- [3. Toolbox Config Directory](#3-toolbox-config-directory) +- [4. blocks.json Schema](#4-blocksjson-schema) +- [5. events.json Schema](#5-eventsjson-schema) +- [6. requirements-pyodide.txt](#6-requirements-pyodidetxt) +- [7. Extraction Pipeline](#7-extraction-pipeline) + - [7.1 Discovery](#71-discovery) + - [7.2 Block Extraction Flow](#72-block-extraction-flow) + - [7.3 Event Extraction Flow](#73-event-extraction-flow) + - [7.4 Dependency Extraction Flow](#74-dependency-extraction-flow) +- [8. Generated Output](#8-generated-output) + - [8.1 blocks.ts](#81-blocksts) + - [8.2 events.ts](#82-eventsts) + - [8.3 dependencies.ts](#83-dependenciests) +- [9. Block Metadata Contract (Block.info())](#9-block-metadata-contract-blockinfo) + - [9.1 Port Label Semantics](#91-port-label-semantics) + - [9.2 Parameter Type Inference](#92-parameter-type-inference) +- [10. Runtime Package Installation](#10-runtime-package-installation) +- [11. Step-by-Step Walkthrough](#11-step-by-step-walkthrough) +- [12. UI Configuration (Optional)](#12-ui-configuration-optional) + +--- + +## 1. Overview + +Each toolbox provides a set of block types (and optionally event types) that PathView discovers, extracts metadata from, and presents in the Block Library panel. The integration requires three things: + +1. **A Python package** with blocks that implement the `Block.info()` classmethod (and optionally an `events` submodule). +2. **A config directory** at `scripts/config//` containing `blocks.json` (and optionally `events.json`). +3. **An entry** in `scripts/config/requirements-pyodide.txt` so both the Pyodide and Flask backends install the package at runtime. + +The extraction pipeline (`npm run extract`) reads the config files, imports the Python packages, calls `Block.info()` on each class, and generates TypeScript source files that PathView consumes at build time. No manual registration beyond these three steps is needed. + +--- + +## 2. Python Package Requirements + +### 2.1 Block Classes + +The toolbox Python package must have an importable module containing block classes. For example, a package named `pathsim-controls` (installed as `pathsim_controls`) would expose blocks via `pathsim_controls.blocks`. + +Each block class must implement the `info()` classmethod returning a dict with the following keys: + +```python +@classmethod +def info(cls): + return { + "input_port_labels": {"x": 0, "y": 1}, # or None or {} + "output_port_labels": {"out": 0}, # or None or {} + "parameters": { + "gain": {"default": 1.0}, + "mode": {"default": "linear"} + }, + "description": "A proportional controller block." + } +``` + +| Key | Type | Description | +|-----|------|-------------| +| `input_port_labels` | `dict`, `None`, or `{}` | Defines input port names and indices. See [Port Label Semantics](#91-port-label-semantics). | +| `output_port_labels` | `dict`, `None`, or `{}` | Defines output port names and indices. See [Port Label Semantics](#91-port-label-semantics). | +| `parameters` | `dict` | Map of parameter names to dicts containing at minimum a `"default"` key. | +| `description` | `str` | RST-formatted docstring. The first line/sentence is used as the short description. | + +If a block does not implement `info()`, the extractor falls back to `__init__` signature introspection, but this is less reliable. All new toolbox blocks should implement `info()`. + +### 2.2 Event Classes + +Event classes live in a separate submodule (e.g., `pathsim_controls.events`). Events do not use an `info()` classmethod. Instead, the extractor inspects the `__init__` signature to discover parameters: + +```python +class ThresholdEvent: + """Triggers when a signal crosses a threshold value.""" + + def __init__(self, func_evt=None, func_act=None, threshold=0.0, tolerance=1e-4): + ... +``` + +Parameter names, default values, and the class docstring are extracted automatically. + +--- + +## 3. Toolbox Config Directory + +Each toolbox has a config directory at: + +``` +scripts/config// +``` + +The directory name should match the pip package name (e.g., `pathsim-chem`, `pathsim-controls`). + +Required contents: + +| File | Required | Description | +|------|----------|-------------| +| `blocks.json` | Yes | Block categories and class names to extract | +| `events.json` | No | Event class names to extract | + +Example directory structure: + +``` +scripts/config/ + schemas/ # JSON schemas (shared, do not modify) + blocks.schema.json + events.schema.json + pathsim/ # Core toolbox + blocks.json + events.json + simulation.json + pathsim-chem/ # Chemical engineering toolbox + blocks.json + pathsim-controls/ # Your new toolbox + blocks.json + events.json +``` + +--- + +## 4. blocks.json Schema + +A `blocks.json` file declares which block classes to extract from a toolbox and how to organize them into categories. + +```json +{ + "$schema": "../schemas/blocks.schema.json", + "toolbox": "pathsim-controls", + "importPath": "pathsim_controls.blocks", + "categories": { + "Controls": ["PIDController", "StateEstimator"] + }, + "extraDocstrings": [] +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `$schema` | string | No | JSON Schema reference for editor validation. Should be `"../schemas/blocks.schema.json"`. | +| `toolbox` | string | Yes | Toolbox identifier. Must match the config directory name. | +| `importPath` | string | Yes | Python import path to the blocks module (e.g., `"pathsim_controls.blocks"`). | +| `categories` | object | Yes | Map of category names to arrays of block entries. Category names appear as section headers in the Block Library panel. | +| `extraDocstrings` | string[] | No | Additional classes to extract docstrings from (not blocks themselves). Used by the core toolbox for `Subsystem` and `Interface`. | + +**Block entries** within each category array can be either: + +- **A string** -- the class name, imported from `importPath`: + ```json + "PIDController" + ``` +- **An object** -- for classes that live in a different module than `importPath`: + ```json + {"class": "PIDController", "import": "pathsim_controls.advanced"} + ``` + +**Real-world example** (`pathsim-chem/blocks.json`): + +```json +{ + "$schema": "../schemas/blocks.schema.json", + "toolbox": "pathsim-chem", + "importPath": "pathsim_chem.tritium", + "categories": { + "Chemical": ["Process", "Bubbler4", "Splitter", "GLC"] + } +} +``` + +--- + +## 5. events.json Schema + +An `events.json` file declares which event classes to extract from a toolbox. + +```json +{ + "$schema": "../schemas/events.schema.json", + "toolbox": "pathsim-controls", + "importPath": "pathsim_controls.events", + "events": ["ThresholdEvent", "TimerEvent"] +} +``` + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `$schema` | string | No | JSON Schema reference for editor validation. Should be `"../schemas/events.schema.json"`. | +| `toolbox` | string | Yes | Toolbox identifier. Must match the config directory name. | +| `importPath` | string | Yes | Python import path to the events module (e.g., `"pathsim_controls.events"`). | +| `events` | string[] | Yes | List of event class names to extract from the module. | + +**Real-world example** (`pathsim/events.json`): + +```json +{ + "$schema": "../schemas/events.schema.json", + "toolbox": "pathsim", + "importPath": "pathsim.events", + "events": [ + "ZeroCrossing", + "ZeroCrossingUp", + "ZeroCrossingDown", + "Schedule", + "ScheduleList", + "Condition" + ] +} +``` + +--- + +## 6. requirements-pyodide.txt + +The file `scripts/config/requirements-pyodide.txt` is the single source of truth for runtime Python dependencies. Both the Pyodide web worker and the Flask backend install packages from this file. + +**Syntax:** + +``` +--pre +pathsim +pathsim-chem>=0.2rc2 # optional +pathsim-controls # optional +``` + +| Syntax Element | Description | +|----------------|-------------| +| `--pre` | Global flag. Applies to all packages below it. Allows pre-release versions (e.g., `rc`, `beta`). | +| `pathsim` | A required package. If installation fails, app startup is blocked. | +| `pathsim-chem>=0.2rc2 # optional` | An optional package with a version specifier. The `# optional` comment sets the package's `required` field to `false` -- installation failure will not block app startup. | +| `>=`, `==`, `~=`, `<`, `<=`, `>` | Standard pip version specifiers. All are supported. | + +**How the file is parsed** (from `ConfigLoader.load_requirements()`): + +1. Lines starting with `#` are ignored (pure comments). +2. The `--pre` flag sets `pre: true` for all subsequent packages. +3. For each package line, the pip spec is the text before any `#` comment. +4. If the comment contains `# optional` (case-insensitive), `required` is set to `false`. +5. The Python import name is derived by stripping version specifiers and replacing `-` with `_` (e.g., `pathsim-chem` becomes `pathsim_chem`). + +--- + +## 7. Extraction Pipeline + +The extraction is run via: + +```bash +npm run extract # Extract all (blocks, events, simulation, dependencies) +python scripts/extract.py # Same, invoked directly +``` + +Selective extraction is also supported: + +```bash +python scripts/extract.py --blocks # Blocks only +python scripts/extract.py --events # Events only +python scripts/extract.py --deps # Dependencies only +python scripts/extract.py --registry # JSON registry only (for pvm2py) +python scripts/extract.py --validate # Validate config files only +``` + +### 7.1 Discovery + +The `discover_toolboxes()` method scans `scripts/config/` for subdirectories that contain a `blocks.json` file (excluding the `schemas/` directory). Toolboxes are auto-discovered -- no manual registration step is needed beyond creating the config directory. + +### 7.2 Block Extraction Flow + +For each discovered toolbox: + +1. Load and validate `blocks.json`. +2. Import the Python module specified in `importPath`. +3. For each class name in each category: + a. Get the class object from the module via `getattr()`. + b. Call `cls.info()` to get the metadata dict. + c. Process `input_port_labels` and `output_port_labels` via `_process_port_labels()` -- converting the dict to a sorted list of names, `None` for variable ports, or `[]` for no ports. + d. For each parameter, infer the type from the default value (see [Parameter Type Inference](#92-parameter-type-inference)). + e. Extract parameter descriptions from the RST docstring. + f. Convert the full docstring to HTML using `docutils` (if available). +4. If `info()` is not available, fall back to `__init__` signature introspection. +5. Process `extraDocstrings` classes (docstring-only extraction, no parameter/port extraction). +6. Generate `src/lib/nodes/generated/blocks.ts`. + +### 7.3 Event Extraction Flow + +For each discovered toolbox that has an `events.json`: + +1. Load and validate `events.json`. +2. Import the Python module specified in `importPath`. +3. For each event class name: + a. Get the class object from the module. + b. Inspect the `__init__` signature for parameters (skipping `self`). + c. Infer parameter types from names and default values. + d. Extract parameter descriptions from the class docstring. + e. Convert the full docstring to HTML. +4. Generate `src/lib/events/generated/events.ts`. + +### 7.4 Dependency Extraction Flow + +1. Parse `scripts/config/requirements-pyodide.txt` into a list of package configs. +2. For each package, import the module and read `__version__` to get the installed version. +3. Pin exact versions for release builds (dev versions keep the original spec). +4. Load `scripts/config/pyodide.json` for Pyodide runtime configuration. +5. Generate `src/lib/constants/dependencies.ts` containing the `PYTHON_PACKAGES` array. + +--- + +## 8. Generated Output + +The extraction pipeline generates three TypeScript files. These are auto-generated and should not be edited by hand. + +### 8.1 blocks.ts + +**Path:** `src/lib/nodes/generated/blocks.ts` + +```typescript +export interface ExtractedParam { + type: string; // "integer", "number", "string", "boolean", "callable", "array", "any" + default: string | null; + description: string; + min?: number; + max?: number; + options?: string[]; +} + +export interface ExtractedBlock { + blockClass: string; + description: string; + docstringHtml: string; + params: Record; + inputs: string[] | null; // null = variable, [] = none, [...] = fixed named + outputs: string[] | null; // null = variable, [] = none, [...] = fixed named +} + +export const extractedBlocks: Record = { ... }; + +export const blockConfig: Record = { + Sources: ["Constant", "Source", ...], + Controls: ["PIDController", "StateEstimator"], + ... +}; + +export const blockImportPaths: Record = { + "Constant": "pathsim.blocks", + "PIDController": "pathsim_controls.blocks", + ... +}; +``` + +The `blockConfig` object maps category names (used as Block Library section headers) to arrays of block class names. The `blockImportPaths` object maps each block class name to its Python import path. + +### 8.2 events.ts + +**Path:** `src/lib/events/generated/events.ts` + +```typescript +import type { EventTypeDefinition } from '../types'; + +export const extractedEvents: EventTypeDefinition[] = [ + { + type: "pathsim.events.ZeroCrossing", // fully qualified Python path + name: "ZeroCrossing", // display name + eventClass: "ZeroCrossing", // class name + description: "...", + docstringHtml: "...", + params: [ + { name: "func_evt", type: "callable", default: "None", description: "..." }, + { name: "func_act", type: "callable", default: "None", description: "..." }, + { name: "tolerance", type: "number", default: "0.0001", description: "..." } + ] + } +]; +``` + +The `type` field uses the fully qualified Python import path (e.g., `"pathsim.events.ZeroCrossing"`). This is used in `.pvm` files to reference event types and for code generation. + +### 8.3 dependencies.ts + +**Path:** `src/lib/constants/dependencies.ts` + +```typescript +export interface PackageConfig { + pip: string; // pip install spec (e.g., "pathsim==0.16.5") + pre: boolean; // whether to use --pre flag + required: boolean; // whether failure blocks startup + import: string; // Python import name (e.g., "pathsim_chem") +} + +export const PYTHON_PACKAGES: PackageConfig[] = [ + { + "pip": "pathsim==0.16.5", + "required": true, + "pre": true, + "import": "pathsim" + }, + { + "pip": "pathsim-chem>=0.2rc2", + "required": false, + "pre": true, + "import": "pathsim_chem" + } +]; +``` + +This file also exports `PYODIDE_VERSION`, `PYODIDE_CDN_URL`, `PYODIDE_PRELOAD`, `PATHVIEW_VERSION`, and `EXTRACTED_VERSIONS`. + +--- + +## 9. Block Metadata Contract (Block.info()) + +The `info()` classmethod is the primary interface between a toolbox's Python code and PathView's extraction pipeline. + +```python +@classmethod +def info(cls): + return { + "input_port_labels": {"x": 0, "y": 1}, + "output_port_labels": {"out": 0}, + "parameters": { + "gain": {"default": 1.0}, + "mode": {"default": "linear"} + }, + "description": "A proportional controller block." + } +``` + +### 9.1 Port Label Semantics + +The values of `input_port_labels` and `output_port_labels` have precise semantics that control both extraction and UI behavior: + +| Value | Meaning | Extracted As | UI Behavior | +|-------|---------|--------------|-------------| +| `None` | Variable/unlimited ports | `null` in TypeScript | Add/remove port buttons shown | +| `{}` | No ports of this direction | `[]` (empty array) | No ports rendered | +| `{"name": index, ...}` | Fixed labeled ports | `["name", ...]` sorted by index | Ports locked, names displayed | + +The extraction function `_process_port_labels()` converts dicts to sorted name lists: + +```python +{"x": 0, "y": 1} # → ["x", "y"] (sorted by index value) +{"out": 0} # → ["out"] +None # → None (variable ports) +{} # → [] (no ports) +``` + +### 9.2 Parameter Type Inference + +The extractor infers TypeScript-side parameter types from Python default values using these rules (evaluated in order): + +| Condition | Inferred Type | +|-----------|---------------| +| Name starts with `func_` or `func`, or default is callable | `"callable"` | +| Default is `True` or `False` | `"boolean"` | +| Default is `int` (and not `bool`) | `"integer"` | +| Default is `float` | `"number"` | +| Default is `str` | `"string"` | +| Default is `list`, `tuple`, or ndarray | `"array"` | +| Default is `None` or anything else | `"any"` | + +Note: `bool` is checked before `int` because in Python `isinstance(True, int)` is `True`. + +--- + +## 10. Runtime Package Installation + +The `PYTHON_PACKAGES` constant (generated from `requirements-pyodide.txt`) drives package installation in both backends: + +**Pyodide backend** (`src/lib/pyodide/backend/pyodide/worker.ts`): +```typescript +for (const pkg of PYTHON_PACKAGES) { + await pyodide.runPythonAsync(` + import micropip + await micropip.install('${pkg.pip}'${pkg.pre ? ', pre=True' : ''}) + `); +} +``` + +**Flask backend** (`src/lib/pyodide/backend/flask/backend.ts`): +```typescript +await fetch(`${baseUrl}/api/init`, { + method: 'POST', + body: JSON.stringify({ packages: PYTHON_PACKAGES }), +}); +``` + +Both backends respect the `required` flag. If a package has `required: false` (from the `# optional` comment in requirements), installation failure is caught and logged but does not block app startup. If `required: true`, a failed installation aborts initialization. + +--- + +## 11. Step-by-Step Walkthrough + +This section walks through adding a hypothetical `pathsim-controls` toolbox with two blocks (`PIDController`, `StateEstimator`) and one event (`ThresholdEvent`). + +### Step 1: Add to requirements + +Edit `scripts/config/requirements-pyodide.txt`: + +``` +--pre +pathsim +pathsim-chem>=0.2rc2 # optional +pathsim-controls # optional +``` + +The `# optional` comment means PathView will continue loading if this package is unavailable. + +### Step 2: Create config directory + +Create the directory `scripts/config/pathsim-controls/`. + +**`scripts/config/pathsim-controls/blocks.json`:** + +```json +{ + "$schema": "../schemas/blocks.schema.json", + "toolbox": "pathsim-controls", + "importPath": "pathsim_controls.blocks", + "categories": { + "Controls": ["PIDController", "StateEstimator"] + } +} +``` + +**`scripts/config/pathsim-controls/events.json`:** + +```json +{ + "$schema": "../schemas/events.schema.json", + "toolbox": "pathsim-controls", + "importPath": "pathsim_controls.events", + "events": ["ThresholdEvent"] +} +``` + +### Step 3: Ensure the Python package is installed + +The extraction script needs to import the package. Install it in your development environment: + +```bash +pip install pathsim-controls +``` + +Verify the blocks module is importable and `info()` works: + +```python +from pathsim_controls.blocks import PIDController +print(PIDController.info()) +``` + +### Step 4: Run the extraction + +```bash +npm run extract +``` + +Expected output: + +``` +PathSim Metadata Extractor +======================================== + +Extracting dependencies... + Pinned pathsim to version 0.16.5 + pathsim-controls ... + +Extracting blocks... + Processing toolbox: pathsim + Extracted Constant + ... + Processing toolbox: pathsim-controls + Extracted PIDController + Extracted StateEstimator +Generated: src/lib/nodes/generated/blocks.ts + +Extracting events... + Processing events from: pathsim + Extracted ZeroCrossing + ... + Processing events from: pathsim-controls + Extracted ThresholdEvent +Generated: src/lib/events/generated/events.ts + +Done! +``` + +### Step 5: Verify in dev server + +```bash +npm run dev +``` + +Open the app in a browser. The Block Library panel should now show a "Controls" section containing "PIDController" and "StateEstimator". The Events panel should list "ThresholdEvent". + +### Step 6: Build for production + +```bash +npm run build +``` + +The generated TypeScript files are compiled into the production bundle. The `PYTHON_PACKAGES` constant ensures runtime installation of `pathsim-controls` in both Pyodide and Flask backends. + +--- + +## 12. UI Configuration (Optional) + +After a toolbox's blocks are extracted and appear in the Block Library, you may want to customize their UI behavior. These configurations are in PathView's TypeScript source and are optional. + +### Port synchronization + +For blocks where each input has a corresponding output (parallel processing blocks), add the block class name to the `syncPortBlocks` set. This hides the output port controls and keeps output count in sync with input count. + +**File:** `src/lib/nodes/uiConfig.ts` + +```typescript +export const syncPortBlocks = new Set([ + 'Integrator', + 'Differentiator', + 'Delay', + // ... existing entries ... + 'PIDController', // add your block here +]); +``` + +### Port labels from parameters + +For blocks that derive port names from a parameter value (e.g., a list of channel labels), add an entry to `portLabelParams`. When the user changes the parameter, port names update automatically. + +**File:** `src/lib/nodes/uiConfig.ts` + +```typescript +export const portLabelParams: Record = { + Scope: { param: 'labels', direction: 'input' }, + Spectrum: { param: 'labels', direction: 'input' }, + Adder: { param: 'operations', direction: 'input', parser: parseOperationsString }, + // Add your block: + MyMuxBlock: { param: 'labels', direction: 'input' }, +}; +``` + +The `PortLabelConfig` interface: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `param` | string | Yes | Parameter name whose value determines port labels. | +| `direction` | `"input"` or `"output"` | Yes | Which port direction the labels apply to. | +| `parser` | function | No | Custom parser to convert the param value to a `string[]`. Default uses `parsePythonList`. | + +### Custom shapes + +Map your toolbox's categories to node shapes using the shape registry. Available built-in shapes: `pill`, `rect`, `circle`, `diamond`, `mixed`, `default`. + +**File:** `src/lib/nodes/shapes/registry.ts` + +```typescript +// Add to the categoryShapeMap or call setCategoryShape: +setCategoryShape('Controls', 'rect'); +``` + +The existing category-to-shape mapping: + +| Category | Shape | +|----------|-------| +| Sources | `pill` | +| Dynamic | `rect` | +| Algebraic | `rect` | +| Mixed | `mixed` | +| Recording | `pill` | +| Subsystem | `rect` | + +Categories not in the map use the `default` shape. + +--- + +## Notes for Toolbox Authors + +1. **`info()` is the contract.** Implement it on every block class. The extractor falls back to `__init__` introspection, but `info()` gives you explicit control over port definitions and parameter metadata. + +2. **Port labels must be consistent.** The index values in port label dicts must be zero-based and contiguous. The extractor sorts by index, so `{"y": 1, "x": 0}` correctly produces `["x", "y"]`. + +3. **Parameter defaults drive type inference.** If your parameter should be a `number` in the UI, make sure the default is a Python `float`, not `None`. Use `None` only when the type truly cannot be determined. + +4. **Docstrings are rendered as HTML.** If `docutils` is installed, RST-formatted docstrings are converted to HTML and displayed in the block documentation panel. Write proper RST with `:param:` directives for best results. + +5. **The `# optional` flag is important.** Mark your toolbox as optional in `requirements-pyodide.txt` unless PathView cannot function without it. This ensures the app starts even if your package has installation issues in a user's browser environment. + +6. **Test the extraction locally.** Run `python scripts/extract.py --validate` to check config file validity, then `python scripts/extract.py --blocks` to verify block extraction before committing. diff --git a/package.json b/package.json index d8024cc1..7ea72aa5 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,9 @@ "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", "lint": "eslint .", - "format": "prettier --write ." + "format": "prettier --write .", + "server": "python -m pathview_server.app", + "build:package": "python scripts/build_package.py" }, "devDependencies": { "@sveltejs/adapter-static": "^3.0.0", diff --git a/pathview_server/__init__.py b/pathview_server/__init__.py new file mode 100644 index 00000000..1431ca56 --- /dev/null +++ b/pathview_server/__init__.py @@ -0,0 +1,7 @@ +"""PathView Server — local Flask backend for PathView.""" + +try: + from importlib.metadata import version + __version__ = version("pathview") +except Exception: + __version__ = "0.5.0" # fallback for editable installs / dev diff --git a/pathview_server/__main__.py b/pathview_server/__main__.py new file mode 100644 index 00000000..4082f5cc --- /dev/null +++ b/pathview_server/__main__.py @@ -0,0 +1,6 @@ +"""Entry point for: python -m pathview_server""" + +from pathview_server.cli import main + +if __name__ == "__main__": + main() diff --git a/pathview_server/app.py b/pathview_server/app.py new file mode 100644 index 00000000..5bf56ba8 --- /dev/null +++ b/pathview_server/app.py @@ -0,0 +1,550 @@ +""" +Flask server for PathView backend. + +Manages worker subprocesses per session. Routes translate HTTP requests +into subprocess messages and relay responses back. + +Each session gets its own worker subprocess with an isolated Python namespace. +""" + +import os +import sys +import json +import queue +import subprocess +import threading +import time +import uuid +import atexit +from pathlib import Path + +from flask import Flask, request, jsonify, send_from_directory +from flask_cors import CORS + +from pathview_server.venv import get_venv_python + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +SESSION_TTL = 3600 # 1 hour of inactivity before cleanup +CLEANUP_INTERVAL = 60 # Check for stale sessions every 60 seconds +EXEC_TIMEOUT = 35 # Server-side timeout for exec/eval (slightly > worker's 30s) +WORKER_SCRIPT = str(Path(__file__).parent / "worker.py") + +# --------------------------------------------------------------------------- +# Session management +# --------------------------------------------------------------------------- + +class Session: + """A worker subprocess bound to a session.""" + + def __init__(self, session_id: str): + self.session_id = session_id + self.last_active = time.time() + self.lock = threading.Lock() + self.process = subprocess.Popen( + [get_venv_python(), "-u", WORKER_SCRIPT], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, # line buffered + ) + self._initialized = False + # Streaming state: background thread reads worker stdout into a queue + self._stream_queue: queue.Queue[dict] = queue.Queue() + self._stream_reader: threading.Thread | None = None + self._streaming = False + + def send_message(self, msg: dict) -> None: + """Write a JSON message to the subprocess stdin.""" + self.last_active = time.time() + line = json.dumps(msg) + "\n" + self.process.stdin.write(line) + self.process.stdin.flush() + + def read_line(self) -> dict | None: + """Read one JSON line from the subprocess stdout.""" + line = self.process.stdout.readline() + if not line: + return None + return json.loads(line.strip()) + + def read_line_timeout(self, timeout: float = EXEC_TIMEOUT) -> dict | None: + """Read one JSON line with a timeout. Returns None on EOF or timeout. + + Raises TimeoutError if no response within the timeout period. + """ + result = [None] + error = [None] + + def reader(): + try: + result[0] = self.read_line() + except Exception as e: + error[0] = e + + t = threading.Thread(target=reader, daemon=True) + t.start() + t.join(timeout) + + if t.is_alive(): + raise TimeoutError(f"Worker unresponsive after {timeout}s") + + if error[0]: + raise error[0] + + return result[0] + + def ensure_initialized(self, packages: list[dict] | None = None) -> list[dict]: + """Initialize the worker if not already done. Returns any messages received.""" + if self._initialized: + return [] + messages = [] + init_msg = {"type": "init"} + if packages: + init_msg["packages"] = packages + self.send_message(init_msg) + while True: + resp = self.read_line() + if resp is None: + raise RuntimeError("Worker process died during initialization") + messages.append(resp) + if resp.get("type") == "ready": + self._initialized = True + break + if resp.get("type") == "error": + raise RuntimeError(resp.get("error", "Unknown init error")) + return messages + + def start_stream_reader(self) -> None: + """Start a background thread that reads worker stdout into the stream queue.""" + self._streaming = True + # Clear any stale messages + while not self._stream_queue.empty(): + try: + self._stream_queue.get_nowait() + except queue.Empty: + break + + def reader(): + while self._streaming: + resp = self.read_line() + if resp is None: + self._stream_queue.put({"type": "error", "error": "Worker process died"}) + self._streaming = False + break + self._stream_queue.put(resp) + if resp.get("type") in ("stream-done", "error"): + self._streaming = False + break + + self._stream_reader = threading.Thread(target=reader, daemon=True) + self._stream_reader.start() + + def stop_stream_reader(self) -> None: + """Signal the stream reader to stop.""" + self._streaming = False + + def wait_for_stream_reader(self, timeout: float = 5) -> None: + """Wait for the stream reader thread to fully exit. + + Must be called before any direct stdout reads (exec/eval) to prevent + concurrent reads on the same pipe which cause JSONDecodeError. + + Actively stops streaming if needed: sends stream-stop to the worker + so it sends stream-done, which the reader thread reads naturally and + exits via its own loop termination. This ensures all final stream-data + messages are read into the queue before the reader exits. + """ + reader = self._stream_reader + if reader is None: + return + + if reader.is_alive(): + # Do NOT set self._streaming = False here — that would cause the + # reader thread to exit early on its next loop check, missing the + # final stream-data and stream-done messages. Instead, send + # stream-stop so the worker sends stream-done, which the reader + # reads and exits naturally (line 137-139 in start_stream_reader). + try: + self.send_message({"type": "stream-stop"}) + except Exception: + pass + reader.join(timeout) + # If reader is still alive after timeout, force-stop it + if reader.is_alive(): + self._streaming = False + reader.join(1) + + # Send noop to unblock the worker's stdin reader thread so the + # worker main loop can resume processing exec/eval messages. + self.flush_worker_reader() + + self._stream_reader = None + # Don't drain the stream queue here — the frontend's poll chain + # still needs the final stream-data/stream-done messages. + # start_stream_reader() clears stale messages when a new stream begins. + + def flush_worker_reader(self) -> None: + """Send a noop message to unblock the worker's stdin reader thread. + + After streaming ends, the worker's reader thread is blocked on stdin.readline(). + This sends a harmless message so the reader thread wakes up, checks stop_event, + and exits — allowing the main thread to resume reading stdin safely. + """ + try: + self.send_message({"type": "noop"}) + except Exception: + pass + + def drain_stream_queue(self, timeout: float = 0) -> list[dict]: + """Drain all messages from the stream queue. + + If timeout > 0, blocks until at least one message arrives or + the timeout expires. This turns polling into long-polling, + eliminating empty responses and reducing HTTP overhead. + """ + messages: list[dict] = [] + if timeout > 0 and self._stream_queue.empty(): + try: + messages.append(self._stream_queue.get(timeout=timeout)) + except queue.Empty: + return messages + while True: + try: + messages.append(self._stream_queue.get_nowait()) + except queue.Empty: + break + return messages + + def is_alive(self) -> bool: + return self.process.poll() is None + + def kill(self) -> None: + """Kill the subprocess.""" + self._streaming = False + try: + self.process.stdin.close() + except Exception: + pass + try: + self.process.kill() + self.process.wait(timeout=5) + except Exception: + pass + + +# Global session store +_sessions: dict[str, Session] = {} +_sessions_lock = threading.Lock() + + +def get_or_create_session(session_id: str) -> Session: + """Get an existing session or create a new one.""" + with _sessions_lock: + session = _sessions.get(session_id) + if session and not session.is_alive(): + # Dead process, remove stale entry + _sessions.pop(session_id, None) + session = None + if session is None: + session = Session(session_id) + _sessions[session_id] = session + return session + + +def remove_session(session_id: str) -> None: + """Kill and remove a session.""" + with _sessions_lock: + session = _sessions.pop(session_id, None) + if session: + session.kill() + + +def cleanup_stale_sessions() -> None: + """Remove sessions that have been inactive beyond TTL.""" + while True: + time.sleep(CLEANUP_INTERVAL) + now = time.time() + stale = [] + with _sessions_lock: + for sid, session in _sessions.items(): + if now - session.last_active > SESSION_TTL: + stale.append(sid) + for sid in stale: + remove_session(sid) + + +_cleanup_started = False + + +def _start_cleanup_thread() -> None: + """Start the cleanup thread once (idempotent).""" + global _cleanup_started + if _cleanup_started: + return + _cleanup_started = True + t = threading.Thread(target=cleanup_stale_sessions, daemon=True) + t.start() + + +def _get_session_id() -> str | None: + """Extract session ID from request headers. Returns None if missing.""" + return request.headers.get("X-Session-ID") + + +# --------------------------------------------------------------------------- +# App factory +# --------------------------------------------------------------------------- + +def create_app(serve_static: bool = False) -> Flask: + """Create the Flask application. + + Args: + serve_static: If True, serve the bundled frontend and skip CORS. + If False, API-only mode with CORS (for dev with Vite). + """ + app = Flask(__name__) + _start_cleanup_thread() + + if not serve_static: + CORS(app, max_age=3600) + + # ----------------------------------------------------------------------- + # API routes + # ----------------------------------------------------------------------- + + @app.route("/api/health", methods=["GET"]) + def health(): + return jsonify({"status": "ok"}) + + @app.route("/api/init", methods=["POST"]) + def api_init(): + """Initialize a session's worker with packages from the frontend config.""" + session_id = _get_session_id() + if not session_id: + return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 + data = request.get_json(force=True) + packages = data.get("packages", []) + + session = get_or_create_session(session_id) + with session.lock: + try: + messages = session.ensure_initialized(packages=packages) + return jsonify({"type": "ready", "messages": messages}) + except Exception as e: + return jsonify({"type": "error", "error": str(e)}), 500 + + def _handle_worker_request(msg: dict, success_type: str) -> tuple: + """Send a message to the worker and collect the response. + + Shared by api_exec (success_type="ok") and api_eval (success_type="value"). + Returns a Flask response tuple. + """ + session_id = _get_session_id() + if not session_id: + return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 + msg_id = msg.get("id", str(uuid.uuid4())) + + session = get_or_create_session(session_id) + with session.lock: + try: + # Wait for any lingering stream reader thread to exit before + # reading stdout — prevents concurrent pipe reads / JSONDecodeError + session.wait_for_stream_reader() + session.ensure_initialized() + session.send_message(msg) + + stdout_lines = [] + stderr_lines = [] + while True: + resp = session.read_line_timeout() + if resp is None: + remove_session(session_id) + return jsonify({"type": "error", "errorType": "worker-crashed", "id": msg_id, "error": "Worker process died"}), 500 + resp_type = resp.get("type") + if resp_type == "stdout": + stdout_lines.append(resp.get("value", "")) + elif resp_type == "stderr": + stderr_lines.append(resp.get("value", "")) + elif resp_type == success_type and resp.get("id") == msg_id: + result = resp + if stdout_lines: + result["stdout"] = "".join(stdout_lines) + if stderr_lines: + result["stderr"] = "".join(stderr_lines) + return jsonify(result) + elif resp_type == "error" and resp.get("id") == msg_id: + result = resp + if stdout_lines: + result["stdout"] = "".join(stdout_lines) + if stderr_lines: + result["stderr"] = "".join(stderr_lines) + return jsonify(result), 400 + + except TimeoutError: + remove_session(session_id) + return jsonify({"type": "error", "errorType": "timeout", "id": msg_id, "error": "Execution timed out"}), 504 + except Exception as e: + return jsonify({"type": "error", "id": msg_id, "error": str(e)}), 500 + + @app.route("/api/exec", methods=["POST"]) + def api_exec(): + """Execute Python code in the session's worker.""" + data = request.get_json(force=True) + msg_id = data.get("id", str(uuid.uuid4())) + return _handle_worker_request( + {"type": "exec", "id": msg_id, "code": data.get("code", "")}, + success_type="ok", + ) + + @app.route("/api/eval", methods=["POST"]) + def api_eval(): + """Evaluate a Python expression in the session's worker.""" + data = request.get_json(force=True) + msg_id = data.get("id", str(uuid.uuid4())) + return _handle_worker_request( + {"type": "eval", "id": msg_id, "expr": data.get("expr", "")}, + success_type="value", + ) + + @app.route("/api/stream/start", methods=["POST"]) + def api_stream_start(): + """Start streaming — sends stream-start to worker and returns immediately. + + A background thread reads worker stdout into a queue. The + frontend long-polls /api/stream/poll to drain that queue. + """ + session_id = _get_session_id() + if not session_id: + return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 + data = request.get_json(force=True) + expr = data.get("expr", "") + msg_id = data.get("id", str(uuid.uuid4())) + + session = get_or_create_session(session_id) + with session.lock: + try: + session.ensure_initialized() + session.send_message({"type": "stream-start", "id": msg_id, "expr": expr}) + session.start_stream_reader() + return jsonify({"status": "started", "id": msg_id}) + except Exception as e: + return jsonify({"type": "error", "error": str(e)}), 500 + + @app.route("/api/stream/poll", methods=["POST"]) + def api_stream_poll(): + """Long-poll for stream messages. + + Blocks up to 100 ms waiting for data before returning. + This eliminates empty responses and reduces HTTP overhead + compared to blind 30 ms polling. + """ + session_id = _get_session_id() + if not session_id: + return jsonify({"type": "error", "error": "Missing X-Session-ID header"}), 400 + with _sessions_lock: + session = _sessions.get(session_id) + if not session: + return jsonify({"messages": [], "done": True}) + messages = session.drain_stream_queue(timeout=0.1) + done = any(m.get("type") in ("stream-done", "error") for m in messages) + if done: + # Send noop to unblock the worker's stdin reader thread + # so the main thread can resume processing exec/eval + session.flush_worker_reader() + return jsonify({"messages": messages, "done": done}) + + @app.route("/api/stream/exec", methods=["POST"]) + def api_stream_exec(): + """Queue code to execute during an active stream.""" + session_id = _get_session_id() + if not session_id: + return jsonify({"error": "Missing X-Session-ID header"}), 400 + data = request.get_json(force=True) + code = data.get("code", "") + + with _sessions_lock: + session = _sessions.get(session_id) + if not session: + return jsonify({"error": "No active session"}), 404 + try: + session.send_message({"type": "stream-exec", "code": code}) + return jsonify({"status": "queued"}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + @app.route("/api/stream/stop", methods=["POST"]) + def api_stream_stop(): + """Stop an active streaming session.""" + session_id = _get_session_id() + if not session_id: + return jsonify({"error": "Missing X-Session-ID header"}), 400 + + with _sessions_lock: + session = _sessions.get(session_id) + if not session: + return jsonify({"status": "stopped"}) + try: + session.send_message({"type": "stream-stop"}) + return jsonify({"status": "stopped"}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + @app.route("/api/session", methods=["DELETE"]) + def api_session_delete(): + """Kill a session's worker subprocess.""" + session_id = _get_session_id() + if not session_id: + return jsonify({"error": "Missing X-Session-ID header"}), 400 + remove_session(session_id) + return jsonify({"status": "terminated"}) + + # ----------------------------------------------------------------------- + # Static file serving (pip package mode) + # ----------------------------------------------------------------------- + + if serve_static: + static_dir = Path(__file__).parent / "static" + + @app.route("/", defaults={"path": ""}) + @app.route("/") + def serve_frontend(path): + """Serve the bundled SvelteKit frontend with SPA fallback.""" + if path.startswith("api/"): + return jsonify({"error": "Not found"}), 404 + + file_path = static_dir / path + if path and file_path.is_file(): + return send_from_directory(static_dir, path) + + # SPA fallback + return send_from_directory(static_dir, "index.html") + + return app + + +# --------------------------------------------------------------------------- +# Cleanup on exit +# --------------------------------------------------------------------------- + +@atexit.register +def _cleanup_all_sessions(): + with _sessions_lock: + for session in _sessions.values(): + session.kill() + _sessions.clear() + + +# --------------------------------------------------------------------------- +# Entry point (dev mode: API-only with CORS) +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + port = int(os.environ.get("PORT", 5000)) + debug = os.environ.get("FLASK_DEBUG", "1") == "1" + print(f"PathView Flask backend (API-only) starting on port {port}") + app = create_app(serve_static=False) + app.run(host="0.0.0.0", port=port, debug=debug, threaded=True) diff --git a/pathview_server/cli.py b/pathview_server/cli.py new file mode 100644 index 00000000..e82b7d11 --- /dev/null +++ b/pathview_server/cli.py @@ -0,0 +1,77 @@ +"""CLI entry point for the pathview command.""" + +import argparse +import sys +import threading +import time +import webbrowser + +from pathview_server import __version__ +from pathview_server.venv import VENV_DIR, ensure_venv + + +def main(): + parser = argparse.ArgumentParser( + prog="pathview", + description="PathView — visual node editor for dynamic systems", + ) + parser.add_argument("command", nargs="?", default="serve", choices=["serve"], + help="Command to run (default: serve)") + parser.add_argument("--port", type=int, default=5000, + help="Port to run the server on (default: 5000)") + parser.add_argument("--host", type=str, default="127.0.0.1", + help="Host to bind to (default: 127.0.0.1)") + parser.add_argument("--no-browser", action="store_true", + help="Don't automatically open the browser") + parser.add_argument("--debug", action="store_true", + help="Run in debug mode") + parser.add_argument("--version", action="version", + version=f"pathview {__version__}") + + args = parser.parse_args() + + ensure_venv() + + from pathview_server.app import create_app + + app = create_app(serve_static=not args.debug) + + if not args.no_browser: + def open_browser_when_ready(): + import urllib.request + health_url = f"http://{args.host}:{args.port}/api/health" + deadline = time.time() + 10 + while time.time() < deadline: + try: + urllib.request.urlopen(health_url, timeout=1) + webbrowser.open(f"http://{args.host}:{args.port}") + return + except Exception: + time.sleep(0.2) + + threading.Thread(target=open_browser_when_ready, daemon=True).start() + + print(f"PathView v{__version__}") + print(f" Python venv: {VENV_DIR}") + print(f"Running at http://{args.host}:{args.port}") + + if args.host == "0.0.0.0": + print("\nWARNING: Binding to 0.0.0.0 makes the server accessible on your network.") + print(" There is no authentication — anyone on your network can execute Python code.") + print(" Only use this on trusted networks.") + + print("\nPress Ctrl+C to stop\n") + + try: + if args.debug: + app.run(host=args.host, port=args.port, debug=True, threaded=True) + else: + from waitress import serve + serve(app, host=args.host, port=args.port, threads=4) + except KeyboardInterrupt: + print("\nStopping PathView server...") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/pathview_server/venv.py b/pathview_server/venv.py new file mode 100644 index 00000000..df8e28a3 --- /dev/null +++ b/pathview_server/venv.py @@ -0,0 +1,34 @@ +"""Virtual environment management for PathView worker subprocesses. + +Creates and manages a dedicated venv at ~/.pathview/venv so that simulation +dependencies (pathsim, pathsim-chem, numpy, etc.) are installed in isolation +rather than polluting the user's global/active environment. +""" + +import subprocess +import sys +from pathlib import Path + +VENV_DIR = Path.home() / ".pathview" / "venv" + + +def get_venv_python() -> str: + """Return path to the venv's Python executable.""" + if sys.platform == "win32": + return str(VENV_DIR / "Scripts" / "python.exe") + return str(VENV_DIR / "bin" / "python") + + +def ensure_venv() -> str: + """Create the venv if it doesn't exist. Returns venv Python path.""" + python = get_venv_python() + if Path(python).exists(): + return python + + print("Creating PathView virtual environment...") + VENV_DIR.parent.mkdir(parents=True, exist_ok=True) + subprocess.run([sys.executable, "-m", "venv", str(VENV_DIR)], check=True) + # Upgrade pip in the venv + subprocess.run([python, "-m", "pip", "install", "--upgrade", "pip", "--quiet"], check=True) + print(f" Virtual environment created at {VENV_DIR}") + return python diff --git a/pathview_server/worker.py b/pathview_server/worker.py new file mode 100644 index 00000000..2546e745 --- /dev/null +++ b/pathview_server/worker.py @@ -0,0 +1,432 @@ +""" +REPL Worker Subprocess for PathView Flask Backend. + +Direct port of worker.ts to Python. Reads JSON messages from stdin, +executes Python code, and writes JSON responses to stdout. + +Same message protocol (REPLRequest/REPLResponse as JSON lines over stdin/stdout). + +Threading model: +- Main thread: reads stdin, processes init/exec/eval synchronously +- During streaming: a reader thread handles stream-stop and stream-exec, + puts non-streaming messages into a leftover queue +- After streaming: main thread drains leftovers before resuming stdin reads +- Stdout lock: thread-safe writing to stdout (protocol messages only) +""" + +import sys +import json +import subprocess +import threading +import traceback +import queue +import ctypes + +# Lock for thread-safe stdout writing (protocol messages only) +_stdout_lock = threading.Lock() + +# Keep a reference to the real stdout pipe — protocol messages go here. +_real_stdout = sys.stdout + +# Worker state +_namespace = {} +_initialized = False + +# Default timeout for exec/eval (seconds) +EXEC_TIMEOUT = 30 + +# Streaming state +_streaming_active = False +_streaming_code_queue = queue.Queue() +_leftover_queue: queue.Queue[dict | None] = queue.Queue() + + +def send(response: dict) -> None: + """Send a JSON response to the parent process via stdout.""" + with _stdout_lock: + _real_stdout.write(json.dumps(response) + "\n") + _real_stdout.flush() + + +class _ProtocolWriter: + """File-like object that routes writes through the worker protocol. + + Installed as sys.stdout/sys.stderr permanently so that ALL output + (print, logging handlers, third-party libraries) is captured and + forwarded to the frontend console. This avoids the stale-reference + bug where StreamHandler(sys.stdout) captures a temporary StringIO. + """ + + def __init__(self, msg_type: str): + self.msg_type = msg_type + self._in_write = False + + def write(self, text: str) -> int: + if text and not self._in_write: + self._in_write = True + try: + send({"type": self.msg_type, "value": text}) + except Exception: + pass + finally: + self._in_write = False + return len(text) if text else 0 + + def flush(self) -> None: + pass + + def isatty(self) -> bool: + return False + + +def read_message(): + """Read one JSON message from stdin. Returns None on EOF.""" + while True: + line = sys.stdin.readline() + if not line: + return None + line = line.strip() + if line: + return json.loads(line) + + +def _install_package(pip_spec: str, pre: bool = False) -> None: + """Install a package via pip if not already available.""" + cmd = [sys.executable, "-m", "pip", "install", pip_spec, "--quiet"] + if pre: + cmd.append("--pre") + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"pip install failed for {pip_spec}: {result.stderr.strip()}") + + +def _ensure_package(pkg: dict) -> None: + """Ensure a package is installed and importable. Mirrors the Pyodide worker loop.""" + import_name = pkg.get("import", "") + pip_spec = pkg.get("pip", import_name) + required = pkg.get("required", False) + pre = pkg.get("pre", False) + + send({"type": "progress", "value": f"Installing {import_name}..."}) + + try: + # Try importing first — skip pip if already installed + exec(f"import {import_name}", _namespace) + except ImportError: + # Not installed — pip install then import + _install_package(pip_spec, pre) + exec(f"import {import_name}", _namespace) + + # Log version if available + try: + version = eval(f"{import_name}.__version__", _namespace) + send({"type": "stdout", "value": f"{import_name} {version} loaded successfully\n"}) + except Exception: + send({"type": "stdout", "value": f"{import_name} loaded successfully\n"}) + + +def initialize(packages: list[dict] | None = None) -> None: + """Initialize the worker: install packages, import standard libs, capture clean globals.""" + global _initialized, _namespace + + if _initialized: + send({"type": "ready"}) + return + + send({"type": "progress", "value": "Initializing Python worker..."}) + + # Replace sys.stdout/stderr with protocol writers BEFORE importing packages. + # Any StreamHandler created later (e.g. by pathsim's LoggerManager singleton) + # will capture these persistent objects, so logging always routes through send(). + sys.stdout = _ProtocolWriter("stdout") + sys.stderr = _ProtocolWriter("stderr") + + # Set up the namespace with common imports + _namespace = {"__builtins__": __builtins__} + exec("import gc", _namespace) + exec("import json", _namespace) + + # Install packages FIRST (pathsim brings numpy as a dependency) + if packages: + send({"type": "progress", "value": "Installing dependencies..."}) + for pkg in packages: + try: + _ensure_package(pkg) + except Exception as e: + if pkg.get("required", False): + raise RuntimeError( + f"Failed to install required package {pkg.get('pip', pkg.get('import', '?'))}: {e}" + ) + send({"type": "stderr", "value": f"Optional package {pkg.get('import', '?')} failed: {e}\n"}) + + # Import numpy AFTER packages are installed (numpy comes with pathsim). + # In a fresh venv without simulation packages, numpy won't be available. + try: + exec("import numpy as np", _namespace) + except Exception: + pass + + _initialized = True + send({"type": "ready"}) + + +def _raise_in_thread(thread_id: int, exc_type: type) -> None: + """Raise an exception in the given thread (best-effort interrupt).""" + ctypes.pythonapi.PyThreadState_SetAsyncExc( + ctypes.c_ulong(thread_id), ctypes.py_object(exc_type) + ) + + +def _run_with_timeout(func, timeout: float = EXEC_TIMEOUT): + """Run func() in a daemon thread with a timeout. + + Returns (result, error_string, traceback_string). + If timeout fires, raises TimeoutError. + """ + result_holder = [None, None, None] # result, error, traceback + + def target(): + try: + result_holder[0] = func() + except Exception as e: + result_holder[1] = str(e) + result_holder[2] = traceback.format_exc() + + t = threading.Thread(target=target, daemon=True) + t.start() + t.join(timeout) + + if t.is_alive(): + # Try to interrupt the stuck thread + _raise_in_thread(t.ident, KeyboardInterrupt) + t.join(2) # Give it 2s to handle the interrupt + raise TimeoutError(f"Execution timed out after {timeout}s") + + if result_holder[1] is not None: + raise _ExecError(result_holder[1], result_holder[2]) + + return result_holder[0] + + +class _ExecError(Exception): + """Wraps an error from user code execution with its traceback.""" + def __init__(self, message: str, tb: str | None = None): + super().__init__(message) + self.tb = tb + + +def exec_code(msg_id: str, code: str) -> None: + """Execute Python code (no return value).""" + if not _initialized: + send({"type": "error", "id": msg_id, "error": "Worker not initialized"}) + return + + try: + _run_with_timeout(lambda: exec(code, _namespace)) + send({"type": "ok", "id": msg_id}) + except TimeoutError as e: + send({"type": "error", "id": msg_id, "error": str(e)}) + except _ExecError as e: + send({"type": "error", "id": msg_id, "error": str(e), "traceback": e.tb}) + + +def eval_expr(msg_id: str, expr: str) -> None: + """Evaluate Python expression and return JSON result.""" + if not _initialized: + send({"type": "error", "id": msg_id, "error": "Worker not initialized"}) + return + + try: + def do_eval(): + exec_code_str = f"_eval_result = {expr}" + exec(exec_code_str, _namespace) + + _run_with_timeout(do_eval) + to_json = _namespace.get("_to_json", str) + result = json.dumps(_namespace["_eval_result"], default=to_json) + send({"type": "value", "id": msg_id, "value": result}) + except TimeoutError as e: + send({"type": "error", "id": msg_id, "error": str(e)}) + except _ExecError as e: + send({"type": "error", "id": msg_id, "error": str(e), "traceback": e.tb}) + + +def _streaming_reader_thread(stop_event: threading.Event) -> None: + """Read stdin during streaming, handling stream-stop and stream-exec. + + Non-streaming messages are saved to _leftover_queue for the main loop. + After stop_event is set, the thread will exit once it reads one more line + (the flush message sent by the server). + """ + global _streaming_active + while True: + line = sys.stdin.readline() + if not line: + # EOF — stop streaming + _streaming_active = False + _leftover_queue.put(None) + break + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + send({"type": "error", "error": f"Invalid JSON: {line}"}) + continue + + # If stop_event is set, we're just flushing — put message in leftovers + if stop_event.is_set(): + _leftover_queue.put(msg) + break + + msg_type = msg.get("type") + if msg_type == "stream-stop": + _streaming_active = False + elif msg_type == "stream-exec": + code = msg.get("code") + if code and _streaming_active: + _streaming_code_queue.put(code) + else: + # Non-streaming message arrived — save for main loop + _leftover_queue.put(msg) + + +def run_streaming_loop(msg_id: str, expr: str) -> None: + """Run streaming loop - steps generator continuously and posts results.""" + global _streaming_active + + if not _initialized: + send({"type": "error", "id": msg_id, "error": "Worker not initialized"}) + return + + _streaming_active = True + # Clear any stale code from previous runs + while not _streaming_code_queue.empty(): + try: + _streaming_code_queue.get_nowait() + except queue.Empty: + break + + # Start reader thread to handle stream-stop and stream-exec + stop_event = threading.Event() + reader = threading.Thread(target=_streaming_reader_thread, args=(stop_event,), daemon=True) + reader.start() + + try: + while _streaming_active: + # Execute any queued code first (for runtime parameter changes) + # Errors in queued code are reported but don't stop the simulation + while True: + try: + code = _streaming_code_queue.get_nowait() + except queue.Empty: + break + try: + exec(code, _namespace) + except Exception as e: + send({"type": "stderr", "value": f"Stream exec error: {e}"}) + + # Step the generator + exec_code_str = f"_eval_result = {expr}" + exec(exec_code_str, _namespace) + raw_result = _namespace["_eval_result"] + done = raw_result.get("done", False) if isinstance(raw_result, dict) else False + + # Check if stopped during Python execution - still send final data + if not _streaming_active: + if not done and (isinstance(raw_result, dict) and raw_result.get("result")): + to_json = _namespace.get("_to_json", str) + send({"type": "stream-data", "id": msg_id, "value": json.dumps(raw_result, default=to_json)}) + break + + # Check if simulation completed + if done: + break + + # Send result and continue + to_json = _namespace.get("_to_json", str) + send({"type": "stream-data", "id": msg_id, "value": json.dumps(raw_result, default=to_json)}) + + except Exception as e: + tb = traceback.format_exc() + send({"type": "error", "id": msg_id, "error": str(e), "traceback": tb}) + finally: + _streaming_active = False + stop_event.set() + # Always send done when loop ends + send({"type": "stream-done", "id": msg_id}) + # Reader thread will exit on the next stdin read (flush message from server) + + +def main() -> None: + """Main loop: read messages from stdin and process them.""" + while True: + # First drain any leftover messages from the streaming reader thread + msg = None + while not _leftover_queue.empty(): + try: + msg = _leftover_queue.get_nowait() + if msg is not None: + break + except queue.Empty: + break + + # If no leftover, read from stdin directly + if msg is None: + msg = read_message() + if msg is None: + # stdin closed, exit + break + + msg_type = msg.get("type") + msg_id = msg.get("id") + code = msg.get("code") + expr = msg.get("expr") + + try: + if msg_type == "init": + initialize(packages=msg.get("packages")) + + elif msg_type == "exec": + if not msg_id or not isinstance(code, str): + raise ValueError("Invalid exec request: missing id or code") + exec_code(msg_id, code) + + elif msg_type == "eval": + if not msg_id or not isinstance(expr, str): + raise ValueError("Invalid eval request: missing id or expr") + eval_expr(msg_id, expr) + + elif msg_type == "stream-start": + if not msg_id or not isinstance(expr, str): + raise ValueError("Invalid stream-start request: missing id or expr") + # Blocking: runs streaming loop, reader thread handles + # stream-stop and stream-exec during this time + run_streaming_loop(msg_id, expr) + + elif msg_type == "stream-stop": + # Only during streaming (handled by reader thread) + pass + + elif msg_type == "stream-exec": + # Only during streaming (handled by reader thread) + pass + + elif msg_type == "noop": + # Flush message from server — ignore + pass + + else: + raise ValueError(f"Unknown message type: {msg_type}") + + except Exception as e: + send({ + "type": "error", + "id": msg_id, + "error": str(e), + }) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..23493c96 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pathview" +version = "0.5.0" +description = "Visual node editor for building and simulating dynamic systems with PathSim" +readme = "README.md" +license = {text = "MIT"} +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering", +] +dependencies = [ + "flask>=3.0", + "flask-cors>=4.0", + "numpy", + "waitress>=3.0", +] + +[project.optional-dependencies] +test = [ + "pytest>=8.0", +] + +[project.urls] +Homepage = "https://view.pathsim.org" +Repository = "https://github.com/pathsim/pathview" + +[project.scripts] +pathview = "pathview_server.cli:main" + +[tool.setuptools] +packages = ["pathview_server"] + +[tool.setuptools.package-data] +pathview_server = ["static/**/*"] diff --git a/scripts/build_package.py b/scripts/build_package.py new file mode 100644 index 00000000..68eb4adf --- /dev/null +++ b/scripts/build_package.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +""" +Build script for the PathView PyPI package. + +1. Builds the SvelteKit frontend (vite build) +2. Copies build/ output to pathview_server/static/ +3. Builds the Python wheel +""" + +import json +import os +import re +import sys +import shutil +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).parent.parent +BUILD_DIR = REPO_ROOT / "build" +STATIC_DIR = REPO_ROOT / "pathview_server" / "static" + + +def _find_npx(): + """Find the npx binary. On Windows, use npx.cmd.""" + name = "npx.cmd" if sys.platform == "win32" else "npx" + path = shutil.which(name) + if not path: + print(f"ERROR: {name} not found on PATH") + sys.exit(1) + return path + + +def run(cmd, **kwargs): + print(f" > {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=kwargs.pop("cwd", REPO_ROOT), **kwargs) + if result.returncode != 0: + print(f"ERROR: command failed (exit {result.returncode})") + sys.exit(result.returncode) + + +def _sync_version(): + """Read version from pyproject.toml and sync to package.json.""" + pyproject = REPO_ROOT / "pyproject.toml" + text = pyproject.read_text() + match = re.search(r'^version\s*=\s*"([^"]+)"', text, re.MULTILINE) + if not match: + print("ERROR: could not find version in pyproject.toml") + sys.exit(1) + version = match.group(1) + + pkg_json_path = REPO_ROOT / "package.json" + pkg = json.loads(pkg_json_path.read_text()) + if pkg.get("version") != version: + print(f" Syncing version {pkg.get('version')} → {version} in package.json") + pkg["version"] = version + pkg_json_path.write_text(json.dumps(pkg, indent=2) + "\n") + return version + + +def main(): + print("[0/4] Syncing version...") + version = _sync_version() + print(f" Version: {version}") + + print("[1/4] Cleaning previous builds...") + for d in [BUILD_DIR, STATIC_DIR, REPO_ROOT / "dist"]: + if d.exists(): + shutil.rmtree(d) + + # Remove egg-info + for p in REPO_ROOT.glob("*.egg-info"): + shutil.rmtree(p) + + print("[2/4] Building SvelteKit frontend...") + env = os.environ.copy() + env["BASE_PATH"] = "" + npx = _find_npx() + run([npx, "vite", "build"], env=env) + + if not (BUILD_DIR / "index.html").exists(): + print("ERROR: build/index.html not found") + sys.exit(1) + + print("[3/4] Copying frontend to pathview_server/static/...") + shutil.copytree(BUILD_DIR, STATIC_DIR) + print(f" Copied {sum(1 for _ in STATIC_DIR.rglob('*') if _.is_file())} files") + + print("[4/4] Building Python wheel...") + run([sys.executable, "-m", "build"]) + + print("\nDone! Output:") + dist = REPO_ROOT / "dist" + if dist.exists(): + for f in sorted(dist.iterdir()): + print(f" {f.name}") + + +if __name__ == "__main__": + main() diff --git a/src/lib/pyodide/backend/flask/backend.ts b/src/lib/pyodide/backend/flask/backend.ts new file mode 100644 index 00000000..cdb19ca6 --- /dev/null +++ b/src/lib/pyodide/backend/flask/backend.ts @@ -0,0 +1,471 @@ +/** + * Flask Backend + * Implements the Backend interface using a Flask server with subprocess workers. + * + * Mirrors the Pyodide worker's message-passing pattern: + * - exec/eval use simple request/response + * - streaming uses start + poll (like postMessage with stream-data/stream-done) + */ + +import type { Backend, BackendState } from '../types'; +import { backendState } from '../state'; +import { TIMEOUTS } from '$lib/constants/python'; +import { STATUS_MESSAGES } from '$lib/constants/messages'; +import { PYTHON_PACKAGES } from '$lib/constants/dependencies'; + +/** Delay between polls (ms). The server uses long-polling (blocks up + * to 100 ms until data arrives), so data delivery is near-instant. + * This interval is just a safety gap between consecutive requests. */ +const STREAM_POLL_INTERVAL = 5; + +/** BroadcastChannel name for cross-tab session coordination */ +const SESSION_CHANNEL = 'flask-session'; + +export class FlaskBackend implements Backend { + private host: string; + private sessionId: string; + private messageId = 0; + private _isStreaming = false; + private streamPollTimer: ReturnType | null = null; + private serverInitPromise: Promise | null = null; + private broadcastChannel: BroadcastChannel | null = null; + + // Stream callbacks — same shape as PyodideBackend's streamState + private streamState: { + onData: ((data: unknown) => void) | null; + onDone: (() => void) | null; + onError: ((error: Error) => void) | null; + } = { onData: null, onDone: null, onError: null }; + + // Output callbacks + private stdoutCallback: ((value: string) => void) | null = null; + private stderrCallback: ((value: string) => void) | null = null; + + constructor(host: string) { + this.host = host.replace(/\/$/, ''); + const stored = typeof localStorage !== 'undefined' ? localStorage.getItem('flask-session-id') : null; + if (stored) { + this.sessionId = stored; + } else { + this.sessionId = crypto.randomUUID(); + if (typeof localStorage !== 'undefined') { + localStorage.setItem('flask-session-id', this.sessionId); + } + } + + // Listen for session termination from other tabs + if (typeof BroadcastChannel !== 'undefined') { + this.broadcastChannel = new BroadcastChannel(SESSION_CHANNEL); + this.broadcastChannel.onmessage = (event) => { + if (event.data?.type === 'session-terminated') { + this.serverInitPromise = null; + backendState.reset(); + } + }; + } + } + + // ------------------------------------------------------------------------- + // Lifecycle + // ------------------------------------------------------------------------- + + async init(): Promise { + const state = this.getState(); + if (state.initialized || state.loading) return; + + backendState.update((s) => ({ + ...s, + loading: true, + error: null, + progress: 'Connecting to Flask server...' + })); + + try { + const resp = await fetch(`${this.host}/api/health`, { + signal: AbortSignal.timeout(TIMEOUTS.INIT) + }); + if (!resp.ok) throw new Error(`Server health check failed: ${resp.status}`); + + backendState.update((s) => ({ ...s, progress: 'Initializing Python worker...' })); + + await this.postInit({ updateProgress: true }); + + this.serverInitPromise = Promise.resolve(); + + backendState.update((s) => ({ + ...s, + initialized: true, + loading: false, + progress: STATUS_MESSAGES.READY + })); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + backendState.update((s) => ({ ...s, loading: false, error: `Flask backend error: ${msg}` })); + throw error; + } + } + + terminate(): void { + this.stopStreaming(); + if (this.streamPollTimer) { + clearTimeout(this.streamPollTimer); + this.streamPollTimer = null; + } + this._isStreaming = false; + this.streamState = { onData: null, onDone: null, onError: null }; + + fetch(`${this.host}/api/session`, { + method: 'DELETE', + headers: { 'X-Session-ID': this.sessionId } + }).catch(() => {}); + + // Notify other tabs that the session was terminated + this.broadcastChannel?.postMessage({ type: 'session-terminated' }); + + this.serverInitPromise = null; + backendState.reset(); + } + + // ------------------------------------------------------------------------- + // State + // ------------------------------------------------------------------------- + + getState(): BackendState { + return backendState.get(); + } + + subscribe(callback: (state: BackendState) => void): () => void { + return backendState.subscribe(callback); + } + + isReady(): boolean { + return this.getState().initialized; + } + + isLoading(): boolean { + return this.getState().loading; + } + + getError(): string | null { + return this.getState().error; + } + + // ------------------------------------------------------------------------- + // Lazy server init + // ------------------------------------------------------------------------- + + private ensureServerInit(): Promise { + if (this.serverInitPromise) return this.serverInitPromise; + + this.serverInitPromise = this.postInit({ updateProgress: false }); + + // Clear on failure so subsequent calls retry instead of returning the rejected promise + this.serverInitPromise.catch(() => { + this.serverInitPromise = null; + }); + + return this.serverInitPromise; + } + + // ------------------------------------------------------------------------- + // Execution + // ------------------------------------------------------------------------- + + async exec(code: string, timeout: number = TIMEOUTS.SIMULATION): Promise { + await this.ensureServerInit(); + const id = this.generateId(); + + const resp = await fetch(`${this.host}/api/exec`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ id, code }), + signal: AbortSignal.timeout(timeout) + }); + + const data = await resp.json(); + if (data.stdout && this.stdoutCallback) this.stdoutCallback(data.stdout); + if (data.stderr && this.stderrCallback) this.stderrCallback(data.stderr); + + if (data.type === 'error') { + this.handleWorkerError(data); + const errorMsg = data.traceback ? `${data.error}\n${data.traceback}` : data.error; + throw new Error(errorMsg); + } + } + + async evaluate(expr: string, timeout: number = TIMEOUTS.SIMULATION): Promise { + await this.ensureServerInit(); + const id = this.generateId(); + + const resp = await fetch(`${this.host}/api/eval`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ id, expr }), + signal: AbortSignal.timeout(timeout) + }); + + const data = await resp.json(); + if (data.stdout && this.stdoutCallback) this.stdoutCallback(data.stdout); + if (data.stderr && this.stderrCallback) this.stderrCallback(data.stderr); + + if (data.type === 'error') { + this.handleWorkerError(data); + const errorMsg = data.traceback ? `${data.error}\n${data.traceback}` : data.error; + throw new Error(errorMsg); + } + + if (data.value === undefined) throw new Error('No value returned from eval'); + return JSON.parse(data.value) as T; + } + + // ------------------------------------------------------------------------- + // Streaming — mirrors Pyodide worker's postMessage pattern via polling + // ------------------------------------------------------------------------- + + startStreaming( + expr: string, + onData: (data: T) => void, + onDone: () => void, + onError: (error: Error) => void + ): void { + if (this._isStreaming) { + this.stopStreaming(); + } + + // Clear any lingering poll timer from a previous stream so it + // doesn't pick up a stale stream-done and fire the NEW onDone. + if (this.streamPollTimer) { + clearTimeout(this.streamPollTimer); + this.streamPollTimer = null; + } + + const id = this.generateId(); + this._isStreaming = true; + this.streamState = { + onData: onData as (data: unknown) => void, + onDone, + onError + }; + + // Start stream on server, then poll for results + this.ensureServerInit() + .then(() => + fetch(`${this.host}/api/stream/start`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ id, expr }) + }) + ) + .then((resp) => resp.json()) + .then((data) => { + if (data.type === 'error') { + throw new Error(data.error); + } + this.pollStreamResults(); + }) + .catch((error) => { + this._isStreaming = false; + onError(error instanceof Error ? error : new Error(String(error))); + this.streamState = { onData: null, onDone: null, onError: null }; + }); + } + + stopStreaming(): void { + if (!this._isStreaming) return; + + // Just send the stop signal — don't disrupt the polling loop. + // The poll loop will naturally pick up stream-done and clean up, + // matching how Pyodide's stopStreaming just sends stream-stop and + // lets worker.onmessage handle the rest. + fetch(`${this.host}/api/stream/stop`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + } + }).catch(() => { + // Network failure — poll loop will also fail and clean up + }); + } + + isStreaming(): boolean { + return this._isStreaming; + } + + execDuringStreaming(code: string): void { + if (!this._isStreaming) { + console.warn('Cannot exec during streaming: no active stream'); + return; + } + + fetch(`${this.host}/api/stream/exec`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ code }) + }).catch(() => {}); + } + + // ------------------------------------------------------------------------- + // Output Callbacks + // ------------------------------------------------------------------------- + + onStdout(callback: (value: string) => void): void { + this.stdoutCallback = callback; + } + + onStderr(callback: (value: string) => void): void { + this.stderrCallback = callback; + } + + // ------------------------------------------------------------------------- + // Private Methods + // ------------------------------------------------------------------------- + + private generateId(): string { + return `repl_${++this.messageId}`; + } + + /** + * POST /api/init with packages and forward worker messages to callbacks. + * Shared by init() (first load with progress UI) and ensureServerInit() (lazy re-init). + */ + private async postInit(opts: { updateProgress: boolean }): Promise { + const resp = await fetch(`${this.host}/api/init`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + }, + body: JSON.stringify({ packages: PYTHON_PACKAGES }), + signal: AbortSignal.timeout(TIMEOUTS.INIT) + }); + const data = await resp.json(); + if (data.type === 'error') throw new Error(data.error); + if (data.messages) { + for (const msg of data.messages) { + if (msg.type === 'stdout' && this.stdoutCallback) this.stdoutCallback(msg.value); + if (msg.type === 'stderr' && this.stderrCallback) this.stderrCallback(msg.value); + if (msg.type === 'progress' && opts.updateProgress) { + backendState.update((s) => ({ ...s, progress: msg.value })); + } + } + } + } + + /** + * Check if a response indicates the worker crashed or timed out. + * If so, clear serverInitPromise so the next request triggers re-init. + */ + private handleWorkerError(data: Record): void { + const errorType = data.errorType as string | undefined; + if (errorType === 'worker-crashed' || errorType === 'timeout') { + this.serverInitPromise = null; + if (this.stderrCallback) { + this.stderrCallback('Python worker crashed, restarting on next request...\n'); + } + } + } + + /** + * Poll the server for stream messages and dispatch them to callbacks. + * This mirrors the Pyodide backend's handleResponse for stream-data/stream-done. + */ + private async pollStreamResults(): Promise { + if (!this._isStreaming) return; + + try { + const resp = await fetch(`${this.host}/api/stream/poll`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Session-ID': this.sessionId + } + }); + + const data = await resp.json(); + + if (!Array.isArray(data?.messages)) { + throw new Error(data?.error || 'Invalid poll response'); + } + + for (const msg of data.messages) { + this.handleStreamMessage(msg); + if (!this._isStreaming) return; // done or error stopped streaming + } + + // Schedule next poll if still streaming + if (this._isStreaming) { + this.streamPollTimer = setTimeout(() => this.pollStreamResults(), STREAM_POLL_INTERVAL); + } + } catch (error) { + this._isStreaming = false; + if (this.streamState.onError) { + this.streamState.onError(error instanceof Error ? error : new Error(String(error))); + } + this.streamState = { onData: null, onDone: null, onError: null }; + } + } + + /** + * Handle a single message from the worker — same dispatch as PyodideBackend.handleResponse + */ + private handleStreamMessage(msg: Record): void { + const type = msg.type as string; + + switch (type) { + case 'stream-data': { + if (this.streamState.onData && msg.value) { + try { + this.streamState.onData(JSON.parse(msg.value as string)); + } catch { + // Ignore parse errors + } + } + break; + } + case 'stream-done': { + this._isStreaming = false; + if (this.streamState.onDone) { + this.streamState.onDone(); + } + this.streamState = { onData: null, onDone: null, onError: null }; + break; + } + case 'stdout': { + if (this.stdoutCallback && msg.value) { + this.stdoutCallback(msg.value as string); + } + break; + } + case 'stderr': { + if (this.stderrCallback && msg.value) { + this.stderrCallback(msg.value as string); + } + break; + } + case 'error': { + this._isStreaming = false; + if (this.streamState.onError) { + const errorMsg = msg.traceback + ? `${msg.error}\n${msg.traceback}` + : (msg.error as string) || 'Unknown error'; + this.streamState.onError(new Error(errorMsg)); + } + this.streamState = { onData: null, onDone: null, onError: null }; + break; + } + } + } +} diff --git a/src/lib/pyodide/backend/index.ts b/src/lib/pyodide/backend/index.ts index aea26ee5..ef04b5ce 100644 --- a/src/lib/pyodide/backend/index.ts +++ b/src/lib/pyodide/backend/index.ts @@ -20,21 +20,74 @@ export { getBackendType, hasBackend, terminateBackend, + setFlaskHost, type BackendType } from './registry'; -// Re-export PyodideBackend for direct use if needed +// Re-export backend implementations export { PyodideBackend } from './pyodide/backend'; +export { FlaskBackend } from './flask/backend'; // ============================================================================ // Backward-Compatible Convenience Functions // These delegate to the current backend and maintain API compatibility // ============================================================================ -import { getBackend } from './registry'; +import { getBackend, switchBackend, setFlaskHost, getBackendType } from './registry'; import { backendState } from './state'; import { consoleStore } from '$lib/stores/console'; +/** + * Initialize backend from URL parameters. + * Reads `?backend=flask` and `?host=...` from the current URL. + * Call this early in page mount, before any backend usage. + */ +export async function initBackendFromUrl(): Promise { + if (typeof window === 'undefined') return; + const params = new URLSearchParams(window.location.search); + const backendParam = params.get('backend'); + const hostParam = params.get('host'); + + if (backendParam === 'flask') { + if (hostParam) { + setFlaskHost(hostParam); + } + switchBackend('flask'); + await init(); + } +} + +/** + * Auto-detect if a Flask backend is available at the same origin. + * Used when the frontend is served by the Flask server (pip package mode). + * URL parameters take precedence — if `?backend=` is set, auto-detection is skipped. + */ +export async function autoDetectBackend(): Promise { + if (typeof window === 'undefined') return; + + // URL params override auto-detection + const params = new URLSearchParams(window.location.search); + if (params.has('backend')) return; + + try { + const response = await fetch('/api/health', { + method: 'GET', + signal: AbortSignal.timeout(2000) + }); + if (response.ok) { + const data = await response.json(); + if (data.status === 'ok') { + setFlaskHost(window.location.origin); + switchBackend('flask'); + // Run full init — sets up callbacks, logs progress, initializes worker + await init(); + } + } + } catch { + // No Flask backend at same origin — will use Pyodide + } +} + // Alias for backward compatibility export const replState = { subscribe: backendState.subscribe diff --git a/src/lib/pyodide/backend/registry.ts b/src/lib/pyodide/backend/registry.ts index e1eaaddc..263ae5b0 100644 --- a/src/lib/pyodide/backend/registry.ts +++ b/src/lib/pyodide/backend/registry.ts @@ -5,11 +5,20 @@ import type { Backend } from './types'; import { PyodideBackend } from './pyodide/backend'; +import { FlaskBackend } from './flask/backend'; -export type BackendType = 'pyodide' | 'local' | 'remote'; +export type BackendType = 'pyodide' | 'flask' | 'remote'; let currentBackend: Backend | null = null; let currentBackendType: BackendType | null = null; +let flaskHost = 'http://localhost:5000'; + +/** + * Set the Flask backend host URL + */ +export function setFlaskHost(host: string): void { + flaskHost = host; +} /** * Get the current backend, creating a Pyodide backend if none exists @@ -29,7 +38,8 @@ export function createBackend(type: BackendType): Backend { switch (type) { case 'pyodide': return new PyodideBackend(); - case 'local': + case 'flask': + return new FlaskBackend(flaskHost); case 'remote': throw new Error(`Backend type '${type}' not yet implemented`); default: diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 7aaad8bc..d6d5c1f7 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -40,6 +40,7 @@ import { openEventDialog } from '$lib/stores/eventDialog'; import type { MenuItemType } from '$lib/components/ContextMenu.svelte'; import { pyodideState, simulationState, initPyodide, stopSimulation, continueStreamingSimulation } from '$lib/pyodide/bridge'; + import { initBackendFromUrl, autoDetectBackend } from '$lib/pyodide/backend'; import { runGraphStreamingSimulation, validateGraphSimulation } from '$lib/pyodide/pathsimRunner'; import { consoleStore } from '$lib/stores/console'; import { newGraph, saveFile, saveAsFile, setupAutoSave, clearAutoSave, debouncedAutoSave, openImportDialog, importFromUrl, currentFileName } from '$lib/schema/fileOps'; @@ -381,6 +382,9 @@ const continueTooltip = { text: "Continue", shortcut: "Shift+Enter" }; onMount(() => { + // Auto-detect same-origin Flask backend (pip package mode), then check URL params + autoDetectBackend().then(() => initBackendFromUrl()); + // Subscribe to stores (with cleanup) const unsubPinnedPreviews = pinnedPreviewsStore.subscribe((pinned) => { showPinnedPreviews = pinned; diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..d3ceb8f5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,41 @@ +"""Shared pytest fixtures for PathView server tests.""" + +import pytest + +from pathview_server.app import create_app, _sessions, _sessions_lock +from pathview_server.venv import ensure_venv + + +@pytest.fixture() +def app(): + """Create a Flask test app (API-only, no static serving).""" + ensure_venv() + application = create_app(serve_static=False) + application.config["TESTING"] = True + yield application + # Clean up all sessions after each test + with _sessions_lock: + for session in _sessions.values(): + session.kill() + _sessions.clear() + + +@pytest.fixture() +def client(app): + """Flask test client.""" + return app.test_client() + + +@pytest.fixture() +def session_id(): + """A stable session ID for tests.""" + return "test-session-001" + + +@pytest.fixture() +def session_headers(session_id): + """Headers with session ID and content type.""" + return { + "X-Session-ID": session_id, + "Content-Type": "application/json", + } diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 00000000..6215521c --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,217 @@ +"""Integration tests for Flask routes.""" + +import json + + +def test_health(client): + resp = client.get("/api/health") + assert resp.status_code == 200 + assert resp.get_json()["status"] == "ok" + + +def test_init_missing_session_id(client): + resp = client.post("/api/init", json={}) + assert resp.status_code == 400 + assert "Missing X-Session-ID" in resp.get_json()["error"] + + +def test_init_creates_session(client, session_headers): + resp = client.post("/api/init", json={"packages": []}, headers=session_headers) + assert resp.status_code == 200 + data = resp.get_json() + assert data["type"] == "ready" + + +def test_exec_simple(client, session_headers): + # Init first + client.post("/api/init", json={"packages": []}, headers=session_headers) + + # Execute code + resp = client.post("/api/exec", json={"code": "x = 42"}, headers=session_headers) + assert resp.status_code == 200 + assert resp.get_json()["type"] == "ok" + + +def test_exec_with_print(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + resp = client.post( + "/api/exec", json={"code": "print('hello world')"}, headers=session_headers + ) + assert resp.status_code == 200 + data = resp.get_json() + assert data["type"] == "ok" + assert "hello world" in data.get("stdout", "") + + +def test_exec_error(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + resp = client.post( + "/api/exec", json={"code": "raise ValueError('test error')"}, headers=session_headers + ) + assert resp.status_code == 400 + data = resp.get_json() + assert data["type"] == "error" + assert "test error" in data["error"] + + +def test_eval_simple(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + # Set a variable, then eval it + client.post("/api/exec", json={"code": "y = 123"}, headers=session_headers) + + resp = client.post("/api/eval", json={"expr": "y"}, headers=session_headers) + assert resp.status_code == 200 + data = resp.get_json() + assert data["type"] == "value" + assert json.loads(data["value"]) == 123 + + +def test_eval_expression(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + resp = client.post("/api/eval", json={"expr": "2 + 3"}, headers=session_headers) + assert resp.status_code == 200 + data = resp.get_json() + assert json.loads(data["value"]) == 5 + + +def test_eval_error(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + resp = client.post( + "/api/eval", json={"expr": "undefined_var"}, headers=session_headers + ) + assert resp.status_code == 400 + data = resp.get_json() + assert data["type"] == "error" + + +def test_session_delete(client, session_headers): + client.post("/api/init", json={"packages": []}, headers=session_headers) + + resp = client.delete("/api/session", headers=session_headers) + assert resp.status_code == 200 + assert resp.get_json()["status"] == "terminated" + + +def test_session_persistence(client, session_headers): + """Variables set in one exec should be available in the next.""" + client.post("/api/init", json={"packages": []}, headers=session_headers) + + client.post("/api/exec", json={"code": "my_var = 'persistent'"}, headers=session_headers) + + resp = client.post("/api/eval", json={"expr": "my_var"}, headers=session_headers) + data = resp.get_json() + assert json.loads(data["value"]) == "persistent" + + +def test_streaming_lifecycle(client, session_headers): + """Test stream start → poll → done cycle.""" + client.post("/api/init", json={"packages": []}, headers=session_headers) + + # Set up a generator that yields one value then is done + client.post( + "/api/exec", + json={"code": "_step = 0\ndef _gen():\n global _step\n _step += 1\n return {'result': _step, 'done': _step >= 2}"}, + headers=session_headers, + ) + + # Start streaming + resp = client.post( + "/api/stream/start", + json={"expr": "_gen()"}, + headers=session_headers, + ) + assert resp.status_code == 200 + assert resp.get_json()["status"] == "started" + + # Poll until done + import time + done = False + messages = [] + deadline = time.time() + 10 + while not done and time.time() < deadline: + resp = client.post("/api/stream/poll", headers=session_headers) + data = resp.get_json() + messages.extend(data.get("messages", [])) + done = data.get("done", False) + if not done: + time.sleep(0.1) + + assert done + types = [m["type"] for m in messages] + assert "stream-data" in types + assert "stream-done" in types + + +def test_stream_stop_then_exec(client, session_headers): + """Regression: exec after stream/stop must not crash with JSONDecodeError. + + The stream reader thread must fully exit before exec reads stdout. + """ + client.post("/api/init", json={"packages": []}, headers=session_headers) + + # Set up a generator that never finishes on its own (needs manual stop) + client.post( + "/api/exec", + json={"code": "_counter = 0\ndef _infinite():\n global _counter\n _counter += 1\n return {'result': _counter, 'done': False}"}, + headers=session_headers, + ) + + # Start streaming + resp = client.post( + "/api/stream/start", + json={"expr": "_infinite()"}, + headers=session_headers, + ) + assert resp.status_code == 200 + + # Let a few polls go through + import time + for _ in range(5): + client.post("/api/stream/poll", headers=session_headers) + time.sleep(0.05) + + # Stop streaming + client.post("/api/stream/stop", headers=session_headers) + + # Poll until done + done = False + deadline = time.time() + 10 + while not done and time.time() < deadline: + resp = client.post("/api/stream/poll", headers=session_headers) + data = resp.get_json() + done = data.get("done", False) + if not done: + time.sleep(0.1) + + # Now exec should work without crashing + resp = client.post( + "/api/exec", + json={"code": "result_after_stop = 'ok'"}, + headers=session_headers, + ) + assert resp.status_code == 200 + assert resp.get_json()["type"] == "ok" + + # Verify the namespace is intact + resp = client.post( + "/api/eval", + json={"expr": "result_after_stop"}, + headers=session_headers, + ) + assert resp.status_code == 200 + assert json.loads(resp.get_json()["value"]) == "ok" + + +def test_exec_missing_session_id(client): + resp = client.post("/api/exec", json={"code": "pass"}) + assert resp.status_code == 400 + + +def test_eval_missing_session_id(client): + resp = client.post("/api/eval", json={"expr": "1"}) + assert resp.status_code == 400 diff --git a/tests/test_worker.py b/tests/test_worker.py new file mode 100644 index 00000000..8397720a --- /dev/null +++ b/tests/test_worker.py @@ -0,0 +1,160 @@ +"""Unit tests for the worker subprocess message protocol.""" + +import json +import subprocess +import sys +from pathlib import Path + +WORKER_SCRIPT = str(Path(__file__).parent.parent / "pathview_server" / "worker.py") + + +def _start_worker(): + """Start a worker subprocess and return it.""" + return subprocess.Popen( + [sys.executable, "-u", WORKER_SCRIPT], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + +def _send(proc, msg): + """Send a JSON message to the worker.""" + proc.stdin.write(json.dumps(msg) + "\n") + proc.stdin.flush() + + +def _recv(proc): + """Read one JSON response from the worker.""" + line = proc.stdout.readline() + if not line: + return None + return json.loads(line.strip()) + + +def _recv_until(proc, target_type, target_id=None): + """Read messages until we get the target type. Returns (target_msg, other_msgs).""" + others = [] + while True: + msg = _recv(proc) + if msg is None: + raise RuntimeError("Worker died unexpectedly") + if msg.get("type") == target_type and (target_id is None or msg.get("id") == target_id): + return msg, others + others.append(msg) + + +def test_init_ready(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + msg, _ = _recv_until(proc, "ready") + assert msg["type"] == "ready" + finally: + proc.kill() + proc.wait() + + +def test_exec_ok(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + _send(proc, {"type": "exec", "id": "e1", "code": "x = 10"}) + msg, _ = _recv_until(proc, "ok", "e1") + assert msg["type"] == "ok" + assert msg["id"] == "e1" + finally: + proc.kill() + proc.wait() + + +def test_exec_error(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + _send(proc, {"type": "exec", "id": "e2", "code": "1/0"}) + msg, _ = _recv_until(proc, "error", "e2") + assert msg["type"] == "error" + assert "ZeroDivisionError" in msg.get("traceback", "") + finally: + proc.kill() + proc.wait() + + +def test_eval_value(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + _send(proc, {"type": "exec", "id": "e3", "code": "z = 42"}) + _recv_until(proc, "ok", "e3") + + _send(proc, {"type": "eval", "id": "v1", "expr": "z"}) + msg, _ = _recv_until(proc, "value", "v1") + assert json.loads(msg["value"]) == 42 + finally: + proc.kill() + proc.wait() + + +def test_eval_error(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + _send(proc, {"type": "eval", "id": "v2", "expr": "undefined_variable"}) + msg, _ = _recv_until(proc, "error", "v2") + assert msg["type"] == "error" + assert "NameError" in msg.get("traceback", "") or "undefined_variable" in msg.get("error", "") + finally: + proc.kill() + proc.wait() + + +def test_exec_print_captured(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + _send(proc, {"type": "exec", "id": "e4", "code": "print('captured')"}) + msg, others = _recv_until(proc, "ok", "e4") + stdout_msgs = [m for m in others if m.get("type") == "stdout"] + assert any("captured" in m.get("value", "") for m in stdout_msgs) + finally: + proc.kill() + proc.wait() + + +def test_exec_before_init(): + proc = _start_worker() + try: + _send(proc, {"type": "exec", "id": "e5", "code": "x = 1"}) + msg, _ = _recv_until(proc, "error", "e5") + assert "not initialized" in msg["error"].lower() + finally: + proc.kill() + proc.wait() + + +def test_double_init(): + proc = _start_worker() + try: + _send(proc, {"type": "init"}) + _recv_until(proc, "ready") + + # Second init should just return ready immediately + _send(proc, {"type": "init"}) + msg, _ = _recv_until(proc, "ready") + assert msg["type"] == "ready" + finally: + proc.kill() + proc.wait()