From 87b5af1094fe5284f87e816b5c311cf3b653a70f Mon Sep 17 00:00:00 2001 From: Andrew Healey Date: Mon, 6 Oct 2025 09:14:32 +0100 Subject: [PATCH 1/6] add forth compiler post --- components/code.tsx | 1 + components/visuals/forth/components.tsx | 1031 +++++++++++++++++++++++ data/posts.ts | 1 + lib/prism-forth.js | 27 + pages/index.tsx | 2 +- posts/compiling-a-forth.md | 410 +++++++++ 6 files changed, 1471 insertions(+), 1 deletion(-) create mode 100644 components/visuals/forth/components.tsx create mode 100644 lib/prism-forth.js create mode 100644 posts/compiling-a-forth.md diff --git a/components/code.tsx b/components/code.tsx index aec0336..1f5d0f5 100644 --- a/components/code.tsx +++ b/components/code.tsx @@ -5,6 +5,7 @@ import Prism from "prism-react-renderer/prism"; require("prismjs/components/prism-rust") require("prismjs/components/prism-lua") require("prismjs/components/prism-lisp") +require("../lib/prism-forth") // -- import Highlight, { defaultProps, Language } from "prism-react-renderer"; diff --git a/components/visuals/forth/components.tsx b/components/visuals/forth/components.tsx new file mode 100644 index 0000000..04d10f2 --- /dev/null +++ b/components/visuals/forth/components.tsx @@ -0,0 +1,1031 @@ +import { useState, useEffect, ReactNode, CSSProperties } from 'react'; + +// Shared terminal styling +const terminalStyles = ` + .terminal { + --bg: #111827; + --fg: #e5e7eb; + --dim: #9ca3af; + background: var(--bg); + color: var(--fg); + font-family: "IBM Plex Mono", monospace; + font-size: 14px; + line-height: 1.25; + padding: 12px 14px; + border-radius: 0.4em; + border: 1px solid rgba(255, 255, 255, 0.08); + width: 100%; + max-width: 100%; + overflow: auto; + overscroll-behavior: contain; + } + .bytecode { + white-space: nowrap; + display: inline-block; + min-width: max-content; + } + .terminal :global(.ch) { + display: inline-block; + width: 1ch; + } + .terminal :global(.comment) { + color: var(--dim); + } + :global(.ch.hl) { + background: #e5e7eb !important; + color: #111827 !important; + border-radius: 2px; + } + :global(.ch.hl-start) { + background: #e5e7eb !important; + color: #111827 !important; + border-radius: 2px 0 0 2px; + } + :global(.ch.hl-mid) { + background: #e5e7eb !important; + color: #111827 !important; + } + :global(.ch.hl-end) { + background: #e5e7eb !important; + color: #111827 !important; + border-radius: 0 2px 2px 0; + } + .separator { + height: 1px; + background: rgba(255, 255, 255, 0.3); + margin: 12px 0; + } + .source-section, .tokens-section, .bytecode-section { + margin-top: 8px; + } + .tokens-label, .section-label { + color: var(--dim); + margin-bottom: 4px; + } + .token-line, .bytecode-line { + font-family: inherit; + line-height: 1.25; + min-height: 1.25em; + } + .token-line.highlighted, .bytecode-line.highlighted { + background: var(--fg); + color: var(--bg); + border-radius: 2px; + } +`; + +const fib10 = `VARIABLE A +VARIABLE B +VARIABLE T + + \\ calculates the nth Fibonacci number +: FIB \\ ( n -- f_n ) + 0 A ! + 1 B ! + 0 \\ start index + DO \\ ( limit start -- ) + A @ B @ + T ! + B @ A ! + T @ B ! + LOOP + A @ +; + +10 FIB . +` + +type LiteralToken = { + type: "literal"; + value: number; +} + +type SymbolToken = { + type: "store" | "load" | "dot" | "colon" | "semicolon" | "add" | "do" | "loop"; +} + +type IdentifierToken = { + type: "identifier"; + value: string; +} + +type Token = LiteralToken | SymbolToken | IdentifierToken; + +async function tokenize(source: string, callback: (highlight: { start: number, end: number }, tokens: Token[]) => Promise): Promise { + const tokens: Token[] = []; + + const isWhitespace = (c: string) => { + return c === " " || c === "\n" || c === "\t"; + } + const isDigit = (c: string) => { + return c >= "0" && c <= "9"; + } + const isLetter = (c: string) => { + return (c >= "a" && c <= "z") || (c >= "A" && c <= "Z") || c === "_"; + } + + let index = 0; + while (index < source.length) { + await callback({ start: index, end: index + 1 }, tokens); + + if (source[index] === "\\") { + const commentStart = index; + while (index < source.length && source[index] !== "\n") { + index++; + await callback({ start: commentStart, end: index }, tokens); + } + index++; + await callback({ start: commentStart, end: index }, tokens); + continue; + } + if (isWhitespace(source[index])) { + index++; + continue; + } + if (source[index] === ":") { + tokens.push({ type: "colon" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + if (source[index] === ";") { + tokens.push({ type: "semicolon" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + if (source[index] === ".") { + tokens.push({ type: "dot" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + if (source[index] === "!") { + tokens.push({ type: "store" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + if (source[index] === "@") { + tokens.push({ type: "load" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + if (isDigit(source[index])) { + const start = index; + let value = ""; + while (isDigit(source[index])) { + value += source[index]; + index++; + } + tokens.push({ type: "literal", value: parseInt(value) }); + await callback({ start, end: index }, tokens); + continue; + } + if (isLetter(source[index])) { + const start = index; + let value = ""; + while (isLetter(source[index])) { + value += source[index]; + index++; + } + + if (value === "DO") { + tokens.push({ type: "do" }); + await callback({ start, end: index }, tokens); + continue; + } + if (value === "LOOP") { + tokens.push({ type: "loop" }); + await callback({ start, end: index }, tokens); + continue; + } + + tokens.push({ type: "identifier", value }); + await callback({ start, end: index }, tokens); + continue; + } + if (source[index] === "+") { + tokens.push({ type: "add" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + + throw new Error(`Unexpected token at index ${index}: ${source[index]}`); + } + + return tokens; +} + +type Op = { + op: "lit", // Push value or address to DS + value: number; +} | { + op: "load", // Pop address from DS, push value at address +} | { + op: "store", // Pop address from DS, pop value from DS, store value at address +} | { + op: "dup2", // Duplicate top two values on DS [a, b] -> [a, b, a, b] +} | { + op: "add", // Pop top two values from DS, push sum to DS +} | { + op: "eq", // Pop top two values from DS, push 1 if equal, 0 if not +} | { + op: "jz", // Pop value from DS, if zero, jump to address + address: number; +} | { + op: "jmp", // Jump to address + address: number; +} | { + op: "call", // Push IP to RS, jump to address + address: number; +} | { + op: "ret", // Pop IP from RS, jump to IP +} | { + op: "rs_push", // Pop from DS, push to RS +} | { + op: "rs_pop", // Pop from RS, push to DS +} | { + op: "drop", // Discard top value from DS +} | { + op: "print", // Pop value from DS, print it +} + +type Bytecode = Op; +type Program = { + bytecode: Bytecode[]; + variableCount: number; +} + +async function compile(tokens: Token[], callback: (highlight: { tokenIdxStart: number, tokenIdxEnd: number }, ops: Bytecode[]) => Promise): Promise { + const bytecode: Bytecode[] = []; + const wordTable: { [key: string]: number } = {}; + const variableTable: { [key: string]: number } = {}; + let currentWordSkipOp: Bytecode | null = null; + let loopStart: number[] = []; + + let index = 0; + while (index < tokens.length) { + const token = tokens[index]; + if (token.type === "identifier") { + if (token.value === "VARIABLE") { + const nextToken = tokens[index + 1]; + if (nextToken.type !== "identifier") { + throw new Error(`Expected identifier after VARIABLE at index ${index}: ${JSON.stringify(nextToken)}`); + } + variableTable[nextToken.value] = Object.keys(variableTable).length; + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 2 }, bytecode); + index += 2; + continue; + } + + if (wordTable[token.value] !== undefined) { + bytecode.push({ op: "call", address: wordTable[token.value] }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + + bytecode.push({ op: "lit", value: variableTable[token.value] }); // Address + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + if (token.type === "colon") { + index++; + const nextToken = tokens[index]; + if (nextToken.type !== "identifier") { + throw new Error(`Expected identifier after colon at index ${index}: ${JSON.stringify(nextToken)}`); + } + if (currentWordSkipOp !== null) { + throw new Error(`Expected semicolon before next word at index ${index}: ${JSON.stringify(nextToken)}`); + } + + currentWordSkipOp = { op: "jmp", address: -1 }; + bytecode.push(currentWordSkipOp); + await callback({ tokenIdxStart: index - 1, tokenIdxEnd: index + 1 }, bytecode); + wordTable[nextToken.value] = bytecode.length; + index++; + continue; + } + if (token.type === "semicolon") { + if (currentWordSkipOp === null) { + throw new Error(`Expected word before semicolon at index ${index}: ${JSON.stringify(token)}`); + } + bytecode.push({ op: "ret" }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + currentWordSkipOp.address = bytecode.length; + currentWordSkipOp = null; + index++; + continue; + } + + if (token.type === "do") { + index++; + // Expect: DS has [limit, start] (start is top) + // Move both to RS: start then limit (RS top becomes limit) + bytecode.push({ op: "rs_push" }) // start -> RS + await callback({ tokenIdxStart: index - 1, tokenIdxEnd: index }, bytecode); + bytecode.push({ op: "rs_push" }) // limit -> RS + await callback({ tokenIdxStart: index - 1, tokenIdxEnd: index }, bytecode); + // Mark first instruction of loop body + loopStart.push(bytecode.length); + continue; + } + + if (token.type === "loop") { + if (loopStart.length === 0) { + throw new Error(`Loop without do at index ${index}: ${JSON.stringify(token)}`); + } + + // Pop limit and i from RS (RS top is limit) + bytecode.push({ op: "rs_pop" }) // limit -> DS + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "rs_pop" }) // i -> DS + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + // Increment i + bytecode.push({ op: "lit", value: 1 }) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "add" }) // i on DS + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + // Duplicate i and limit for compare and possible restore + bytecode.push({ op: "dup2" }) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "eq" }) // eq flag on DS + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + const loopStartAddress = loopStart.pop()!; // first instr of loop body + + // Branch: continue when not equal (eq==0), exit when equal + const continueAddress = bytecode.length + 4; // skip equal-path (2 drops + jmp) + bytecode.push({ op: "jz", address: continueAddress }) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + // Equal path (fallthrough): cleanup and exit + bytecode.push({ op: "drop" }) // drop i + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "drop" }) // drop limit + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + const afterBlockAddress = bytecode.length + 1 /* jmp */ + 3 /* continue block */; + bytecode.push({ op: "jmp", address: afterBlockAddress }) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + // Continue path: + // address == continueAddress + bytecode.push({ op: "rs_push" }) // i -> RS (top) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "rs_push" }) // limit -> RS + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "jmp", address: loopStartAddress }) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + index++; + continue; + } + + if (token.type === "literal") { + bytecode.push({ op: "lit", value: token.value }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + if (token.type === "store") { + bytecode.push({ op: "store" }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + if (token.type === "load") { + bytecode.push({ op: "load" }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + if (token.type === "add") { + bytecode.push({ op: "add" }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + if (token.type === "dot") { + bytecode.push({ op: "print" }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + + throw new Error(`Unexpected token at index ${index}: ${JSON.stringify(token)}`); + } + + return { bytecode, variableCount: Object.keys(variableTable).length }; +} + +async function vm(program: Program, callback: (highlight: { ip: number }, dataStack: number[], returnStack: number[], variableTable: number[]) => Promise) { + const dataStack: number[] = []; + const returnStack: number[] = []; + const variableTable: number[] = new Array(program.variableCount).fill(0); + + const dsPop = () => { + const value = dataStack.pop(); + if (value === undefined) { + throw new Error("Data stack underflow"); + } + return value; + } + const rsPop = () => { + const value = returnStack.pop(); + if (value === undefined) { + throw new Error("Return stack underflow"); + } + return value; + } + + let ip = 0; + while (ip < program.bytecode.length) { + const cur = program.bytecode[ip]; + + await callback({ ip }, dataStack, returnStack, variableTable); + + if (cur.op === "lit") { + dataStack.push(cur.value); + ip++; + continue; + } else if (cur.op === "store") { + const address = dsPop(); + const value = dsPop(); + variableTable[address] = value; + ip++; + continue; + } else if (cur.op === "load") { + const address = dsPop(); + dataStack.push(variableTable[address]); + ip++; + continue; + } else if (cur.op === "jmp") { + ip = cur.address; + continue; + } else if (cur.op === "jz") { + if (dsPop() === 0) { + ip = cur.address; + continue; + } + ip++; + continue; + } else if (cur.op === "call") { + ip++ + returnStack.push(ip); + ip = cur.address; + continue; + } else if (cur.op === "ret") { + ip = rsPop(); + continue; + } else if (cur.op === "rs_push") { + returnStack.push(dsPop()); + ip++; + continue; + } else if (cur.op === "rs_pop") { + dataStack.push(rsPop()); + ip++; + continue; + } else if (cur.op === "add") { + dataStack.push(dsPop() + dsPop()); + ip++; + continue; + } else if (cur.op === "dup2") { + if (dataStack.length < 2) { + throw new Error("Data stack underflow"); + } + const top = dataStack[dataStack.length - 1]; + const secondFromTop = dataStack[dataStack.length - 2]; + // Duplicate in-order: [a, b] -> [a, b, a, b] + dataStack.push(secondFromTop); + dataStack.push(top); + ip++; + continue; + } else if (cur.op === "eq") { + dataStack.push(dsPop() === dsPop() ? 1 : 0); + ip++; + continue; + } else if (cur.op === "drop") { + dsPop(); + ip++; + continue; + } else if (cur.op === "print") { + console.log(dsPop()); + ip++; + continue; + } + + // @ts-expect-error lints for missing branches + throw new Error(`Unknown opcode: ${cur.op}`); + } +} + +export function Tokenizer() { + const [highlight, setHighlight] = useState<{ start: number, end: number }>({ start: 0, end: 0 }); + const [tokens, setTokens] = useState([]); + const [isRunning, setIsRunning] = useState(false); + + const runTokenizer = async () => { + if (isRunning) return; + setIsRunning(true); + setTokens([]); + setHighlight({ start: 0, end: 0 }); + + try { + await tokenize(fib10, async (newHighlight, newTokens) => { + setHighlight(newHighlight); + setTokens([...newTokens]); + await new Promise(resolve => setTimeout(resolve, 150)); + }); + } catch (error) { + console.error('Tokenization error:', error); + } + + // Sleep for 2 seconds before allowing next run + await new Promise(resolve => setTimeout(resolve, 2000)); + setIsRunning(false); + }; + + // Auto-start tokenization loop + useEffect(() => { + const loop = async () => { + while (true) { + await runTokenizer(); + } + }; + loop(); + }, []); + + const charNodes: ReactNode[] = []; + let inComment = false; + + const getHighlightClass = (i: number) => { + if (i < highlight.start || i >= highlight.end) return ''; + if (highlight.end - highlight.start === 1) return ' hl'; + if (i === highlight.start) return ' hl-start'; + if (i === highlight.end - 1) return ' hl-end'; + return ' hl-mid'; + }; + + for (let i = 0; i < fib10.length; i++) { + const ch = fib10[i]; + + if (ch === '\n') { + inComment = false; // Reset comment flag on new line + charNodes.push(
); + continue; + } + + if (ch === '\\') { + inComment = true; // Start comment from backslash + } + + const highlightClass = getHighlightClass(i); + const commentClass = inComment ? ' comment' : ''; + const classes = `ch${highlightClass}${commentClass}`; + + let inlineStyle: CSSProperties = { + display: 'inline-block', + width: '1ch' + }; + + if (inComment) { + inlineStyle.color = '#9ca3af'; + } + + if (highlightClass) { + inlineStyle.background = '#e5e7eb'; + inlineStyle.color = '#111827'; + + if (highlightClass === ' hl') { + inlineStyle.borderRadius = '2px'; + } else if (highlightClass === ' hl-start') { + inlineStyle.borderRadius = '2px 0 0 2px'; + } else if (highlightClass === ' hl-end') { + inlineStyle.borderRadius = '0 2px 2px 0'; + } + // hl-mid has no border radius + } + + charNodes.push( + {ch === ' ' ? '\u00A0' : ch} + ); + } + + // Get the latest 5 tokens for display, pad with empty slots if needed + const latestTokens = tokens.slice(-5); + + const renderTokenLine = (i: number) => { + if (i < latestTokens.length) { + const token = latestTokens[i]; + const tokenIndex = tokens.length - latestTokens.length + i; + let tokenStr = ''; + let prefix = ''; + if (token.type === 'literal') { + tokenStr = token.value.toString(); + prefix = 'literal'; + } else if (token.type === 'identifier') { + tokenStr = token.value; + prefix = 'identifier'; + } else { + tokenStr = token.type.toUpperCase(); + prefix = 'symbol'; + } + + // Fixed-width layout for alignment + const indexPart = `${tokenIndex.toString().padStart(2, '\u00A0')}:`; + const valuePart = `\u00A0${tokenStr}`; + const prefixPart = `(${prefix})`; + const totalBeforePrefix = indexPart.length + valuePart.length; + const paddingNeeded = Math.max(0, 14 - totalBeforePrefix); + const padding = '\u00A0'.repeat(paddingNeeded); + + return ( +
+ {indexPart} + {valuePart} + {padding} + {prefixPart} +
+ ); + } else { + return ( +
+ {i.toString().padStart(2, '\u00A0')}: +
+ ); + } + }; + + return ( +
+
+
Source Code
+
{charNodes}
+
+
+
+
Tokens
+ {Array.from({ length: 5 }, (_, i) => renderTokenLine(i))} +
+ +
+ ); +} + +export function Compiler() { + const [highlightRange, setHighlightRange] = useState<{ start: number, end: number }>({ start: -1, end: -1 }); + const [tokens, setTokens] = useState([]); + const [bytecode, setBytecode] = useState([]); + const [isRunning, setIsRunning] = useState(false); + + const runCompiler = async () => { + if (isRunning) return; + setIsRunning(true); + setTokens([]); + setBytecode([]); + setHighlightRange({ start: -1, end: -1 }); + + try { + // First tokenize the source + const allTokens = await tokenize(fib10, async () => { + // No-op callback for tokenization, just need the tokens + }); + + // Then compile with highlighting + await compile(allTokens, async (highlight, newBytecode) => { + setHighlightRange({ start: highlight.tokenIdxStart, end: highlight.tokenIdxEnd }); + setTokens([...allTokens]); + setBytecode([...newBytecode]); + await new Promise(resolve => setTimeout(resolve, 500)); + }); + } catch (error) { + console.error('Compilation error:', error); + } + + // Sleep for 2 seconds before next run + await new Promise(resolve => setTimeout(resolve, 2000)); + setIsRunning(false); + }; + + useEffect(() => { + const loop = async () => { + while (true) { + await runCompiler(); + } + }; + loop(); + }, []); + + // Calculate which tokens to show + const visibleTokenCount = 10; + let startIndex = 0; + + // If we have a highlighted token and it would be below the visible area, scroll + if (highlightRange.start >= 0 && tokens.length > 0) { + const maxVisibleIndex = startIndex + visibleTokenCount - 1; + if (highlightRange.start > maxVisibleIndex) { + startIndex = Math.max(0, highlightRange.start - visibleTokenCount + 1); + } + } + + const visibleTokens = tokens.slice(startIndex, startIndex + visibleTokenCount); + + const renderTokenLine = (i: number) => { + if (i < visibleTokens.length) { + const token = visibleTokens[i]; + const tokenIndex = startIndex + i; + const isHighlighted = tokenIndex >= highlightRange.start && tokenIndex < highlightRange.end; + + let tokenStr = ''; + let prefix = ''; + if (token.type === 'literal') { + tokenStr = token.value.toString(); + prefix = 'literal'; + } else if (token.type === 'identifier') { + tokenStr = token.value; + prefix = 'identifier'; + } else { + tokenStr = token.type.toUpperCase(); + prefix = 'symbol'; + } + + // Fixed-width layout for alignment + const indexPart = `${tokenIndex.toString().padStart(2, '\u00A0')}:`; + const valuePart = `\u00A0${tokenStr}`; + const prefixPart = `(${prefix})`; + const totalBeforePrefix = indexPart.length + valuePart.length; + const paddingNeeded = Math.max(0, 13 - totalBeforePrefix); + const padding = '\u00A0'.repeat(paddingNeeded); + + return ( +
+ {indexPart} + {valuePart} + {padding} + {prefixPart} +
+ ); + } else { + return ( +
+ {(startIndex + i).toString().padStart(2, '\u00A0')}: +
+ ); + } + }; + + // Latest bytecode ops + const latestBytecode = bytecode.slice(-10); + const renderBytecodeLine = (i: number) => { + if (i < latestBytecode.length) { + const op = latestBytecode[i]; + const opIndex = bytecode.length - latestBytecode.length + i; + + let opStr = ''; + if (op.op === 'lit') { + opStr = `lit ${op.value}`; + } else if (op.op === 'jz') { + opStr = `jz ${op.address}`; + } else if (op.op === 'jmp') { + opStr = `jmp ${op.address}`; + } else if (op.op === 'call') { + opStr = `call ${op.address}`; + } else { + opStr = op.op; + } + + const indexPart = `${opIndex.toString().padStart(2, '\u00A0')}:`; + const valuePart = `\u00A0${opStr}`; + + return ( +
+ {indexPart} + {valuePart} +
+ ); + } else { + return ( +
+ {i.toString().padStart(2, '\u00A0')}: +
+ ); + } + }; + + return ( +
+
+
Tokens
+ {Array.from({ length: 10 }, (_, i) => renderTokenLine(i))} +
+
+
+
Bytecode
+ {Array.from({ length: 10 }, (_, i) => renderBytecodeLine(i))} +
+ +
+ ); +} + +export function VM() { + const [highlightIP, setHighlightIP] = useState(-1); + const [bytecode, setBytecode] = useState([]); + const [dataStack, setDataStack] = useState([]); + const [returnStack, setReturnStack] = useState([]); + const [variableTable, setVariableTable] = useState([]); + const [isRunning, setIsRunning] = useState(false); + + const runVM = async () => { + if (isRunning) return; + setIsRunning(true); + setBytecode([]); + setDataStack([]); + setReturnStack([]); + setVariableTable([]); + setHighlightIP(-1); + + try { + // First tokenize and compile to get the program + const allTokens = await tokenize(fib10, async () => { + // No-op + }); + + const program = await compile(allTokens, async () => { + // No-op + }); + + setBytecode(program.bytecode); + + // Run the VM with highlighting + await vm(program, async (highlight, newDataStack, newReturnStack, newVariableTable) => { + setHighlightIP(highlight.ip); + setDataStack([...newDataStack]); + setReturnStack([...newReturnStack]); + setVariableTable([...newVariableTable]); + await new Promise(resolve => setTimeout(resolve, 750)); + }); + } catch (error) { + console.error('VM execution error:', error); + } + + // Sleep for 2 seconds before next run + await new Promise(resolve => setTimeout(resolve, 2000)); + setIsRunning(false); + }; + + useEffect(() => { + const loop = async () => { + while (true) { + await runVM(); + } + }; + loop(); + }, []); + + // Calculate which bytecode lines to show (scrolled around IP) + const visibleBytecodeCount = 15; + let startIndex = 0; + + // If we have a highlighted IP and it would be below the visible area, scroll + if (highlightIP >= 0 && bytecode.length > 0) { + const maxVisibleIndex = startIndex + visibleBytecodeCount - 1; + if (highlightIP > maxVisibleIndex) { + startIndex = Math.max(0, highlightIP - Math.floor(visibleBytecodeCount / 2)); + } else if (highlightIP < startIndex) { + startIndex = Math.max(0, highlightIP - Math.floor(visibleBytecodeCount / 2)); + } + // Ensure we don't scroll past the end! + if (startIndex + visibleBytecodeCount > bytecode.length) { + startIndex = Math.max(0, bytecode.length - visibleBytecodeCount); + } + } + + const visibleBytecode = bytecode.slice(startIndex, startIndex + visibleBytecodeCount); + + const renderBytecodeLine = (i: number) => { + if (i < visibleBytecode.length) { + const op = visibleBytecode[i]; + const opIndex = startIndex + i; + const isHighlighted = opIndex === highlightIP; + + let opStr = ''; + if (op.op === 'lit') { + opStr = `lit ${op.value}`; + } else if (op.op === 'jz') { + opStr = `jz ${op.address}`; + } else if (op.op === 'jmp') { + opStr = `jmp ${op.address}`; + } else if (op.op === 'call') { + opStr = `call ${op.address}`; + } else { + opStr = op.op; + } + + const indexPart = `${opIndex.toString().padStart(2, '\u00A0')}:`; + const valuePart = `\u00A0${opStr}`; + + return ( +
+ {indexPart} + {valuePart} +
+ ); + } else { + return ( +
+ {(startIndex + i).toString().padStart(2, '\u00A0')}: +
+ ); + } + }; + + // Format stack display (show up to 5 items, most recent at top) + const formatStack = (stack: number[], name: string) => { + const displayStack = stack.slice(-5).reverse(); // Show last 5, most recent first + const lines: ReactNode[] = []; + + for (let i = 0; i < 5; i++) { + if (i < displayStack.length) { + const value = displayStack[i]; + const stackIndex = stack.length - 1 - i; + lines.push( +
+ {stackIndex.toString().padStart(2, ' ')}: + {value} +
+ ); + } else { + lines.push( +
+ +
+ ); + } + } + + return lines; + }; + + // Format variable table display + const formatVariables = () => { + const lines: ReactNode[] = []; + + for (let i = 0; i < variableTable.length; i++) { + const value = variableTable[i]; + lines.push( +
+ {i.toString().padStart(2, ' ')}: + {value} +
+ ); + } + + return lines; + }; + + return ( +
+
+
Bytecode
+ {Array.from({ length: visibleBytecodeCount }, (_, i) => renderBytecodeLine(i))} +
+
+
+
VM State
+
+
+
Data Stack
+ {formatStack(dataStack, 'DS')} +
+
+
Return Stack
+ {formatStack(returnStack, 'RS')} +
+
+
Variables
+ {formatVariables()} +
+
+
+ +
+ ); +} + +// When this file is ran with `bun` this outputs the 10th Fibonacci number (55) +if ('Bun' in globalThis && globalThis.Bun.main.includes('forth')) { + tokenize(fib10, async (_, __) => { /* no-op */ }) + .then(tokens => { + compile(tokens, async (_, __) => { /* no-op */ }) + .then(program => { + vm(program, async (_, __, ___, ____) => { /* no-op */ }); + }); + }); +} diff --git a/data/posts.ts b/data/posts.ts index f958754..2f6fb7a 100644 --- a/data/posts.ts +++ b/data/posts.ts @@ -7,6 +7,7 @@ export const popularPosts = [ // Starred posts (not in any specific order) export const postStars = [ + "compiling-a-forth", "counting-words-at-simd-speed", "optimizing-my-disk-usage-program", "maybe-the-fastest-disk-usage-program-on-macos", diff --git a/lib/prism-forth.js b/lib/prism-forth.js new file mode 100644 index 0000000..47a135a --- /dev/null +++ b/lib/prism-forth.js @@ -0,0 +1,27 @@ +(function (Prism) { + Prism.languages.forth = { + "comment": { + pattern: /\\\s.*$/m, + greedy: true, + }, + "string": { + pattern: /"(?:[^"\\]|\\.)*"/, + greedy: true, + }, + "number": /\b-?\d+(?:\.\d+)?\b/, + "keyword": + /\b(?:VARIABLE|DO|LOOP|IF|THEN|ELSE|BEGIN|UNTIL|WHILE|REPEAT)\b/i, + "builtin": + /\b(?:@|!|\+|-|\*|\/|MOD|=|<|>|AND|OR|NOT|DUP|DROP|SWAP|OVER|ROT|\.)\b/, + "definition": { + pattern: /(:)\s+(\S+)/, + lookbehind: true, + inside: { + "keyword": /^:/, + "function": /\S+/, + }, + }, + "variable": /\b[A-Z]+\b/, + "punctuation": /[;]/, + }; +})(Prism); diff --git a/pages/index.tsx b/pages/index.tsx index 0b15960..91a46c8 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -72,7 +72,7 @@ export default function Home({ allPostsData, description, words }) { My experimental package manager uses simple concurrency patterns to be faster than every package manager aside from Bun (which is 11% faster) when cold-installing from a lockfile.

- I've created a few small programming languages and related tools, including a Lisp-to-JavaScript compiler, which I turned into an optimizing compiler, and for which I wrote a bytecode VM. + I've created a few small programming languages and related tools, including a Forth compiler, a Lisp-to-JavaScript compiler, which I turned into an optimizing compiler, and for which I wrote a bytecode VM. I also built an interpreted language with a C-style syntax, which I profiled and made faster; I later added a WebAssembly compiler and a line profiler. I also ported an expression engine to Rust.

diff --git a/posts/compiling-a-forth.md b/posts/compiling-a-forth.md new file mode 100644 index 0000000..b329262 --- /dev/null +++ b/posts/compiling-a-forth.md @@ -0,0 +1,410 @@ +--- +title: "Compiling a Forth" +date: "2025-10-06" +tags: ["forth"] +description: "A bytecode compiler and VM for a Forth-like language." +--- + +I was curious how Forth worked so I built a bytecode compiler and a VM for a Forth-like language, as well as some visualizations to show how it all works. + +You don't need to know anything about Forth to follow along, aside from the fact it's a stack-oriented language. + +Here's a small program that prints the number three. + +```forth +3 . +``` + +The number (`3`) is pushed to the data stack, and then the dot (`.`) pops it from the data stack and prints it. + +We'll need more Forth features than this to build interesting programs. + +Forth has two built-in stacks. The data stack (sometimes just called "the stack") and the return stack. When a word is called in Forth (words are like functions) the address of the next instruction is pushed to the return stack. When the word finishes executing, the return stack is popped into the instruction pointer. + +```forth +\ (1) word declaration +: PRINT10 + + \ (3) the word body is executed + 10 . + + \ (4) ";" compiles an exit – at runtime it pops the return stack + \ into the instruction pointer. +; + +\ (2) instruction pointer lands on a word, +\ the next address is pushed to the return stack, +\ and the instruction pointer is set to the word address +PRINT10 + +\ (5) next address is executed +``` + +As well as words, my compiler also supports `DO`/`LOOP`s. These use the return stack too. When `DO` executes, it pops the limit and the iterator from the data stack and stores them in the return stack. This allows the inner loop to freely operate on the data stack. When `LOOP` executes, it pops the limit and iterator from the return stack, adds one to the iterator and compares it to the limit (and exits or loops again). + +There are also variables, which can be declared with `VARIABLE X`, loaded with `X @`, and stored with `1 X !`. + +Putting these features together, here's how you can build `10` by adding `1` repeatedly. + +```forth +VARIABLE A + +: RUN + 0 A ! \ initialize A + 10 0 DO \ push limit and iterator for DO + \ DO places these on the return stack + A @ 1 + A ! \ A = A + 1 + LOOP \ increment i and exits when i == limit + A @ . \ prints 10 +; + +RUN +``` + +This set of features is enough for us to calculate numbers from the Fibonacci series, which is the example program I'll be using throughout the rest of this post. + +## Tokenizing + +Tokenization translates raw text into meaningful symbols. + +To turn source code into tokens, we scan through the code, skipping over whitespace and appending tokens to a list. Syntax that's a single character is turned straight into a token but multi-character syntax needs to be grouped together. For example, entire comments are discarded, and while they are being discarded, we need to track that we're "within" a comment. + +Identifiers, like keywords like `DO` or `LOOP`, or custom variables like `MYLONGVAR`, become single tokens. + +First, a visualization of what's happening: + +

+ +And here's a trimmed version of my tokenizer: + +```tsx +function tokenize(source: string): Token[] { + const tokens: Token[] = []; + + let index = 0; + while (index < source.length) { + + // Consume and discard everything on a line after '\' + if (source[index] === "\\") { + const commentStart = index; + while (index < source.length && source[index] !== "\n") { + index++; + } + index++; + continue; + } + + // Skip over whitespace + if (isWhitespace(source[index])) { + index++; + continue; + } + + if (source[index] === "@") { + tokens.push({ type: "load" }); + index++; + continue; + } + + // Handle identifiers + if (isLetter(source[index])) { + const start = index; + let value = ""; + while (isLetter(source[index])) { + value += source[index]; + index++; + } + + // Special-case the keywords + if (value === "DO") { + tokens.push({ type: "do" }); + continue; + } + if (value === "LOOP") { + tokens.push({ type: "loop" }); + continue; + } + + tokens.push({ type: "identifier", value }); + continue; + } + + // .. trimmed other tokens, see source + } + + return tokens; +} +``` + +With our list of tokens, we're ready to start generating bytecode for the VM. + +## Generating Bytecode + +Usually, in a compiler, the step after tokenization is _parsing_ where an abstract syntax tree is built. However, the feature set of my Forth is so small, that I decided to generate bytecode directly from the list of tokens. + +_After_ bytecode generation, my VM needs two things: + +- A list of operations for the VM's instruction pointer to navigate +- The number of variables that the program refers to + +The latter tells the VM how many variables to allocate (a zero-initialized array). Variables in source (e.g., `A`, `B`) become integer indices into this array. + +This means that my bytecode generation step needs to keep track of variables that have been seen before so that I can output the correct memory address (i.e. an index into the variable table). + +I'll show the full list of bytecode operations and then a few of the steps for handling specific tokens. + +```tsx +type Op = { + op: "lit", // Push value or address to DS + value: number; +} | { + op: "load", // Pop address from DS, push value at address +} | { + op: "store", // Pop address from DS, pop value from DS, store value at address +} | { + op: "dup2", // Duplicate top two values on DS [a, b] -> [a, b, a, b] +} | { + op: "add", // Pop top two values from DS, push sum to DS +} | { + op: "eq", // Pop top two values from DS, push 1 if equal, 0 if not +} | { + op: "jz", // Pop value from DS, if zero, jump to address + address: number; +} | { + op: "jmp", // Jump to address + address: number; +} | { + op: "call", // Push IP to RS, jump to address + address: number; +} | { + op: "ret", // Pop IP from RS, jump to IP +} | { + op: "rs_push", // Pop from DS, push to RS +} | { + op: "rs_pop", // Pop from RS, push to DS +} | { + op: "drop", // Discard top value from DS +} | { + op: "print", // Pop value from DS, print it +} +``` + +The bytecode generation step scans through the list of tokens and, as it processes them, it appends to a list of bytecode and increments the variable count to set up the correct references. + +Identifier tokens are either variable references, or words (function calls). + +```tsx +let index = 0; +while (index < tokens.length) { + const token = tokens[index]; + + if (token.type === "identifier") { + if (token.value === "VARIABLE") { + const nextToken = tokens[index + 1]; + + // Store a binding of variable name to memory address + variableTable[nextToken.value] = Object.keys(variableTable).length; + index += 2; + continue; + } + + // If the variable has been declared as a word like `: FIB10` + // then we have previously stored the bytecode offset which we + // will set the instruction pointer to at runtime + if (wordTable[token.value] !== undefined) { + bytecode.push({ op: "call", address: wordTable[token.value] }); + index++; + continue; + } + + // If it's not a variable declaration, or a word, then we + // look up the memory address + bytecode.push({ op: "lit", value: variableTable[token.value] }); + index++; + continue; + } + + // .. +} +``` + +Setting up the `DO`/`LOOP` bytecode generation was the trickiest part of this project. It's a minefield of possible off-by-one errors. It's also not easy to read and understand but I've chosen to put it here anyway because even just glancing over it should help you understand how the loop variables (limit, iterator) and instruction pointer jumps are combined to execute loops in Forth. + +```tsx + // .. + + if (token.type === "do") { + index++; + + // Expect: DS has [limit, start] (start is top) + // Move both to RS: start then limit (RS top becomes limit) + bytecode.push({ op: "rs_push" }) // start -> RS + bytecode.push({ op: "rs_push" }) // limit -> RS + + // Mark first instruction of loop body + loopStart.push(bytecode.length); + continue; + } + + if (token.type === "loop") { + + // Pop limit and i from RS (RS top is limit) + bytecode.push({ op: "rs_pop" }) // limit -> DS + bytecode.push({ op: "rs_pop" }) // i -> DS + + // Increment i + bytecode.push({ op: "lit", value: 1 }) + bytecode.push({ op: "add" }) // i on DS + + // Duplicate i and limit for compare and possible restore + bytecode.push({ op: "dup2" }) + bytecode.push({ op: "eq" }) // eq flag on DS + + const loopStartAddress = loopStart.pop(); // first instr of loop body + + // Branch: continue when not equal (eq==0), exit when equal + const continueAddress = bytecode.length + 4; // skip equal-path (2 drops + jmp) + bytecode.push({ op: "jz", address: continueAddress }) + + // Equal path (fallthrough): cleanup and exit + bytecode.push({ op: "drop" }) // drop i + bytecode.push({ op: "drop" }) // drop limit + const afterBlockAddress = bytecode.length + 1 /* jmp */ + 3 /* continue block */; + bytecode.push({ op: "jmp", address: afterBlockAddress }) + + // Continue path: + // address == continueAddress + bytecode.push({ op: "rs_push" }) // i -> RS (top) + bytecode.push({ op: "rs_push" }) // limit -> RS + bytecode.push({ op: "jmp", address: loopStartAddress }) + + index++; + continue; + } + + // .. +``` + +The rest of the token branches are more straightforward. Tokens like dot, store, load, and print all map directly to bytecode operations. + +The colon token branch sets the bytecode offset for the word name which allows identifiers to become word calls as we saw above. + +Now we've earned a visualization break. + +
+ +## VM + +Writing the VM felt a little bit like dessert. Manually stepping through the bytecode as I worked on the generation logic gave me fairly good confidence that I was heading in the right direction, I only came across one or two off-by-one bugs when putting the VM together. Essentially, I had designed it ahead-of-time. + +The VM scans through the bytecode operations using the instruction pointer (which starts at `0`). The instruction pointer can jump around as it encounters `jmp` (jump to offset) or `jz` (conditional jump). + +It manages the data stack, return stack, and the variable table (i.e. memory addresses). + +
+ +Here's a trimmed version of the VM: + +```tsx +function vm(program: Program) => { + const dataStack: number[] = []; + const returnStack: number[] = []; + const variableTable: number[] = new Array(program.variableCount).fill(0); + + let ip = 0; + while (ip < program.bytecode.length) { + const cur = program.bytecode[ip]; + + if (cur.op === "lit") { + dataStack.push(cur.value); // Literal or memory address + ip++; + continue; + } else if (cur.op === "store") { + const address = dsPop(); + const value = dsPop(); + variableTable[address] = value; + ip++; + continue; + } else if (cur.op === "jmp") { + ip = cur.address; + continue; + } else if (cur.op === "jz") { + if (dsPop() === 0) { + ip = cur.address; + continue; + } + ip++; + continue; + } else if (cur.op === "call") { + ip++ + returnStack.push(ip); + ip = cur.address; + continue; + } else if (cur.op === "ret") { + ip = rsPop(); + continue; + } + + // .. trimmed other ops, see source + } +} +``` + +The code for my compiler and VM are [embedded in this website](https://github.com/healeycodes/healeycodes.com/blob/main/components/visuals/forth/components.tsx). I've been iterating on it by just running the TypeScript file: + +```bash +bun ./components/visuals/forth/components.tsx +55 # 10th Fibonacci number +``` + +The visuals are React components with sleeps. In order to display the progress of the different steps (tokenizing, bytecode generation, VM), I first got each working and then added a callback which takes the current data and then sleeps. + +So the VM function is actually async and accepts this callback: + +```tsx +// VM +async function vm(program: Program, callback: + ( + highlight: { ip: number }, + dataStack: number[], + returnStack: number[], + variableTable: number[] + ) => Promise) { + + // .. inside VM loop + await callback({ ip }, dataStack, returnStack, variableTable); + // .. + +} +``` + +And the component calls it and passes `setState` functions: + +```tsx +// Component +export function VM() { + + // .. inside useEffect + await vm(program, async (highlight, newDataStack, newReturnStack, newVariableTable) => { + setHighlightIP(highlight.ip); + setDataStack([...newDataStack]); + setReturnStack([...newReturnStack]); + setVariableTable([...newVariableTable]); + await new Promise(resolve => setTimeout(resolve, 500)); + }); + // .. + +} +``` + +For the Forth code snippets in this post, I had to write [a Prism plugin](https://github.com/healeycodes/healeycodes.com/blob/main/lib/prism-forth.js) to get syntax highlighting working. Now that I've learned how to do this, I'll be using this method for syntax highlighting for the more esoteric (or, original) programming languages I write about! + +## Discrepancies + +I described my compiler/VM as _Forth-like_ because it's a little bit different from how Forth works. + +My implementation compiles to bytecode ahead-of-time. Forth is traditionally interactive. Words are interpreted and executed as they are entered, and only colon definitions are compiled. Forth uses threaded code where words contain lists of addresses pointing to other words instead of a different bytecode offset. + +Real Forth uses a dynamic dictionary that can be altered at runtime with new variables or word definitions. As I mentioned earlier, my word bodies are compiled with jump-over logic in the main execution stream. Also, my variables compile to `lit address` operations but real Forth variables return their address when executed directly. + +These are just a few of the differences but I feel like my Forth-like compiler and VM capture enough of the spirit of Forth! From 2d55941943d06bcc56aff58bd2788d277c689ca2 Mon Sep 17 00:00:00 2001 From: Andrew Healey Date: Mon, 6 Oct 2025 11:44:57 +0100 Subject: [PATCH 2/6] fix useEffect --- components/visuals/forth/components.tsx | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/components/visuals/forth/components.tsx b/components/visuals/forth/components.tsx index 04d10f2..aefcb29 100644 --- a/components/visuals/forth/components.tsx +++ b/components/visuals/forth/components.tsx @@ -553,12 +553,16 @@ export function Tokenizer() { // Auto-start tokenization loop useEffect(() => { + let shouldContinue = true; const loop = async () => { - while (true) { + while (shouldContinue) { await runTokenizer(); } }; loop(); + return () => { + shouldContinue = false; + }; }, []); const charNodes: ReactNode[] = []; @@ -714,10 +718,16 @@ export function Compiler() { }; useEffect(() => { + let shouldContinue = true; const loop = async () => { - while (true) { - await runCompiler(); + while (shouldContinue) { + while (true) { + await runCompiler(); + } } + return () => { + shouldContinue = false; + }; }; loop(); }, []); @@ -881,10 +891,14 @@ export function VM() { }; useEffect(() => { + let shouldContinue = true; const loop = async () => { - while (true) { + while (shouldContinue) { await runVM(); } + return () => { + shouldContinue = false; + }; }; loop(); }, []); From c5c928d4fe5ef491a786444a9800a253a21587f9 Mon Sep 17 00:00:00 2001 From: Andrew Healey Date: Mon, 6 Oct 2025 13:06:03 +0100 Subject: [PATCH 3/6] better cancellation --- components/visuals/forth/components.tsx | 70 +++++++++++++++---------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/components/visuals/forth/components.tsx b/components/visuals/forth/components.tsx index aefcb29..29c2a14 100644 --- a/components/visuals/forth/components.tsx +++ b/components/visuals/forth/components.tsx @@ -530,14 +530,15 @@ export function Tokenizer() { const [tokens, setTokens] = useState([]); const [isRunning, setIsRunning] = useState(false); - const runTokenizer = async () => { - if (isRunning) return; + const runTokenizer = async (shouldStop: () => boolean) => { + if (isRunning || shouldStop()) return; setIsRunning(true); setTokens([]); setHighlight({ start: 0, end: 0 }); try { await tokenize(fib10, async (newHighlight, newTokens) => { + if (shouldStop()) return; setHighlight(newHighlight); setTokens([...newTokens]); await new Promise(resolve => setTimeout(resolve, 150)); @@ -547,21 +548,26 @@ export function Tokenizer() { } // Sleep for 2 seconds before allowing next run - await new Promise(resolve => setTimeout(resolve, 2000)); + if (!shouldStop()) { + await new Promise(resolve => setTimeout(resolve, 2000)); + } setIsRunning(false); }; // Auto-start tokenization loop useEffect(() => { - let shouldContinue = true; + // Only run in browser environment + if (typeof window === 'undefined') return; + + let cancelled = false; const loop = async () => { - while (shouldContinue) { - await runTokenizer(); + while (!cancelled) { + await runTokenizer(() => cancelled); } }; loop(); return () => { - shouldContinue = false; + cancelled = true; }; }, []); @@ -688,8 +694,8 @@ export function Compiler() { const [bytecode, setBytecode] = useState([]); const [isRunning, setIsRunning] = useState(false); - const runCompiler = async () => { - if (isRunning) return; + const runCompiler = async (shouldStop: () => boolean) => { + if (isRunning || shouldStop()) return; setIsRunning(true); setTokens([]); setBytecode([]); @@ -703,6 +709,7 @@ export function Compiler() { // Then compile with highlighting await compile(allTokens, async (highlight, newBytecode) => { + if (shouldStop()) return; setHighlightRange({ start: highlight.tokenIdxStart, end: highlight.tokenIdxEnd }); setTokens([...allTokens]); setBytecode([...newBytecode]); @@ -713,23 +720,26 @@ export function Compiler() { } // Sleep for 2 seconds before next run - await new Promise(resolve => setTimeout(resolve, 2000)); + if (!shouldStop()) { + await new Promise(resolve => setTimeout(resolve, 2000)); + } setIsRunning(false); }; useEffect(() => { - let shouldContinue = true; + // Only run in browser environment + if (typeof window === 'undefined') return; + + let cancelled = false; const loop = async () => { - while (shouldContinue) { - while (true) { - await runCompiler(); - } + while (!cancelled) { + await runCompiler(() => cancelled); } - return () => { - shouldContinue = false; - }; }; loop(); + return () => { + cancelled = true; + }; }, []); // Calculate which tokens to show @@ -852,8 +862,8 @@ export function VM() { const [variableTable, setVariableTable] = useState([]); const [isRunning, setIsRunning] = useState(false); - const runVM = async () => { - if (isRunning) return; + const runVM = async (shouldStop: () => boolean) => { + if (isRunning || shouldStop()) return; setIsRunning(true); setBytecode([]); setDataStack([]); @@ -875,6 +885,7 @@ export function VM() { // Run the VM with highlighting await vm(program, async (highlight, newDataStack, newReturnStack, newVariableTable) => { + if (shouldStop()) return; setHighlightIP(highlight.ip); setDataStack([...newDataStack]); setReturnStack([...newReturnStack]); @@ -886,21 +897,26 @@ export function VM() { } // Sleep for 2 seconds before next run - await new Promise(resolve => setTimeout(resolve, 2000)); + if (!shouldStop()) { + await new Promise(resolve => setTimeout(resolve, 2000)); + } setIsRunning(false); }; useEffect(() => { - let shouldContinue = true; + // Only run in browser environment + if (typeof window === 'undefined') return; + + let cancelled = false; const loop = async () => { - while (shouldContinue) { - await runVM(); + while (!cancelled) { + await runVM(() => cancelled); } - return () => { - shouldContinue = false; - }; }; loop(); + return () => { + cancelled = true; + }; }, []); // Calculate which bytecode lines to show (scrolled around IP) From 9669b072832b630d29f0fa06372aec073cfa8072 Mon Sep 17 00:00:00 2001 From: Andrew Healey Date: Mon, 6 Oct 2025 13:39:54 +0100 Subject: [PATCH 4/6] add debug logs --- components/visuals/forth/components.tsx | 351 +++++++++++++----------- 1 file changed, 198 insertions(+), 153 deletions(-) diff --git a/components/visuals/forth/components.tsx b/components/visuals/forth/components.tsx index 29c2a14..666f014 100644 --- a/components/visuals/forth/components.tsx +++ b/components/visuals/forth/components.tsx @@ -525,52 +525,65 @@ async function vm(program: Program, callback: (highlight: { ip: number }, dataSt } } -export function Tokenizer() { - const [highlight, setHighlight] = useState<{ start: number, end: number }>({ start: 0, end: 0 }); - const [tokens, setTokens] = useState([]); - const [isRunning, setIsRunning] = useState(false); - - const runTokenizer = async (shouldStop: () => boolean) => { - if (isRunning || shouldStop()) return; - setIsRunning(true); - setTokens([]); - setHighlight({ start: 0, end: 0 }); - - try { - await tokenize(fib10, async (newHighlight, newTokens) => { - if (shouldStop()) return; - setHighlight(newHighlight); - setTokens([...newTokens]); - await new Promise(resolve => setTimeout(resolve, 150)); - }); - } catch (error) { - console.error('Tokenization error:', error); - } +const TOKENIZER_WAIT_TIME = 150; +const TOKENIZER_FINISH_TIME = 2000; - // Sleep for 2 seconds before allowing next run - if (!shouldStop()) { - await new Promise(resolve => setTimeout(resolve, 2000)); - } - setIsRunning(false); - }; +export function Tokenizer() { + const [terminal, setTerminal] = useState(null); - // Auto-start tokenization loop useEffect(() => { - // Only run in browser environment - if (typeof window === 'undefined') return; - + console.log('[Tokenizer] Component mounted, starting loop'); let cancelled = false; - const loop = async () => { + (async () => { while (!cancelled) { - await runTokenizer(() => cancelled); + console.log('[Tokenizer] Starting tokenization run'); + await runTokenizer(() => cancelled, (node) => { + console.log('[Tokenizer] Setting terminal node'); + setTerminal(node); + }); + console.log('[Tokenizer] Tokenization run complete, waiting before next run'); + await new Promise((resolve) => setTimeout(resolve, TOKENIZER_FINISH_TIME)); } - }; - loop(); + })(); + return () => { + console.log('[Tokenizer] Component unmounting, cancelling'); cancelled = true; - }; + } }, []); + return ( +
{terminal}
+ ); +} + +async function runTokenizer(shouldStop: () => boolean, setTerminal: (node: React.ReactNode) => void) { + console.log('[runTokenizer] Starting tokenization process'); + const tokens: Token[] = []; + let highlight = { start: 0, end: 0 }; + + try { + await tokenize(fib10, async (newHighlight, newTokens) => { + if (shouldStop()) { + console.log('[runTokenizer] Stopping due to cancellation'); + return; + } + + highlight = newHighlight; + tokens.splice(0, tokens.length, ...newTokens); + console.log(`[runTokenizer] Tokenized ${tokens.length} tokens, highlighting ${highlight.start}-${highlight.end}`); + + const terminalNode = renderTokenizer(highlight, tokens); + setTerminal(terminalNode); + + await new Promise(resolve => setTimeout(resolve, TOKENIZER_WAIT_TIME)); + }); + } catch (error) { + console.error('[runTokenizer] Tokenization error:', error); + } +} + +function renderTokenizer(highlight: { start: number, end: number }, tokens: Token[]): React.ReactNode { const charNodes: ReactNode[] = []; let inComment = false; @@ -586,13 +599,13 @@ export function Tokenizer() { const ch = fib10[i]; if (ch === '\n') { - inComment = false; // Reset comment flag on new line + inComment = false; charNodes.push(
); continue; } if (ch === '\\') { - inComment = true; // Start comment from backslash + inComment = true; } const highlightClass = getHighlightClass(i); @@ -619,7 +632,6 @@ export function Tokenizer() { } else if (highlightClass === ' hl-end') { inlineStyle.borderRadius = '0 2px 2px 0'; } - // hl-mid has no border radius } charNodes.push( @@ -627,7 +639,6 @@ export function Tokenizer() { ); } - // Get the latest 5 tokens for display, pad with empty slots if needed const latestTokens = tokens.slice(-5); const renderTokenLine = (i: number) => { @@ -647,7 +658,6 @@ export function Tokenizer() { prefix = 'symbol'; } - // Fixed-width layout for alignment const indexPart = `${tokenIndex.toString().padStart(2, '\u00A0')}:`; const valuePart = `\u00A0${tokenStr}`; const prefixPart = `(${prefix})`; @@ -688,65 +698,84 @@ export function Tokenizer() { ); } -export function Compiler() { - const [highlightRange, setHighlightRange] = useState<{ start: number, end: number }>({ start: -1, end: -1 }); - const [tokens, setTokens] = useState([]); - const [bytecode, setBytecode] = useState([]); - const [isRunning, setIsRunning] = useState(false); - - const runCompiler = async (shouldStop: () => boolean) => { - if (isRunning || shouldStop()) return; - setIsRunning(true); - setTokens([]); - setBytecode([]); - setHighlightRange({ start: -1, end: -1 }); - - try { - // First tokenize the source - const allTokens = await tokenize(fib10, async () => { - // No-op callback for tokenization, just need the tokens - }); - - // Then compile with highlighting - await compile(allTokens, async (highlight, newBytecode) => { - if (shouldStop()) return; - setHighlightRange({ start: highlight.tokenIdxStart, end: highlight.tokenIdxEnd }); - setTokens([...allTokens]); - setBytecode([...newBytecode]); - await new Promise(resolve => setTimeout(resolve, 500)); - }); - } catch (error) { - console.error('Compilation error:', error); - } +const COMPILER_WAIT_TIME = 500; +const COMPILER_FINISH_TIME = 2000; - // Sleep for 2 seconds before next run - if (!shouldStop()) { - await new Promise(resolve => setTimeout(resolve, 2000)); - } - setIsRunning(false); - }; +export function Compiler() { + const [terminal, setTerminal] = useState(null); useEffect(() => { - // Only run in browser environment - if (typeof window === 'undefined') return; - + console.log('[Compiler] Component mounted, starting loop'); let cancelled = false; - const loop = async () => { + (async () => { while (!cancelled) { - await runCompiler(() => cancelled); + console.log('[Compiler] Starting compilation run'); + await runCompiler(() => cancelled, (node) => { + console.log('[Compiler] Setting terminal node'); + setTerminal(node); + }); + console.log('[Compiler] Compilation run complete, waiting before next run'); + await new Promise((resolve) => setTimeout(resolve, COMPILER_FINISH_TIME)); } - }; - loop(); + })(); + return () => { + console.log('[Compiler] Component unmounting, cancelling'); cancelled = true; - }; + } }, []); - // Calculate which tokens to show + return ( +
{terminal}
+ ); +} + +async function runCompiler(shouldStop: () => boolean, setTerminal: (node: React.ReactNode) => void) { + console.log('[runCompiler] Starting compilation process'); + let tokens: Token[] = []; + let bytecode: Bytecode[] = []; + let highlightRange = { start: -1, end: -1 }; + + try { + // First tokenize the source + console.log('[runCompiler] Tokenizing source'); + const allTokens = await tokenize(fib10, async () => { + // No-op callback for tokenization, just need the tokens + }); + + if (shouldStop()) { + console.log('[runCompiler] Stopping due to cancellation after tokenization'); + return; + } + + tokens = allTokens; + console.log(`[runCompiler] Tokenized ${tokens.length} tokens`); + + // Then compile with highlighting + await compile(allTokens, async (highlight, newBytecode) => { + if (shouldStop()) { + console.log('[runCompiler] Stopping due to cancellation during compilation'); + return; + } + + highlightRange = { start: highlight.tokenIdxStart, end: highlight.tokenIdxEnd }; + bytecode = [...newBytecode]; + console.log(`[runCompiler] Compiled ${bytecode.length} bytecode ops, highlighting tokens ${highlightRange.start}-${highlightRange.end}`); + + const terminalNode = renderCompiler(highlightRange, tokens, bytecode); + setTerminal(terminalNode); + + await new Promise(resolve => setTimeout(resolve, COMPILER_WAIT_TIME)); + }); + } catch (error) { + console.error('[runCompiler] Compilation error:', error); + } +} + +function renderCompiler(highlightRange: { start: number, end: number }, tokens: Token[], bytecode: Bytecode[]): React.ReactNode { const visibleTokenCount = 10; let startIndex = 0; - // If we have a highlighted token and it would be below the visible area, scroll if (highlightRange.start >= 0 && tokens.length > 0) { const maxVisibleIndex = startIndex + visibleTokenCount - 1; if (highlightRange.start > maxVisibleIndex) { @@ -775,7 +804,6 @@ export function Compiler() { prefix = 'symbol'; } - // Fixed-width layout for alignment const indexPart = `${tokenIndex.toString().padStart(2, '\u00A0')}:`; const valuePart = `\u00A0${tokenStr}`; const prefixPart = `(${prefix})`; @@ -800,7 +828,6 @@ export function Compiler() { } }; - // Latest bytecode ops const latestBytecode = bytecode.slice(-10); const renderBytecodeLine = (i: number) => { if (i < latestBytecode.length) { @@ -854,76 +881,97 @@ export function Compiler() { ); } -export function VM() { - const [highlightIP, setHighlightIP] = useState(-1); - const [bytecode, setBytecode] = useState([]); - const [dataStack, setDataStack] = useState([]); - const [returnStack, setReturnStack] = useState([]); - const [variableTable, setVariableTable] = useState([]); - const [isRunning, setIsRunning] = useState(false); - - const runVM = async (shouldStop: () => boolean) => { - if (isRunning || shouldStop()) return; - setIsRunning(true); - setBytecode([]); - setDataStack([]); - setReturnStack([]); - setVariableTable([]); - setHighlightIP(-1); - - try { - // First tokenize and compile to get the program - const allTokens = await tokenize(fib10, async () => { - // No-op - }); - - const program = await compile(allTokens, async () => { - // No-op - }); - - setBytecode(program.bytecode); - - // Run the VM with highlighting - await vm(program, async (highlight, newDataStack, newReturnStack, newVariableTable) => { - if (shouldStop()) return; - setHighlightIP(highlight.ip); - setDataStack([...newDataStack]); - setReturnStack([...newReturnStack]); - setVariableTable([...newVariableTable]); - await new Promise(resolve => setTimeout(resolve, 750)); - }); - } catch (error) { - console.error('VM execution error:', error); - } +const VM_WAIT_TIME = 750; +const VM_FINISH_TIME = 2000; - // Sleep for 2 seconds before next run - if (!shouldStop()) { - await new Promise(resolve => setTimeout(resolve, 2000)); - } - setIsRunning(false); - }; +export function VM() { + const [terminal, setTerminal] = useState(null); useEffect(() => { - // Only run in browser environment - if (typeof window === 'undefined') return; - + console.log('[VM] Component mounted, starting loop'); let cancelled = false; - const loop = async () => { + (async () => { while (!cancelled) { - await runVM(() => cancelled); + console.log('[VM] Starting VM run'); + await runVM(() => cancelled, (node) => { + console.log('[VM] Setting terminal node'); + setTerminal(node); + }); + console.log('[VM] VM run complete, waiting before next run'); + await new Promise((resolve) => setTimeout(resolve, VM_FINISH_TIME)); } - }; - loop(); + })(); + return () => { + console.log('[VM] Component unmounting, cancelling'); cancelled = true; - }; + } }, []); - // Calculate which bytecode lines to show (scrolled around IP) + return ( +
{terminal}
+ ); +} + +async function runVM(shouldStop: () => boolean, setTerminal: (node: React.ReactNode) => void) { + console.log('[runVM] Starting VM execution process'); + let bytecode: Bytecode[] = []; + let highlightIP = -1; + let dataStack: number[] = []; + let returnStack: number[] = []; + let variableTable: number[] = []; + + try { + // First tokenize and compile to get the program + console.log('[runVM] Tokenizing and compiling source'); + const allTokens = await tokenize(fib10, async () => { + // No-op + }); + + if (shouldStop()) { + console.log('[runVM] Stopping due to cancellation after tokenization'); + return; + } + + const program = await compile(allTokens, async () => { + // No-op + }); + + if (shouldStop()) { + console.log('[runVM] Stopping due to cancellation after compilation'); + return; + } + + bytecode = program.bytecode; + console.log(`[runVM] Compiled program with ${bytecode.length} bytecode ops`); + + // Run the VM with highlighting + await vm(program, async (highlight, newDataStack, newReturnStack, newVariableTable) => { + if (shouldStop()) { + console.log('[runVM] Stopping due to cancellation during VM execution'); + return; + } + + highlightIP = highlight.ip; + dataStack = [...newDataStack]; + returnStack = [...newReturnStack]; + variableTable = [...newVariableTable]; + console.log(`[runVM] VM step at IP ${highlightIP}, DS: [${dataStack.join(',')}], RS: [${returnStack.join(',')}]`); + + const terminalNode = renderVM(highlightIP, bytecode, dataStack, returnStack, variableTable); + setTerminal(terminalNode); + + await new Promise(resolve => setTimeout(resolve, VM_WAIT_TIME)); + }); + } catch (error) { + console.error('[runVM] VM execution error:', error); + } +} + +function renderVM(highlightIP: number, bytecode: Bytecode[], dataStack: number[], returnStack: number[], variableTable: number[]): React.ReactNode { const visibleBytecodeCount = 15; let startIndex = 0; - // If we have a highlighted IP and it would be below the visible area, scroll if (highlightIP >= 0 && bytecode.length > 0) { const maxVisibleIndex = startIndex + visibleBytecodeCount - 1; if (highlightIP > maxVisibleIndex) { @@ -931,7 +979,6 @@ export function VM() { } else if (highlightIP < startIndex) { startIndex = Math.max(0, highlightIP - Math.floor(visibleBytecodeCount / 2)); } - // Ensure we don't scroll past the end! if (startIndex + visibleBytecodeCount > bytecode.length) { startIndex = Math.max(0, bytecode.length - visibleBytecodeCount); } @@ -976,9 +1023,8 @@ export function VM() { } }; - // Format stack display (show up to 5 items, most recent at top) - const formatStack = (stack: number[], name: string) => { - const displayStack = stack.slice(-5).reverse(); // Show last 5, most recent first + const formatStack = (stack: number[]) => { + const displayStack = stack.slice(-5).reverse(); const lines: ReactNode[] = []; for (let i = 0; i < 5; i++) { @@ -1003,7 +1049,6 @@ export function VM() { return lines; }; - // Format variable table display const formatVariables = () => { const lines: ReactNode[] = []; @@ -1032,11 +1077,11 @@ export function VM() {
Data Stack
- {formatStack(dataStack, 'DS')} + {formatStack(dataStack)}
Return Stack
- {formatStack(returnStack, 'RS')} + {formatStack(returnStack)}
Variables
From 37c18722ba2f67fe6eb3b5a5d2a9ace43c9350a8 Mon Sep 17 00:00:00 2001 From: Andrew Healey Date: Mon, 6 Oct 2025 14:05:41 +0100 Subject: [PATCH 5/6] try different jsx --- pages/[id].tsx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pages/[id].tsx b/pages/[id].tsx index cf21e2c..c583972 100644 --- a/pages/[id].tsx +++ b/pages/[id].tsx @@ -16,7 +16,8 @@ import Date from "../components/date"; import Newsletter from "../components/newsletter"; import { AldousBroder, IntroMaze, RandomDFS, TreeDiameter, WilsonsAlgorithm } from "../components/visuals/mazes/components"; import { KnightAttack, WhitePawnAttacks } from "../components/visuals/bitboards/components"; -import { Fibonacci, HelloWorld, Ladder } from "../components/visuals/icepath/components"; +import { Fibonacci, HelloWorld, Ladder } from "../components/visuals/icepath/components"; +import { Compiler, Tokenizer, VM } from "../components/visuals/forth/components"; const isVideo = /\.mp4$/; @@ -188,6 +189,14 @@ export default function Post({ } else if (props.id === 'knightAttack') { return KnightAttack() } + } else if (type === "div" && props.className === "forth") { + if (props.id === 'tokenizer') { + return + } else if (props.id === 'compiler') { + return + } else if (props.id === 'vm') { + return + } } return React.createElement(type, props, children); }, From 3c32383984adc8854a070f59b271a2b65aa17883 Mon Sep 17 00:00:00 2001 From: Andrew Healey Date: Mon, 6 Oct 2025 14:13:27 +0100 Subject: [PATCH 6/6] rm debug logs --- components/visuals/forth/components.tsx | 31 ------------------------- 1 file changed, 31 deletions(-) diff --git a/components/visuals/forth/components.tsx b/components/visuals/forth/components.tsx index 666f014..ed842b0 100644 --- a/components/visuals/forth/components.tsx +++ b/components/visuals/forth/components.tsx @@ -532,22 +532,17 @@ export function Tokenizer() { const [terminal, setTerminal] = useState(null); useEffect(() => { - console.log('[Tokenizer] Component mounted, starting loop'); let cancelled = false; (async () => { while (!cancelled) { - console.log('[Tokenizer] Starting tokenization run'); await runTokenizer(() => cancelled, (node) => { - console.log('[Tokenizer] Setting terminal node'); setTerminal(node); }); - console.log('[Tokenizer] Tokenization run complete, waiting before next run'); await new Promise((resolve) => setTimeout(resolve, TOKENIZER_FINISH_TIME)); } })(); return () => { - console.log('[Tokenizer] Component unmounting, cancelling'); cancelled = true; } }, []); @@ -558,20 +553,17 @@ export function Tokenizer() { } async function runTokenizer(shouldStop: () => boolean, setTerminal: (node: React.ReactNode) => void) { - console.log('[runTokenizer] Starting tokenization process'); const tokens: Token[] = []; let highlight = { start: 0, end: 0 }; try { await tokenize(fib10, async (newHighlight, newTokens) => { if (shouldStop()) { - console.log('[runTokenizer] Stopping due to cancellation'); return; } highlight = newHighlight; tokens.splice(0, tokens.length, ...newTokens); - console.log(`[runTokenizer] Tokenized ${tokens.length} tokens, highlighting ${highlight.start}-${highlight.end}`); const terminalNode = renderTokenizer(highlight, tokens); setTerminal(terminalNode); @@ -705,22 +697,17 @@ export function Compiler() { const [terminal, setTerminal] = useState(null); useEffect(() => { - console.log('[Compiler] Component mounted, starting loop'); let cancelled = false; (async () => { while (!cancelled) { - console.log('[Compiler] Starting compilation run'); await runCompiler(() => cancelled, (node) => { - console.log('[Compiler] Setting terminal node'); setTerminal(node); }); - console.log('[Compiler] Compilation run complete, waiting before next run'); await new Promise((resolve) => setTimeout(resolve, COMPILER_FINISH_TIME)); } })(); return () => { - console.log('[Compiler] Component unmounting, cancelling'); cancelled = true; } }, []); @@ -731,36 +718,30 @@ export function Compiler() { } async function runCompiler(shouldStop: () => boolean, setTerminal: (node: React.ReactNode) => void) { - console.log('[runCompiler] Starting compilation process'); let tokens: Token[] = []; let bytecode: Bytecode[] = []; let highlightRange = { start: -1, end: -1 }; try { // First tokenize the source - console.log('[runCompiler] Tokenizing source'); const allTokens = await tokenize(fib10, async () => { // No-op callback for tokenization, just need the tokens }); if (shouldStop()) { - console.log('[runCompiler] Stopping due to cancellation after tokenization'); return; } tokens = allTokens; - console.log(`[runCompiler] Tokenized ${tokens.length} tokens`); // Then compile with highlighting await compile(allTokens, async (highlight, newBytecode) => { if (shouldStop()) { - console.log('[runCompiler] Stopping due to cancellation during compilation'); return; } highlightRange = { start: highlight.tokenIdxStart, end: highlight.tokenIdxEnd }; bytecode = [...newBytecode]; - console.log(`[runCompiler] Compiled ${bytecode.length} bytecode ops, highlighting tokens ${highlightRange.start}-${highlightRange.end}`); const terminalNode = renderCompiler(highlightRange, tokens, bytecode); setTerminal(terminalNode); @@ -888,22 +869,17 @@ export function VM() { const [terminal, setTerminal] = useState(null); useEffect(() => { - console.log('[VM] Component mounted, starting loop'); let cancelled = false; (async () => { while (!cancelled) { - console.log('[VM] Starting VM run'); await runVM(() => cancelled, (node) => { - console.log('[VM] Setting terminal node'); setTerminal(node); }); - console.log('[VM] VM run complete, waiting before next run'); await new Promise((resolve) => setTimeout(resolve, VM_FINISH_TIME)); } })(); return () => { - console.log('[VM] Component unmounting, cancelling'); cancelled = true; } }, []); @@ -914,7 +890,6 @@ export function VM() { } async function runVM(shouldStop: () => boolean, setTerminal: (node: React.ReactNode) => void) { - console.log('[runVM] Starting VM execution process'); let bytecode: Bytecode[] = []; let highlightIP = -1; let dataStack: number[] = []; @@ -923,13 +898,11 @@ async function runVM(shouldStop: () => boolean, setTerminal: (node: React.ReactN try { // First tokenize and compile to get the program - console.log('[runVM] Tokenizing and compiling source'); const allTokens = await tokenize(fib10, async () => { // No-op }); if (shouldStop()) { - console.log('[runVM] Stopping due to cancellation after tokenization'); return; } @@ -938,17 +911,14 @@ async function runVM(shouldStop: () => boolean, setTerminal: (node: React.ReactN }); if (shouldStop()) { - console.log('[runVM] Stopping due to cancellation after compilation'); return; } bytecode = program.bytecode; - console.log(`[runVM] Compiled program with ${bytecode.length} bytecode ops`); // Run the VM with highlighting await vm(program, async (highlight, newDataStack, newReturnStack, newVariableTable) => { if (shouldStop()) { - console.log('[runVM] Stopping due to cancellation during VM execution'); return; } @@ -956,7 +926,6 @@ async function runVM(shouldStop: () => boolean, setTerminal: (node: React.ReactN dataStack = [...newDataStack]; returnStack = [...newReturnStack]; variableTable = [...newVariableTable]; - console.log(`[runVM] VM step at IP ${highlightIP}, DS: [${dataStack.join(',')}], RS: [${returnStack.join(',')}]`); const terminalNode = renderVM(highlightIP, bytecode, dataStack, returnStack, variableTable); setTerminal(terminalNode);