diff --git a/components/code.tsx b/components/code.tsx index aec0336..1f5d0f5 100644 --- a/components/code.tsx +++ b/components/code.tsx @@ -5,6 +5,7 @@ import Prism from "prism-react-renderer/prism"; require("prismjs/components/prism-rust") require("prismjs/components/prism-lua") require("prismjs/components/prism-lisp") +require("../lib/prism-forth") // -- import Highlight, { defaultProps, Language } from "prism-react-renderer"; diff --git a/components/visuals/forth/components.tsx b/components/visuals/forth/components.tsx new file mode 100644 index 0000000..ed842b0 --- /dev/null +++ b/components/visuals/forth/components.tsx @@ -0,0 +1,1075 @@ +import { useState, useEffect, ReactNode, CSSProperties } from 'react'; + +// Shared terminal styling +const terminalStyles = ` + .terminal { + --bg: #111827; + --fg: #e5e7eb; + --dim: #9ca3af; + background: var(--bg); + color: var(--fg); + font-family: "IBM Plex Mono", monospace; + font-size: 14px; + line-height: 1.25; + padding: 12px 14px; + border-radius: 0.4em; + border: 1px solid rgba(255, 255, 255, 0.08); + width: 100%; + max-width: 100%; + overflow: auto; + overscroll-behavior: contain; + } + .bytecode { + white-space: nowrap; + display: inline-block; + min-width: max-content; + } + .terminal :global(.ch) { + display: inline-block; + width: 1ch; + } + .terminal :global(.comment) { + color: var(--dim); + } + :global(.ch.hl) { + background: #e5e7eb !important; + color: #111827 !important; + border-radius: 2px; + } + :global(.ch.hl-start) { + background: #e5e7eb !important; + color: #111827 !important; + border-radius: 2px 0 0 2px; + } + :global(.ch.hl-mid) { + background: #e5e7eb !important; + color: #111827 !important; + } + :global(.ch.hl-end) { + background: #e5e7eb !important; + color: #111827 !important; + border-radius: 0 2px 2px 0; + } + .separator { + height: 1px; + background: rgba(255, 255, 255, 0.3); + margin: 12px 0; + } + .source-section, .tokens-section, .bytecode-section { + margin-top: 8px; + } + .tokens-label, .section-label { + color: var(--dim); + margin-bottom: 4px; + } + .token-line, .bytecode-line { + font-family: inherit; + line-height: 1.25; + min-height: 1.25em; + } + .token-line.highlighted, .bytecode-line.highlighted { + background: var(--fg); + color: var(--bg); + border-radius: 2px; + } +`; + +const fib10 = `VARIABLE A +VARIABLE B +VARIABLE T + + \\ calculates the nth Fibonacci number +: FIB \\ ( n -- f_n ) + 0 A ! + 1 B ! + 0 \\ start index + DO \\ ( limit start -- ) + A @ B @ + T ! + B @ A ! + T @ B ! + LOOP + A @ +; + +10 FIB . +` + +type LiteralToken = { + type: "literal"; + value: number; +} + +type SymbolToken = { + type: "store" | "load" | "dot" | "colon" | "semicolon" | "add" | "do" | "loop"; +} + +type IdentifierToken = { + type: "identifier"; + value: string; +} + +type Token = LiteralToken | SymbolToken | IdentifierToken; + +async function tokenize(source: string, callback: (highlight: { start: number, end: number }, tokens: Token[]) => Promise): Promise { + const tokens: Token[] = []; + + const isWhitespace = (c: string) => { + return c === " " || c === "\n" || c === "\t"; + } + const isDigit = (c: string) => { + return c >= "0" && c <= "9"; + } + const isLetter = (c: string) => { + return (c >= "a" && c <= "z") || (c >= "A" && c <= "Z") || c === "_"; + } + + let index = 0; + while (index < source.length) { + await callback({ start: index, end: index + 1 }, tokens); + + if (source[index] === "\\") { + const commentStart = index; + while (index < source.length && source[index] !== "\n") { + index++; + await callback({ start: commentStart, end: index }, tokens); + } + index++; + await callback({ start: commentStart, end: index }, tokens); + continue; + } + if (isWhitespace(source[index])) { + index++; + continue; + } + if (source[index] === ":") { + tokens.push({ type: "colon" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + if (source[index] === ";") { + tokens.push({ type: "semicolon" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + if (source[index] === ".") { + tokens.push({ type: "dot" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + if (source[index] === "!") { + tokens.push({ type: "store" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + if (source[index] === "@") { + tokens.push({ type: "load" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + if (isDigit(source[index])) { + const start = index; + let value = ""; + while (isDigit(source[index])) { + value += source[index]; + index++; + } + tokens.push({ type: "literal", value: parseInt(value) }); + await callback({ start, end: index }, tokens); + continue; + } + if (isLetter(source[index])) { + const start = index; + let value = ""; + while (isLetter(source[index])) { + value += source[index]; + index++; + } + + if (value === "DO") { + tokens.push({ type: "do" }); + await callback({ start, end: index }, tokens); + continue; + } + if (value === "LOOP") { + tokens.push({ type: "loop" }); + await callback({ start, end: index }, tokens); + continue; + } + + tokens.push({ type: "identifier", value }); + await callback({ start, end: index }, tokens); + continue; + } + if (source[index] === "+") { + tokens.push({ type: "add" }); + await callback({ start: index, end: index + 1 }, tokens); + index++; + continue; + } + + throw new Error(`Unexpected token at index ${index}: ${source[index]}`); + } + + return tokens; +} + +type Op = { + op: "lit", // Push value or address to DS + value: number; +} | { + op: "load", // Pop address from DS, push value at address +} | { + op: "store", // Pop address from DS, pop value from DS, store value at address +} | { + op: "dup2", // Duplicate top two values on DS [a, b] -> [a, b, a, b] +} | { + op: "add", // Pop top two values from DS, push sum to DS +} | { + op: "eq", // Pop top two values from DS, push 1 if equal, 0 if not +} | { + op: "jz", // Pop value from DS, if zero, jump to address + address: number; +} | { + op: "jmp", // Jump to address + address: number; +} | { + op: "call", // Push IP to RS, jump to address + address: number; +} | { + op: "ret", // Pop IP from RS, jump to IP +} | { + op: "rs_push", // Pop from DS, push to RS +} | { + op: "rs_pop", // Pop from RS, push to DS +} | { + op: "drop", // Discard top value from DS +} | { + op: "print", // Pop value from DS, print it +} + +type Bytecode = Op; +type Program = { + bytecode: Bytecode[]; + variableCount: number; +} + +async function compile(tokens: Token[], callback: (highlight: { tokenIdxStart: number, tokenIdxEnd: number }, ops: Bytecode[]) => Promise): Promise { + const bytecode: Bytecode[] = []; + const wordTable: { [key: string]: number } = {}; + const variableTable: { [key: string]: number } = {}; + let currentWordSkipOp: Bytecode | null = null; + let loopStart: number[] = []; + + let index = 0; + while (index < tokens.length) { + const token = tokens[index]; + if (token.type === "identifier") { + if (token.value === "VARIABLE") { + const nextToken = tokens[index + 1]; + if (nextToken.type !== "identifier") { + throw new Error(`Expected identifier after VARIABLE at index ${index}: ${JSON.stringify(nextToken)}`); + } + variableTable[nextToken.value] = Object.keys(variableTable).length; + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 2 }, bytecode); + index += 2; + continue; + } + + if (wordTable[token.value] !== undefined) { + bytecode.push({ op: "call", address: wordTable[token.value] }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + + bytecode.push({ op: "lit", value: variableTable[token.value] }); // Address + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + if (token.type === "colon") { + index++; + const nextToken = tokens[index]; + if (nextToken.type !== "identifier") { + throw new Error(`Expected identifier after colon at index ${index}: ${JSON.stringify(nextToken)}`); + } + if (currentWordSkipOp !== null) { + throw new Error(`Expected semicolon before next word at index ${index}: ${JSON.stringify(nextToken)}`); + } + + currentWordSkipOp = { op: "jmp", address: -1 }; + bytecode.push(currentWordSkipOp); + await callback({ tokenIdxStart: index - 1, tokenIdxEnd: index + 1 }, bytecode); + wordTable[nextToken.value] = bytecode.length; + index++; + continue; + } + if (token.type === "semicolon") { + if (currentWordSkipOp === null) { + throw new Error(`Expected word before semicolon at index ${index}: ${JSON.stringify(token)}`); + } + bytecode.push({ op: "ret" }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + currentWordSkipOp.address = bytecode.length; + currentWordSkipOp = null; + index++; + continue; + } + + if (token.type === "do") { + index++; + // Expect: DS has [limit, start] (start is top) + // Move both to RS: start then limit (RS top becomes limit) + bytecode.push({ op: "rs_push" }) // start -> RS + await callback({ tokenIdxStart: index - 1, tokenIdxEnd: index }, bytecode); + bytecode.push({ op: "rs_push" }) // limit -> RS + await callback({ tokenIdxStart: index - 1, tokenIdxEnd: index }, bytecode); + // Mark first instruction of loop body + loopStart.push(bytecode.length); + continue; + } + + if (token.type === "loop") { + if (loopStart.length === 0) { + throw new Error(`Loop without do at index ${index}: ${JSON.stringify(token)}`); + } + + // Pop limit and i from RS (RS top is limit) + bytecode.push({ op: "rs_pop" }) // limit -> DS + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "rs_pop" }) // i -> DS + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + // Increment i + bytecode.push({ op: "lit", value: 1 }) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "add" }) // i on DS + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + // Duplicate i and limit for compare and possible restore + bytecode.push({ op: "dup2" }) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "eq" }) // eq flag on DS + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + const loopStartAddress = loopStart.pop()!; // first instr of loop body + + // Branch: continue when not equal (eq==0), exit when equal + const continueAddress = bytecode.length + 4; // skip equal-path (2 drops + jmp) + bytecode.push({ op: "jz", address: continueAddress }) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + // Equal path (fallthrough): cleanup and exit + bytecode.push({ op: "drop" }) // drop i + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "drop" }) // drop limit + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + const afterBlockAddress = bytecode.length + 1 /* jmp */ + 3 /* continue block */; + bytecode.push({ op: "jmp", address: afterBlockAddress }) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + // Continue path: + // address == continueAddress + bytecode.push({ op: "rs_push" }) // i -> RS (top) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "rs_push" }) // limit -> RS + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + bytecode.push({ op: "jmp", address: loopStartAddress }) + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + + index++; + continue; + } + + if (token.type === "literal") { + bytecode.push({ op: "lit", value: token.value }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + if (token.type === "store") { + bytecode.push({ op: "store" }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + if (token.type === "load") { + bytecode.push({ op: "load" }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + if (token.type === "add") { + bytecode.push({ op: "add" }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + if (token.type === "dot") { + bytecode.push({ op: "print" }); + await callback({ tokenIdxStart: index, tokenIdxEnd: index + 1 }, bytecode); + index++; + continue; + } + + throw new Error(`Unexpected token at index ${index}: ${JSON.stringify(token)}`); + } + + return { bytecode, variableCount: Object.keys(variableTable).length }; +} + +async function vm(program: Program, callback: (highlight: { ip: number }, dataStack: number[], returnStack: number[], variableTable: number[]) => Promise) { + const dataStack: number[] = []; + const returnStack: number[] = []; + const variableTable: number[] = new Array(program.variableCount).fill(0); + + const dsPop = () => { + const value = dataStack.pop(); + if (value === undefined) { + throw new Error("Data stack underflow"); + } + return value; + } + const rsPop = () => { + const value = returnStack.pop(); + if (value === undefined) { + throw new Error("Return stack underflow"); + } + return value; + } + + let ip = 0; + while (ip < program.bytecode.length) { + const cur = program.bytecode[ip]; + + await callback({ ip }, dataStack, returnStack, variableTable); + + if (cur.op === "lit") { + dataStack.push(cur.value); + ip++; + continue; + } else if (cur.op === "store") { + const address = dsPop(); + const value = dsPop(); + variableTable[address] = value; + ip++; + continue; + } else if (cur.op === "load") { + const address = dsPop(); + dataStack.push(variableTable[address]); + ip++; + continue; + } else if (cur.op === "jmp") { + ip = cur.address; + continue; + } else if (cur.op === "jz") { + if (dsPop() === 0) { + ip = cur.address; + continue; + } + ip++; + continue; + } else if (cur.op === "call") { + ip++ + returnStack.push(ip); + ip = cur.address; + continue; + } else if (cur.op === "ret") { + ip = rsPop(); + continue; + } else if (cur.op === "rs_push") { + returnStack.push(dsPop()); + ip++; + continue; + } else if (cur.op === "rs_pop") { + dataStack.push(rsPop()); + ip++; + continue; + } else if (cur.op === "add") { + dataStack.push(dsPop() + dsPop()); + ip++; + continue; + } else if (cur.op === "dup2") { + if (dataStack.length < 2) { + throw new Error("Data stack underflow"); + } + const top = dataStack[dataStack.length - 1]; + const secondFromTop = dataStack[dataStack.length - 2]; + // Duplicate in-order: [a, b] -> [a, b, a, b] + dataStack.push(secondFromTop); + dataStack.push(top); + ip++; + continue; + } else if (cur.op === "eq") { + dataStack.push(dsPop() === dsPop() ? 1 : 0); + ip++; + continue; + } else if (cur.op === "drop") { + dsPop(); + ip++; + continue; + } else if (cur.op === "print") { + console.log(dsPop()); + ip++; + continue; + } + + // @ts-expect-error lints for missing branches + throw new Error(`Unknown opcode: ${cur.op}`); + } +} + +const TOKENIZER_WAIT_TIME = 150; +const TOKENIZER_FINISH_TIME = 2000; + +export function Tokenizer() { + const [terminal, setTerminal] = useState(null); + + useEffect(() => { + let cancelled = false; + (async () => { + while (!cancelled) { + await runTokenizer(() => cancelled, (node) => { + setTerminal(node); + }); + await new Promise((resolve) => setTimeout(resolve, TOKENIZER_FINISH_TIME)); + } + })(); + + return () => { + cancelled = true; + } + }, []); + + return ( +
{terminal}
+ ); +} + +async function runTokenizer(shouldStop: () => boolean, setTerminal: (node: React.ReactNode) => void) { + const tokens: Token[] = []; + let highlight = { start: 0, end: 0 }; + + try { + await tokenize(fib10, async (newHighlight, newTokens) => { + if (shouldStop()) { + return; + } + + highlight = newHighlight; + tokens.splice(0, tokens.length, ...newTokens); + + const terminalNode = renderTokenizer(highlight, tokens); + setTerminal(terminalNode); + + await new Promise(resolve => setTimeout(resolve, TOKENIZER_WAIT_TIME)); + }); + } catch (error) { + console.error('[runTokenizer] Tokenization error:', error); + } +} + +function renderTokenizer(highlight: { start: number, end: number }, tokens: Token[]): React.ReactNode { + const charNodes: ReactNode[] = []; + let inComment = false; + + const getHighlightClass = (i: number) => { + if (i < highlight.start || i >= highlight.end) return ''; + if (highlight.end - highlight.start === 1) return ' hl'; + if (i === highlight.start) return ' hl-start'; + if (i === highlight.end - 1) return ' hl-end'; + return ' hl-mid'; + }; + + for (let i = 0; i < fib10.length; i++) { + const ch = fib10[i]; + + if (ch === '\n') { + inComment = false; + charNodes.push(
); + continue; + } + + if (ch === '\\') { + inComment = true; + } + + const highlightClass = getHighlightClass(i); + const commentClass = inComment ? ' comment' : ''; + const classes = `ch${highlightClass}${commentClass}`; + + let inlineStyle: CSSProperties = { + display: 'inline-block', + width: '1ch' + }; + + if (inComment) { + inlineStyle.color = '#9ca3af'; + } + + if (highlightClass) { + inlineStyle.background = '#e5e7eb'; + inlineStyle.color = '#111827'; + + if (highlightClass === ' hl') { + inlineStyle.borderRadius = '2px'; + } else if (highlightClass === ' hl-start') { + inlineStyle.borderRadius = '2px 0 0 2px'; + } else if (highlightClass === ' hl-end') { + inlineStyle.borderRadius = '0 2px 2px 0'; + } + } + + charNodes.push( + {ch === ' ' ? '\u00A0' : ch} + ); + } + + const latestTokens = tokens.slice(-5); + + const renderTokenLine = (i: number) => { + if (i < latestTokens.length) { + const token = latestTokens[i]; + const tokenIndex = tokens.length - latestTokens.length + i; + let tokenStr = ''; + let prefix = ''; + if (token.type === 'literal') { + tokenStr = token.value.toString(); + prefix = 'literal'; + } else if (token.type === 'identifier') { + tokenStr = token.value; + prefix = 'identifier'; + } else { + tokenStr = token.type.toUpperCase(); + prefix = 'symbol'; + } + + const indexPart = `${tokenIndex.toString().padStart(2, '\u00A0')}:`; + const valuePart = `\u00A0${tokenStr}`; + const prefixPart = `(${prefix})`; + const totalBeforePrefix = indexPart.length + valuePart.length; + const paddingNeeded = Math.max(0, 14 - totalBeforePrefix); + const padding = '\u00A0'.repeat(paddingNeeded); + + return ( +
+ {indexPart} + {valuePart} + {padding} + {prefixPart} +
+ ); + } else { + return ( +
+ {i.toString().padStart(2, '\u00A0')}: +
+ ); + } + }; + + return ( +
+
+
Source Code
+
{charNodes}
+
+
+
+
Tokens
+ {Array.from({ length: 5 }, (_, i) => renderTokenLine(i))} +
+ +
+ ); +} + +const COMPILER_WAIT_TIME = 500; +const COMPILER_FINISH_TIME = 2000; + +export function Compiler() { + const [terminal, setTerminal] = useState(null); + + useEffect(() => { + let cancelled = false; + (async () => { + while (!cancelled) { + await runCompiler(() => cancelled, (node) => { + setTerminal(node); + }); + await new Promise((resolve) => setTimeout(resolve, COMPILER_FINISH_TIME)); + } + })(); + + return () => { + cancelled = true; + } + }, []); + + return ( +
{terminal}
+ ); +} + +async function runCompiler(shouldStop: () => boolean, setTerminal: (node: React.ReactNode) => void) { + let tokens: Token[] = []; + let bytecode: Bytecode[] = []; + let highlightRange = { start: -1, end: -1 }; + + try { + // First tokenize the source + const allTokens = await tokenize(fib10, async () => { + // No-op callback for tokenization, just need the tokens + }); + + if (shouldStop()) { + return; + } + + tokens = allTokens; + + // Then compile with highlighting + await compile(allTokens, async (highlight, newBytecode) => { + if (shouldStop()) { + return; + } + + highlightRange = { start: highlight.tokenIdxStart, end: highlight.tokenIdxEnd }; + bytecode = [...newBytecode]; + + const terminalNode = renderCompiler(highlightRange, tokens, bytecode); + setTerminal(terminalNode); + + await new Promise(resolve => setTimeout(resolve, COMPILER_WAIT_TIME)); + }); + } catch (error) { + console.error('[runCompiler] Compilation error:', error); + } +} + +function renderCompiler(highlightRange: { start: number, end: number }, tokens: Token[], bytecode: Bytecode[]): React.ReactNode { + const visibleTokenCount = 10; + let startIndex = 0; + + if (highlightRange.start >= 0 && tokens.length > 0) { + const maxVisibleIndex = startIndex + visibleTokenCount - 1; + if (highlightRange.start > maxVisibleIndex) { + startIndex = Math.max(0, highlightRange.start - visibleTokenCount + 1); + } + } + + const visibleTokens = tokens.slice(startIndex, startIndex + visibleTokenCount); + + const renderTokenLine = (i: number) => { + if (i < visibleTokens.length) { + const token = visibleTokens[i]; + const tokenIndex = startIndex + i; + const isHighlighted = tokenIndex >= highlightRange.start && tokenIndex < highlightRange.end; + + let tokenStr = ''; + let prefix = ''; + if (token.type === 'literal') { + tokenStr = token.value.toString(); + prefix = 'literal'; + } else if (token.type === 'identifier') { + tokenStr = token.value; + prefix = 'identifier'; + } else { + tokenStr = token.type.toUpperCase(); + prefix = 'symbol'; + } + + const indexPart = `${tokenIndex.toString().padStart(2, '\u00A0')}:`; + const valuePart = `\u00A0${tokenStr}`; + const prefixPart = `(${prefix})`; + const totalBeforePrefix = indexPart.length + valuePart.length; + const paddingNeeded = Math.max(0, 13 - totalBeforePrefix); + const padding = '\u00A0'.repeat(paddingNeeded); + + return ( +
+ {indexPart} + {valuePart} + {padding} + {prefixPart} +
+ ); + } else { + return ( +
+ {(startIndex + i).toString().padStart(2, '\u00A0')}: +
+ ); + } + }; + + const latestBytecode = bytecode.slice(-10); + const renderBytecodeLine = (i: number) => { + if (i < latestBytecode.length) { + const op = latestBytecode[i]; + const opIndex = bytecode.length - latestBytecode.length + i; + + let opStr = ''; + if (op.op === 'lit') { + opStr = `lit ${op.value}`; + } else if (op.op === 'jz') { + opStr = `jz ${op.address}`; + } else if (op.op === 'jmp') { + opStr = `jmp ${op.address}`; + } else if (op.op === 'call') { + opStr = `call ${op.address}`; + } else { + opStr = op.op; + } + + const indexPart = `${opIndex.toString().padStart(2, '\u00A0')}:`; + const valuePart = `\u00A0${opStr}`; + + return ( +
+ {indexPart} + {valuePart} +
+ ); + } else { + return ( +
+ {i.toString().padStart(2, '\u00A0')}: +
+ ); + } + }; + + return ( +
+
+
Tokens
+ {Array.from({ length: 10 }, (_, i) => renderTokenLine(i))} +
+
+
+
Bytecode
+ {Array.from({ length: 10 }, (_, i) => renderBytecodeLine(i))} +
+ +
+ ); +} + +const VM_WAIT_TIME = 750; +const VM_FINISH_TIME = 2000; + +export function VM() { + const [terminal, setTerminal] = useState(null); + + useEffect(() => { + let cancelled = false; + (async () => { + while (!cancelled) { + await runVM(() => cancelled, (node) => { + setTerminal(node); + }); + await new Promise((resolve) => setTimeout(resolve, VM_FINISH_TIME)); + } + })(); + + return () => { + cancelled = true; + } + }, []); + + return ( +
{terminal}
+ ); +} + +async function runVM(shouldStop: () => boolean, setTerminal: (node: React.ReactNode) => void) { + let bytecode: Bytecode[] = []; + let highlightIP = -1; + let dataStack: number[] = []; + let returnStack: number[] = []; + let variableTable: number[] = []; + + try { + // First tokenize and compile to get the program + const allTokens = await tokenize(fib10, async () => { + // No-op + }); + + if (shouldStop()) { + return; + } + + const program = await compile(allTokens, async () => { + // No-op + }); + + if (shouldStop()) { + return; + } + + bytecode = program.bytecode; + + // Run the VM with highlighting + await vm(program, async (highlight, newDataStack, newReturnStack, newVariableTable) => { + if (shouldStop()) { + return; + } + + highlightIP = highlight.ip; + dataStack = [...newDataStack]; + returnStack = [...newReturnStack]; + variableTable = [...newVariableTable]; + + const terminalNode = renderVM(highlightIP, bytecode, dataStack, returnStack, variableTable); + setTerminal(terminalNode); + + await new Promise(resolve => setTimeout(resolve, VM_WAIT_TIME)); + }); + } catch (error) { + console.error('[runVM] VM execution error:', error); + } +} + +function renderVM(highlightIP: number, bytecode: Bytecode[], dataStack: number[], returnStack: number[], variableTable: number[]): React.ReactNode { + const visibleBytecodeCount = 15; + let startIndex = 0; + + if (highlightIP >= 0 && bytecode.length > 0) { + const maxVisibleIndex = startIndex + visibleBytecodeCount - 1; + if (highlightIP > maxVisibleIndex) { + startIndex = Math.max(0, highlightIP - Math.floor(visibleBytecodeCount / 2)); + } else if (highlightIP < startIndex) { + startIndex = Math.max(0, highlightIP - Math.floor(visibleBytecodeCount / 2)); + } + if (startIndex + visibleBytecodeCount > bytecode.length) { + startIndex = Math.max(0, bytecode.length - visibleBytecodeCount); + } + } + + const visibleBytecode = bytecode.slice(startIndex, startIndex + visibleBytecodeCount); + + const renderBytecodeLine = (i: number) => { + if (i < visibleBytecode.length) { + const op = visibleBytecode[i]; + const opIndex = startIndex + i; + const isHighlighted = opIndex === highlightIP; + + let opStr = ''; + if (op.op === 'lit') { + opStr = `lit ${op.value}`; + } else if (op.op === 'jz') { + opStr = `jz ${op.address}`; + } else if (op.op === 'jmp') { + opStr = `jmp ${op.address}`; + } else if (op.op === 'call') { + opStr = `call ${op.address}`; + } else { + opStr = op.op; + } + + const indexPart = `${opIndex.toString().padStart(2, '\u00A0')}:`; + const valuePart = `\u00A0${opStr}`; + + return ( +
+ {indexPart} + {valuePart} +
+ ); + } else { + return ( +
+ {(startIndex + i).toString().padStart(2, '\u00A0')}: +
+ ); + } + }; + + const formatStack = (stack: number[]) => { + const displayStack = stack.slice(-5).reverse(); + const lines: ReactNode[] = []; + + for (let i = 0; i < 5; i++) { + if (i < displayStack.length) { + const value = displayStack[i]; + const stackIndex = stack.length - 1 - i; + lines.push( +
+ {stackIndex.toString().padStart(2, ' ')}: + {value} +
+ ); + } else { + lines.push( +
+ +
+ ); + } + } + + return lines; + }; + + const formatVariables = () => { + const lines: ReactNode[] = []; + + for (let i = 0; i < variableTable.length; i++) { + const value = variableTable[i]; + lines.push( +
+ {i.toString().padStart(2, ' ')}: + {value} +
+ ); + } + + return lines; + }; + + return ( +
+
+
Bytecode
+ {Array.from({ length: visibleBytecodeCount }, (_, i) => renderBytecodeLine(i))} +
+
+
+
VM State
+
+
+
Data Stack
+ {formatStack(dataStack)} +
+
+
Return Stack
+ {formatStack(returnStack)} +
+
+
Variables
+ {formatVariables()} +
+
+
+ +
+ ); +} + +// When this file is ran with `bun` this outputs the 10th Fibonacci number (55) +if ('Bun' in globalThis && globalThis.Bun.main.includes('forth')) { + tokenize(fib10, async (_, __) => { /* no-op */ }) + .then(tokens => { + compile(tokens, async (_, __) => { /* no-op */ }) + .then(program => { + vm(program, async (_, __, ___, ____) => { /* no-op */ }); + }); + }); +} diff --git a/data/posts.ts b/data/posts.ts index f958754..2f6fb7a 100644 --- a/data/posts.ts +++ b/data/posts.ts @@ -7,6 +7,7 @@ export const popularPosts = [ // Starred posts (not in any specific order) export const postStars = [ + "compiling-a-forth", "counting-words-at-simd-speed", "optimizing-my-disk-usage-program", "maybe-the-fastest-disk-usage-program-on-macos", diff --git a/lib/prism-forth.js b/lib/prism-forth.js new file mode 100644 index 0000000..47a135a --- /dev/null +++ b/lib/prism-forth.js @@ -0,0 +1,27 @@ +(function (Prism) { + Prism.languages.forth = { + "comment": { + pattern: /\\\s.*$/m, + greedy: true, + }, + "string": { + pattern: /"(?:[^"\\]|\\.)*"/, + greedy: true, + }, + "number": /\b-?\d+(?:\.\d+)?\b/, + "keyword": + /\b(?:VARIABLE|DO|LOOP|IF|THEN|ELSE|BEGIN|UNTIL|WHILE|REPEAT)\b/i, + "builtin": + /\b(?:@|!|\+|-|\*|\/|MOD|=|<|>|AND|OR|NOT|DUP|DROP|SWAP|OVER|ROT|\.)\b/, + "definition": { + pattern: /(:)\s+(\S+)/, + lookbehind: true, + inside: { + "keyword": /^:/, + "function": /\S+/, + }, + }, + "variable": /\b[A-Z]+\b/, + "punctuation": /[;]/, + }; +})(Prism); diff --git a/pages/[id].tsx b/pages/[id].tsx index cf21e2c..c583972 100644 --- a/pages/[id].tsx +++ b/pages/[id].tsx @@ -16,7 +16,8 @@ import Date from "../components/date"; import Newsletter from "../components/newsletter"; import { AldousBroder, IntroMaze, RandomDFS, TreeDiameter, WilsonsAlgorithm } from "../components/visuals/mazes/components"; import { KnightAttack, WhitePawnAttacks } from "../components/visuals/bitboards/components"; -import { Fibonacci, HelloWorld, Ladder } from "../components/visuals/icepath/components"; +import { Fibonacci, HelloWorld, Ladder } from "../components/visuals/icepath/components"; +import { Compiler, Tokenizer, VM } from "../components/visuals/forth/components"; const isVideo = /\.mp4$/; @@ -188,6 +189,14 @@ export default function Post({ } else if (props.id === 'knightAttack') { return KnightAttack() } + } else if (type === "div" && props.className === "forth") { + if (props.id === 'tokenizer') { + return + } else if (props.id === 'compiler') { + return + } else if (props.id === 'vm') { + return + } } return React.createElement(type, props, children); }, diff --git a/pages/index.tsx b/pages/index.tsx index 0b15960..91a46c8 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -72,7 +72,7 @@ export default function Home({ allPostsData, description, words }) { My experimental package manager uses simple concurrency patterns to be faster than every package manager aside from Bun (which is 11% faster) when cold-installing from a lockfile.

- I've created a few small programming languages and related tools, including a Lisp-to-JavaScript compiler, which I turned into an optimizing compiler, and for which I wrote a bytecode VM. + I've created a few small programming languages and related tools, including a Forth compiler, a Lisp-to-JavaScript compiler, which I turned into an optimizing compiler, and for which I wrote a bytecode VM. I also built an interpreted language with a C-style syntax, which I profiled and made faster; I later added a WebAssembly compiler and a line profiler. I also ported an expression engine to Rust.

diff --git a/posts/compiling-a-forth.md b/posts/compiling-a-forth.md new file mode 100644 index 0000000..b329262 --- /dev/null +++ b/posts/compiling-a-forth.md @@ -0,0 +1,410 @@ +--- +title: "Compiling a Forth" +date: "2025-10-06" +tags: ["forth"] +description: "A bytecode compiler and VM for a Forth-like language." +--- + +I was curious how Forth worked so I built a bytecode compiler and a VM for a Forth-like language, as well as some visualizations to show how it all works. + +You don't need to know anything about Forth to follow along, aside from the fact it's a stack-oriented language. + +Here's a small program that prints the number three. + +```forth +3 . +``` + +The number (`3`) is pushed to the data stack, and then the dot (`.`) pops it from the data stack and prints it. + +We'll need more Forth features than this to build interesting programs. + +Forth has two built-in stacks. The data stack (sometimes just called "the stack") and the return stack. When a word is called in Forth (words are like functions) the address of the next instruction is pushed to the return stack. When the word finishes executing, the return stack is popped into the instruction pointer. + +```forth +\ (1) word declaration +: PRINT10 + + \ (3) the word body is executed + 10 . + + \ (4) ";" compiles an exit – at runtime it pops the return stack + \ into the instruction pointer. +; + +\ (2) instruction pointer lands on a word, +\ the next address is pushed to the return stack, +\ and the instruction pointer is set to the word address +PRINT10 + +\ (5) next address is executed +``` + +As well as words, my compiler also supports `DO`/`LOOP`s. These use the return stack too. When `DO` executes, it pops the limit and the iterator from the data stack and stores them in the return stack. This allows the inner loop to freely operate on the data stack. When `LOOP` executes, it pops the limit and iterator from the return stack, adds one to the iterator and compares it to the limit (and exits or loops again). + +There are also variables, which can be declared with `VARIABLE X`, loaded with `X @`, and stored with `1 X !`. + +Putting these features together, here's how you can build `10` by adding `1` repeatedly. + +```forth +VARIABLE A + +: RUN + 0 A ! \ initialize A + 10 0 DO \ push limit and iterator for DO + \ DO places these on the return stack + A @ 1 + A ! \ A = A + 1 + LOOP \ increment i and exits when i == limit + A @ . \ prints 10 +; + +RUN +``` + +This set of features is enough for us to calculate numbers from the Fibonacci series, which is the example program I'll be using throughout the rest of this post. + +## Tokenizing + +Tokenization translates raw text into meaningful symbols. + +To turn source code into tokens, we scan through the code, skipping over whitespace and appending tokens to a list. Syntax that's a single character is turned straight into a token but multi-character syntax needs to be grouped together. For example, entire comments are discarded, and while they are being discarded, we need to track that we're "within" a comment. + +Identifiers, like keywords like `DO` or `LOOP`, or custom variables like `MYLONGVAR`, become single tokens. + +First, a visualization of what's happening: + +

+ +And here's a trimmed version of my tokenizer: + +```tsx +function tokenize(source: string): Token[] { + const tokens: Token[] = []; + + let index = 0; + while (index < source.length) { + + // Consume and discard everything on a line after '\' + if (source[index] === "\\") { + const commentStart = index; + while (index < source.length && source[index] !== "\n") { + index++; + } + index++; + continue; + } + + // Skip over whitespace + if (isWhitespace(source[index])) { + index++; + continue; + } + + if (source[index] === "@") { + tokens.push({ type: "load" }); + index++; + continue; + } + + // Handle identifiers + if (isLetter(source[index])) { + const start = index; + let value = ""; + while (isLetter(source[index])) { + value += source[index]; + index++; + } + + // Special-case the keywords + if (value === "DO") { + tokens.push({ type: "do" }); + continue; + } + if (value === "LOOP") { + tokens.push({ type: "loop" }); + continue; + } + + tokens.push({ type: "identifier", value }); + continue; + } + + // .. trimmed other tokens, see source + } + + return tokens; +} +``` + +With our list of tokens, we're ready to start generating bytecode for the VM. + +## Generating Bytecode + +Usually, in a compiler, the step after tokenization is _parsing_ where an abstract syntax tree is built. However, the feature set of my Forth is so small, that I decided to generate bytecode directly from the list of tokens. + +_After_ bytecode generation, my VM needs two things: + +- A list of operations for the VM's instruction pointer to navigate +- The number of variables that the program refers to + +The latter tells the VM how many variables to allocate (a zero-initialized array). Variables in source (e.g., `A`, `B`) become integer indices into this array. + +This means that my bytecode generation step needs to keep track of variables that have been seen before so that I can output the correct memory address (i.e. an index into the variable table). + +I'll show the full list of bytecode operations and then a few of the steps for handling specific tokens. + +```tsx +type Op = { + op: "lit", // Push value or address to DS + value: number; +} | { + op: "load", // Pop address from DS, push value at address +} | { + op: "store", // Pop address from DS, pop value from DS, store value at address +} | { + op: "dup2", // Duplicate top two values on DS [a, b] -> [a, b, a, b] +} | { + op: "add", // Pop top two values from DS, push sum to DS +} | { + op: "eq", // Pop top two values from DS, push 1 if equal, 0 if not +} | { + op: "jz", // Pop value from DS, if zero, jump to address + address: number; +} | { + op: "jmp", // Jump to address + address: number; +} | { + op: "call", // Push IP to RS, jump to address + address: number; +} | { + op: "ret", // Pop IP from RS, jump to IP +} | { + op: "rs_push", // Pop from DS, push to RS +} | { + op: "rs_pop", // Pop from RS, push to DS +} | { + op: "drop", // Discard top value from DS +} | { + op: "print", // Pop value from DS, print it +} +``` + +The bytecode generation step scans through the list of tokens and, as it processes them, it appends to a list of bytecode and increments the variable count to set up the correct references. + +Identifier tokens are either variable references, or words (function calls). + +```tsx +let index = 0; +while (index < tokens.length) { + const token = tokens[index]; + + if (token.type === "identifier") { + if (token.value === "VARIABLE") { + const nextToken = tokens[index + 1]; + + // Store a binding of variable name to memory address + variableTable[nextToken.value] = Object.keys(variableTable).length; + index += 2; + continue; + } + + // If the variable has been declared as a word like `: FIB10` + // then we have previously stored the bytecode offset which we + // will set the instruction pointer to at runtime + if (wordTable[token.value] !== undefined) { + bytecode.push({ op: "call", address: wordTable[token.value] }); + index++; + continue; + } + + // If it's not a variable declaration, or a word, then we + // look up the memory address + bytecode.push({ op: "lit", value: variableTable[token.value] }); + index++; + continue; + } + + // .. +} +``` + +Setting up the `DO`/`LOOP` bytecode generation was the trickiest part of this project. It's a minefield of possible off-by-one errors. It's also not easy to read and understand but I've chosen to put it here anyway because even just glancing over it should help you understand how the loop variables (limit, iterator) and instruction pointer jumps are combined to execute loops in Forth. + +```tsx + // .. + + if (token.type === "do") { + index++; + + // Expect: DS has [limit, start] (start is top) + // Move both to RS: start then limit (RS top becomes limit) + bytecode.push({ op: "rs_push" }) // start -> RS + bytecode.push({ op: "rs_push" }) // limit -> RS + + // Mark first instruction of loop body + loopStart.push(bytecode.length); + continue; + } + + if (token.type === "loop") { + + // Pop limit and i from RS (RS top is limit) + bytecode.push({ op: "rs_pop" }) // limit -> DS + bytecode.push({ op: "rs_pop" }) // i -> DS + + // Increment i + bytecode.push({ op: "lit", value: 1 }) + bytecode.push({ op: "add" }) // i on DS + + // Duplicate i and limit for compare and possible restore + bytecode.push({ op: "dup2" }) + bytecode.push({ op: "eq" }) // eq flag on DS + + const loopStartAddress = loopStart.pop(); // first instr of loop body + + // Branch: continue when not equal (eq==0), exit when equal + const continueAddress = bytecode.length + 4; // skip equal-path (2 drops + jmp) + bytecode.push({ op: "jz", address: continueAddress }) + + // Equal path (fallthrough): cleanup and exit + bytecode.push({ op: "drop" }) // drop i + bytecode.push({ op: "drop" }) // drop limit + const afterBlockAddress = bytecode.length + 1 /* jmp */ + 3 /* continue block */; + bytecode.push({ op: "jmp", address: afterBlockAddress }) + + // Continue path: + // address == continueAddress + bytecode.push({ op: "rs_push" }) // i -> RS (top) + bytecode.push({ op: "rs_push" }) // limit -> RS + bytecode.push({ op: "jmp", address: loopStartAddress }) + + index++; + continue; + } + + // .. +``` + +The rest of the token branches are more straightforward. Tokens like dot, store, load, and print all map directly to bytecode operations. + +The colon token branch sets the bytecode offset for the word name which allows identifiers to become word calls as we saw above. + +Now we've earned a visualization break. + +
+ +## VM + +Writing the VM felt a little bit like dessert. Manually stepping through the bytecode as I worked on the generation logic gave me fairly good confidence that I was heading in the right direction, I only came across one or two off-by-one bugs when putting the VM together. Essentially, I had designed it ahead-of-time. + +The VM scans through the bytecode operations using the instruction pointer (which starts at `0`). The instruction pointer can jump around as it encounters `jmp` (jump to offset) or `jz` (conditional jump). + +It manages the data stack, return stack, and the variable table (i.e. memory addresses). + +
+ +Here's a trimmed version of the VM: + +```tsx +function vm(program: Program) => { + const dataStack: number[] = []; + const returnStack: number[] = []; + const variableTable: number[] = new Array(program.variableCount).fill(0); + + let ip = 0; + while (ip < program.bytecode.length) { + const cur = program.bytecode[ip]; + + if (cur.op === "lit") { + dataStack.push(cur.value); // Literal or memory address + ip++; + continue; + } else if (cur.op === "store") { + const address = dsPop(); + const value = dsPop(); + variableTable[address] = value; + ip++; + continue; + } else if (cur.op === "jmp") { + ip = cur.address; + continue; + } else if (cur.op === "jz") { + if (dsPop() === 0) { + ip = cur.address; + continue; + } + ip++; + continue; + } else if (cur.op === "call") { + ip++ + returnStack.push(ip); + ip = cur.address; + continue; + } else if (cur.op === "ret") { + ip = rsPop(); + continue; + } + + // .. trimmed other ops, see source + } +} +``` + +The code for my compiler and VM are [embedded in this website](https://github.com/healeycodes/healeycodes.com/blob/main/components/visuals/forth/components.tsx). I've been iterating on it by just running the TypeScript file: + +```bash +bun ./components/visuals/forth/components.tsx +55 # 10th Fibonacci number +``` + +The visuals are React components with sleeps. In order to display the progress of the different steps (tokenizing, bytecode generation, VM), I first got each working and then added a callback which takes the current data and then sleeps. + +So the VM function is actually async and accepts this callback: + +```tsx +// VM +async function vm(program: Program, callback: + ( + highlight: { ip: number }, + dataStack: number[], + returnStack: number[], + variableTable: number[] + ) => Promise) { + + // .. inside VM loop + await callback({ ip }, dataStack, returnStack, variableTable); + // .. + +} +``` + +And the component calls it and passes `setState` functions: + +```tsx +// Component +export function VM() { + + // .. inside useEffect + await vm(program, async (highlight, newDataStack, newReturnStack, newVariableTable) => { + setHighlightIP(highlight.ip); + setDataStack([...newDataStack]); + setReturnStack([...newReturnStack]); + setVariableTable([...newVariableTable]); + await new Promise(resolve => setTimeout(resolve, 500)); + }); + // .. + +} +``` + +For the Forth code snippets in this post, I had to write [a Prism plugin](https://github.com/healeycodes/healeycodes.com/blob/main/lib/prism-forth.js) to get syntax highlighting working. Now that I've learned how to do this, I'll be using this method for syntax highlighting for the more esoteric (or, original) programming languages I write about! + +## Discrepancies + +I described my compiler/VM as _Forth-like_ because it's a little bit different from how Forth works. + +My implementation compiles to bytecode ahead-of-time. Forth is traditionally interactive. Words are interpreted and executed as they are entered, and only colon definitions are compiled. Forth uses threaded code where words contain lists of addresses pointing to other words instead of a different bytecode offset. + +Real Forth uses a dynamic dictionary that can be altered at runtime with new variables or word definitions. As I mentioned earlier, my word bodies are compiled with jump-over logic in the main execution stream. Also, my variables compile to `lit address` operations but real Forth variables return their address when executed directly. + +These are just a few of the differences but I feel like my Forth-like compiler and VM capture enough of the spirit of Forth!