feat: enable async token counting with piscina workers

Port tokenizer logic into a Piscina worker pool so token counting no
longer blocks the main thread and exposes async APIs across IPC.
Render live token estimates in chat input with Suspense fallback and
tighten defensive assertions while adapting tests to the new async flow.

Author: ThomasK33

.github/workflows/chromatic.yml

@@ -36,4 +36,3 @@ jobs:
           projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
           exitZeroOnChanges: true
           onlyChanged: true
-

.github/workflows/publish-npm.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - main
     tags:
-      - 'v*'
+      - "v*"
   workflow_dispatch:
 
 permissions:
@@ -24,12 +24,12 @@ jobs:
 
       - uses: ./.github/actions/setup-cmux
         with:
-          install-imagemagick: 'true'
+          install-imagemagick: "true"
 
       # Sets up .npmrc with the auth token
       - uses: actions/setup-node@v4
         with:
-          registry-url: 'https://registry.npmjs.org'
+          registry-url: "https://registry.npmjs.org"
 
       - run: sudo npm i -g npm@latest
 
@@ -38,10 +38,10 @@ jobs:
         run: |
           # Get base version from package.json
           BASE_VERSION=$(node -p "require('./package.json').version")
-          
+
           # Generate git describe version
           GIT_DESCRIBE=$(git describe --tags --always --dirty 2>/dev/null || echo "unknown")
-          
+
           if [[ $GITHUB_REF == refs/tags/* ]]; then
             # For tags, use the base version as-is (stable release)
             NPM_VERSION="${BASE_VERSION}"
@@ -56,13 +56,13 @@ jobs:
             NPM_TAG="next"
             echo "Publishing pre-release: ${NPM_VERSION}"
           fi
-          
+
           echo "version=${NPM_VERSION}" >> $GITHUB_OUTPUT
           echo "tag=${NPM_TAG}" >> $GITHUB_OUTPUT
-          
+
           # Update package.json with the new version
           node -e "const fs = require('fs'); const pkg = JSON.parse(fs.readFileSync('package.json')); pkg.version = '${NPM_VERSION}'; fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');"
-          
+
           echo "Updated package.json to version ${NPM_VERSION}"
 
       - name: Generate version file
@@ -76,7 +76,7 @@ jobs:
         run: |
           PACKAGE_NAME=$(node -p "require('./package.json').name")
           VERSION="${{ steps.version.outputs.version }}"
-          
+
           if npm view "${PACKAGE_NAME}@${VERSION}" version &>/dev/null; then
             echo "exists=true" >> $GITHUB_OUTPUT
             echo "Version ${VERSION} already exists on npm"
@@ -95,7 +95,7 @@ jobs:
           PACKAGE_NAME=$(node -p "require('./package.json').name")
           VERSION="${{ steps.version.outputs.version }}"
           TAG="${{ steps.version.outputs.tag }}"
-          
+
           echo "Version ${VERSION} already published, updating dist-tag to ${TAG}"
           npm dist-tag add "${PACKAGE_NAME}@${VERSION}" "${TAG}"
 

.github/workflows/terminal-bench.yml

@@ -4,26 +4,26 @@ on:
   workflow_dispatch:
     inputs:
       dataset:
-        description: 'Terminal-Bench dataset to use'
+        description: "Terminal-Bench dataset to use"
         required: false
-        default: 'terminal-bench-core==0.1.1'
+        default: "terminal-bench-core==0.1.1"
         type: string
       concurrency:
-        description: 'Number of concurrent tasks (--n-concurrent)'
+        description: "Number of concurrent tasks (--n-concurrent)"
         required: false
-        default: '4'
+        default: "4"
         type: string
       livestream:
-        description: 'Enable livestream mode'
+        description: "Enable livestream mode"
         required: false
         default: true
         type: boolean
       sample_size:
-        description: 'Number of random tasks to run (empty = all tasks)'
+        description: "Number of random tasks to run (empty = all tasks)"
         required: false
         type: string
       extra_args:
-        description: 'Additional arguments to pass to terminal-bench'
+        description: "Additional arguments to pass to terminal-bench"
         required: false
         type: string
 
@@ -69,4 +69,3 @@ jobs:
             terminal-bench-results/
             *.json
           if-no-files-found: warn
-

.storybook/mocks/version.ts

@@ -6,4 +6,3 @@ export const VERSION = {
   git_describe: "v1.0.0",
   buildTime: "2024-01-24T17:41:00Z", // 9:41 AM PST
 };
-

bun.lock

@@ -6,4 +6,3 @@
   "exitOnceUploaded": true,
   "autoAcceptChanges": false
 }
-

chromatic.config.json

@@ -14,16 +14,16 @@ coverage:
     # Project coverage status - informational only
     project:
       default:
-        target: auto           # Compare to base commit
-        threshold: 5%          # Allow 5% drop without changing status
-        informational: true    # Won't block PRs
+        target: auto # Compare to base commit
+        threshold: 5% # Allow 5% drop without changing status
+        informational: true # Won't block PRs
 
-    # Patch coverage status - informational only  
+    # Patch coverage status - informational only
     patch:
       default:
-        target: auto           # Compare to base commit
-        threshold: 5%          # Allow 5% drop without changing status
-        informational: true    # Won't block PRs
+        target: auto # Compare to base commit
+        threshold: 5% # Allow 5% drop without changing status
+        informational: true # Won't block PRs
 
 # Flag configuration to track unit vs integration test coverage
 flags:

codecov.yml

@@ -15,4 +15,3 @@
     "utils": "@/lib/utils"
   }
 }
-

components.json

@@ -69,6 +69,7 @@
     "lru-cache": "^11.2.2",
     "markdown-it": "^14.1.0",
     "minimist": "^1.2.8",
+    "piscina": "^5.1.3",
     "rehype-harden": "^1.1.5",
     "shescape": "^2.1.6",
     "source-map-support": "^0.5.21",

package.json

@@ -32,4 +32,3 @@
     }
   ]
 }
-