fix: allow tokenizer fallback when Piscina unavailable

Detect Bun runtime and bypass Piscina so token counting works in tests.
Assert encoding responses and reuse worker logic when pooling is off.
Disable Git signing in tests and align fixtures with current adapters.

Author: ThomasK33

src/git.test.ts

@@ -21,6 +21,7 @@ describe("createWorktree", () => {
     await execAsync(`git init`, { cwd: tempGitRepo });
     await execAsync(`git config user.email "test@example.com"`, { cwd: tempGitRepo });
     await execAsync(`git config user.name "Test User"`, { cwd: tempGitRepo });
+    await execAsync(`git config commit.gpgsign false`, { cwd: tempGitRepo });
     await execAsync(`echo "test" > README.md`, { cwd: tempGitRepo });
     await execAsync(`git add .`, { cwd: tempGitRepo });
     await execAsync(`git commit -m "Initial commit"`, { cwd: tempGitRepo });

src/runtime/sshConnectionPool.test.ts

@@ -1,4 +1,5 @@
 import * as os from "os";
+import * as path from "path";
 import { getControlPath } from "./sshConnectionPool";
 import type { SSHRuntimeConfig } from "./SSHRuntime";
 
@@ -127,6 +128,9 @@ describe("username isolation", () => {
 
     // The path should be deterministic for this user
     expect(controlPath).toBe(getControlPath(config));
-    expect(controlPath).toMatch(/^\/tmp\/cmux-ssh-[a-f0-9]{12}$/);
+
+    const expectedPrefix = path.join(os.tmpdir(), "cmux-ssh-");
+    expect(controlPath.startsWith(expectedPrefix)).toBe(true);
+    expect(controlPath).toMatch(/cmux-ssh-[a-f0-9]{12}$/);
   });
 });

src/services/ipcMain.ts

@@ -1,5 +1,4 @@
 import assert from "@/utils/assert";
-import crypto from "crypto";
 import type { BrowserWindow, IpcMain as ElectronIpcMain } from "electron";
 import { spawn, spawnSync } from "child_process";
 import * as fs from "fs";
@@ -29,7 +28,6 @@ import { BashExecutionService } from "@/services/bashExecutionService";
 import { InitStateManager } from "@/services/initStateManager";
 import { createRuntime } from "@/runtime/runtimeFactory";
 import type { RuntimeConfig } from "@/types/runtime";
-import { uniqueSuffix } from "@/utils/hasher";
 /**
  * IpcMain - Manages all IPC handlers and service coordination
  *

src/services/streamManager.test.ts

@@ -379,7 +379,7 @@ describe("StreamManager - Unavailable Tool Handling", () => {
       messageId: "test-message-1",
       token: "test-token",
       startTime: Date.now(),
-      model: "test-model",
+      model: "anthropic:claude-sonnet-4-5",
       historySequence: 1,
       parts: [],
       lastPartialWriteTime: 0,

src/stores/WorkspaceConsumerManager.ts

@@ -250,8 +250,8 @@ export class WorkspaceConsumerManager {
       typeof queueMicrotask === "function"
         ? queueMicrotask
         : (callback: () => void) => {
-          void Promise.resolve().then(callback);
-        };
+            void Promise.resolve().then(callback);
+          };
 
     schedule(() => {
       this.pendingNotifications.delete(workspaceId);

src/utils/main/tokenizer.ts

@@ -5,6 +5,10 @@ import { getAvailableTools, getToolSchemas } from "@/utils/tools/toolDefinitions
 import Piscina from "piscina";
 import { createRequire } from "module";
 import type { CountTokensInput } from "./tokenizer.worker";
+import {
+  countTokens as workerCountTokens,
+  encodingName as workerEncodingName,
+} from "./tokenizer.worker";
 import { models, type ModelName } from "ai-tokenizer";
 
 /**
@@ -27,11 +31,18 @@ const DEFAULT_WARM_MODELS = [
 ] as const;
 
 const requireForResolve = createRequire(__filename);
-const workerPath = requireForResolve.resolve("./tokenizer-pool.worker");
-const tokenizerPool = new Piscina({
-  filename: workerPath,
-  idleTimeout: Infinity,
-});
+const supportsPiscina =
+  typeof process !== "undefined" &&
+  !!process.versions?.node &&
+  !("bun" in (process.versions ?? {}));
+
+const workerPath = supportsPiscina ? requireForResolve.resolve("./tokenizer.worker") : null;
+const tokenizerPool = supportsPiscina
+  ? new Piscina({
+      filename: workerPath!,
+      idleTimeout: Infinity,
+    })
+  : null;
 
 const encodingPromises = new Map<ModelName, Promise<string>>();
 const inFlightCounts = new Map<string, Promise<number>>();
@@ -57,6 +68,12 @@ function normalizeModelKey(modelName: string): ModelName {
 }
 
 async function resolveEncoding(modelName: ModelName): Promise<string> {
+  if (!tokenizerPool) {
+    const encoding = workerEncodingName(modelName);
+    assert(typeof encoding === "string" && encoding.length > 0);
+    return encoding;
+  }
+
   let promise = encodingPromises.get(modelName);
   if (!promise) {
     promise = tokenizerPool
@@ -94,6 +111,16 @@ async function countTokensInternal(modelName: ModelName, text: string): Promise<
     return cached;
   }
 
+  if (!tokenizerPool) {
+    const value = workerCountTokens({ modelName, input: text });
+    assert(
+      typeof value === "number" && Number.isFinite(value) && value >= 0,
+      "Tokenizer must return a non-negative finite token count"
+    );
+    tokenCountCache.set(key, value);
+    return value;
+  }
+
   let pending = inFlightCounts.get(key);
   if (!pending) {
     const payload: CountTokensInput = { modelName, input: text };

src/utils/main/tokenizer.worker.ts

@@ -1,10 +1,7 @@
 import assert from "@/utils/assert";
 import { Tokenizer, models } from "ai-tokenizer";
 import type { ModelName } from "ai-tokenizer";
-// import * as claude from "ai-tokenizer/encoding/claude";
-// import * as o200k_base from "ai-tokenizer/encoding/o200k_base";
-//
-// import * as encoding from "ai-tokenizer/encoding";
+import * as encoding from "ai-tokenizer/encoding";
 
 export interface CountTokensInput {
   modelName: ModelName;
@@ -13,43 +10,25 @@ export interface CountTokensInput {
 
 const tokenizerCache = new Map<ModelName, Tokenizer>();
 
-type importedClaude = typeof import("ai-tokenizer/encoding/claude");
-type importedO200k_base = typeof import("ai-tokenizer/encoding/o200k_base");
-let claude: importedClaude | null = null;
-let o200k_base: importedO200k_base | null = null;
-
-async function jitLoadEncoding(encoding: string): Promise<importedClaude | importedO200k_base> {
-  if (encoding == "claude") {
-    claude ??= await import("ai-tokenizer/encoding/claude");
-    return claude;
-  }
-
-  o200k_base ??= await import("ai-tokenizer/encoding/o200k_base");
-  return o200k_base;
-}
-
-async function getTokenizer(modelName: ModelName): Promise<Tokenizer> {
+function getTokenizer(modelName: ModelName): Tokenizer {
   const cached = tokenizerCache.get(modelName);
   if (cached) {
     return cached;
   }
 
   const model = models[modelName];
   assert(model, `Unknown tokenizer model '${modelName}'`);
-  assert(
-    model.encoding == "claude" || model.encoding == "o200k_base",
-    `Unknown encoding '${model.encoding}' for '${modelName}'`
-  );
 
-  const encodingModule = await jitLoadEncoding(model.encoding);
+  const encodingModule = encoding[model.encoding];
+  assert(encodingModule, `Unknown tokenizer encoding '${model.encoding}'`);
 
   const tokenizer = new Tokenizer(encodingModule);
   tokenizerCache.set(modelName, tokenizer);
   return tokenizer;
 }
 
-export async function countTokens({ modelName, input }: CountTokensInput): Promise<number> {
-  const tokenizer = await getTokenizer(modelName);
+export function countTokens({ modelName, input }: CountTokensInput): number {
+  const tokenizer = getTokenizer(modelName);
   const count = tokenizer.count(input);
   return count;
 }

src/utils/tokens/tokenStatsCalculator.test.ts

@@ -343,7 +343,7 @@ describe("mergeResults", () => {
       { consumer: "Assistant", promise: Promise.resolve(200) },
     ];
     const results = [100, 200];
-    const toolDefinitions = new Map();
+    const toolDefinitions = new Map<string, number>();
     const systemMessageTokens = 0;
 
     const consumerMap = mergeResults(jobs, results, toolDefinitions, systemMessageTokens);
@@ -358,7 +358,7 @@ describe("mergeResults", () => {
       { consumer: "User", promise: Promise.resolve(50) },
     ];
     const results = [100, 50];
-    const toolDefinitions = new Map();
+    const toolDefinitions = new Map<string, number>();
     const systemMessageTokens = 0;
 
     const consumerMap = mergeResults(jobs, results, toolDefinitions, systemMessageTokens);
@@ -372,7 +372,7 @@ describe("mergeResults", () => {
       { consumer: "Read", promise: Promise.resolve(50) },
     ];
     const results = [100, 50];
-    const toolDefinitions = new Map([["Read", 25]]);
+    const toolDefinitions = new Map<string, number>([["Read", 25]]);
     const systemMessageTokens = 0;
 
     const consumerMap = mergeResults(jobs, results, toolDefinitions, systemMessageTokens);
@@ -384,7 +384,7 @@ describe("mergeResults", () => {
   test("adds system message tokens", () => {
     const jobs: TokenCountJob[] = [];
     const results: number[] = [];
-    const toolDefinitions = new Map();
+    const toolDefinitions = new Map<string, number>();
     const systemMessageTokens = 300;
 
     const consumerMap = mergeResults(jobs, results, toolDefinitions, systemMessageTokens);
@@ -398,7 +398,7 @@ describe("mergeResults", () => {
       { consumer: "Assistant", promise: Promise.resolve(100) },
     ];
     const results = [0, 100];
-    const toolDefinitions = new Map();
+    const toolDefinitions = new Map<string, number>();
     const systemMessageTokens = 0;
 
     const consumerMap = mergeResults(jobs, results, toolDefinitions, systemMessageTokens);