Add tokenCache configuration option (fixes #119)

Author: bhouston

packages/agent/src/core/tokens.ts

@@ -73,6 +73,7 @@ export class TokenUsage {
 export class TokenTracker {
   public tokenUsage = new TokenUsage();
   public children: TokenTracker[] = [];
+  public tokenCache?: boolean;
 
   constructor(
     public readonly name: string = 'unnamed',

packages/agent/src/core/toolAgent/toolAgentCore.ts

@@ -57,11 +57,14 @@ export const toolAgent = async (
       });
     });
 
-    // Apply cache control to messages for token caching
-    const messagesWithCacheControl = [
-      createCacheControlMessageFromSystemPrompt(systemPrompt),
-      ...addCacheControlToMessages(messages),
-    ];
+    // Apply cache control to messages for token caching if enabled
+    const messagesWithCacheControl =
+      tokenTracker.tokenCache !== false && context.tokenCache !== false
+        ? [
+            createCacheControlMessageFromSystemPrompt(systemPrompt),
+            ...addCacheControlToMessages(messages),
+          ]
+        : [{ role: 'system', content: systemPrompt }, ...messages];
 
     const generateTextProps = {
       model: config.model,

packages/agent/src/core/types.ts

@@ -18,6 +18,7 @@ export type ToolContext = {
   tokenTracker: TokenTracker;
   githubMode: boolean;
   customPrompt?: string;
+  tokenCache?: boolean;
 };
 
 export type Tool<TParams = Record<string, any>, TReturn = any> = {

packages/cli/README.md

@@ -125,6 +125,7 @@ mycoder --modelProvider openai --modelName gpt-4o-2024-05-13 "Your prompt here"
 - `pageFilter`: Method to process webpage content: 'simple', 'none', or 'readability' (default: `none`)
 - `ollamaBaseUrl`: Base URL for Ollama API (default: `http://localhost:11434/api`)
 - `customPrompt`: Custom instructions to append to the system prompt for both main agent and sub-agents (default: `""`)
+- `tokenCache`: Enable token caching for LLM API calls (default: `true`)
 
 Example:
 
@@ -143,6 +144,9 @@ mycoder config set ollamaBaseUrl http://your-ollama-server:11434/api
 
 # Set custom instructions for the agent
 mycoder config set customPrompt "Always prioritize readability and simplicity in your code. Prefer TypeScript over JavaScript when possible."
+
+# Disable token caching for LLM API calls
+mycoder config set tokenCache false
 ```
 
 ## Environment Variables

packages/cli/src/commands/$default.ts

@@ -88,6 +88,9 @@ export const command: CommandModule<SharedOptions, DefaultArgs> = {
       undefined,
       argv.tokenUsage ? LogLevel.info : LogLevel.debug,
     );
+    // Use command line option if provided, otherwise use config value
+    tokenTracker.tokenCache =
+      argv.tokenCache !== undefined ? argv.tokenCache : userConfig.tokenCache;
 
     try {
       // Get configuration for model provider and name
@@ -177,6 +180,8 @@ export const command: CommandModule<SharedOptions, DefaultArgs> = {
         tokenTracker,
         githubMode: config.githubMode,
         customPrompt: config.customPrompt,
+        tokenCache:
+          argv.tokenCache !== undefined ? argv.tokenCache : config.tokenCache,
       });
 
       const output =

packages/cli/src/options.ts

@@ -10,6 +10,7 @@ export type SharedOptions = {
   readonly modelProvider?: string;
   readonly modelName?: string;
   readonly profile?: boolean;
+  readonly tokenCache?: boolean;
 };
 
 export const sharedOptions = {
@@ -72,4 +73,8 @@ export const sharedOptions = {
     description: 'Custom Sentry DSN for error tracking',
     hidden: true,
   } as const,
+  tokenCache: {
+    type: 'boolean',
+    description: 'Enable token caching for LLM API calls',
+  } as const,
 };

packages/cli/src/settings/config.ts

@@ -17,6 +17,7 @@ const defaultConfig = {
   ollamaBaseUrl: 'http://localhost:11434/api',
   customPrompt: '',
   profile: false,
+  tokenCache: true,
 };
 
 export type Config = typeof defaultConfig;