Merge pull request #121 from drivecore/feature/add-token-cache-option

Add tokenCache configuration option

Author: bhouston

.changeset/itchy-hounds-cross.md

@@ -0,0 +1,6 @@
+---
+'mycoder-agent': patch
+'mycoder': patch
+---
+
+Add ability to enable/disable token caching via config values

packages/agent/src/core/tokens.ts

@@ -73,6 +73,7 @@ export class TokenUsage {
 export class TokenTracker {
   public tokenUsage = new TokenUsage();
   public children: TokenTracker[] = [];
+  public tokenCache?: boolean;
 
   constructor(
     public readonly name: string = 'unnamed',

packages/agent/src/core/toolAgent/toolAgentCore.ts

@@ -57,11 +57,20 @@ export const toolAgent = async (
       });
     });
 
-    // Apply cache control to messages for token caching
-    const messagesWithCacheControl = [
-      createCacheControlMessageFromSystemPrompt(systemPrompt),
-      ...addCacheControlToMessages(messages),
-    ];
+    // Apply cache control to messages for token caching if enabled
+    const messagesWithCacheControl =
+      tokenTracker.tokenCache !== false && context.tokenCache !== false
+        ? [
+            createCacheControlMessageFromSystemPrompt(systemPrompt),
+            ...addCacheControlToMessages(messages),
+          ]
+        : [
+            {
+              role: 'system',
+              content: systemPrompt,
+            } as CoreMessage,
+            ...messages,
+          ];
 
     const generateTextProps = {
       model: config.model,

packages/agent/src/core/types.ts

@@ -18,6 +18,7 @@ export type ToolContext = {
   tokenTracker: TokenTracker;
   githubMode: boolean;
   customPrompt?: string;
+  tokenCache?: boolean;
 };
 
 export type Tool<TParams = Record<string, any>, TReturn = any> = {

packages/cli/README.md

@@ -125,6 +125,7 @@ mycoder --modelProvider openai --modelName gpt-4o-2024-05-13 "Your prompt here"
 - `pageFilter`: Method to process webpage content: 'simple', 'none', or 'readability' (default: `none`)
 - `ollamaBaseUrl`: Base URL for Ollama API (default: `http://localhost:11434/api`)
 - `customPrompt`: Custom instructions to append to the system prompt for both main agent and sub-agents (default: `""`)
+- `tokenCache`: Enable token caching for LLM API calls (default: `true`)
 
 Example:
 
@@ -143,6 +144,9 @@ mycoder config set ollamaBaseUrl http://your-ollama-server:11434/api
 
 # Set custom instructions for the agent
 mycoder config set customPrompt "Always prioritize readability and simplicity in your code. Prefer TypeScript over JavaScript when possible."
+
+# Disable token caching for LLM API calls
+mycoder config set tokenCache false
 ```
 
 ## Environment Variables

packages/cli/src/commands/$default.ts

@@ -92,6 +92,10 @@ export const command: CommandModule<SharedOptions, DefaultArgs> = {
     try {
       // Get configuration for model provider and name
       const userConfig = getConfig();
+      // Use command line option if provided, otherwise use config value
+      tokenTracker.tokenCache =
+        argv.tokenCache !== undefined ? argv.tokenCache : userConfig.tokenCache;
+
       const userModelProvider = argv.modelProvider || userConfig.modelProvider;
       const userModelName = argv.modelName || userConfig.modelName;
       const userMaxTokens = argv.maxTokens || userConfig.maxTokens;
@@ -181,6 +185,8 @@ export const command: CommandModule<SharedOptions, DefaultArgs> = {
         tokenTracker,
         githubMode: config.githubMode,
         customPrompt: config.customPrompt,
+        tokenCache:
+          argv.tokenCache !== undefined ? argv.tokenCache : config.tokenCache,
       });
 
       const output =

packages/cli/src/options.ts

@@ -12,6 +12,7 @@ export type SharedOptions = {
   readonly maxTokens?: number;
   readonly temperature?: number;
   readonly profile?: boolean;
+  readonly tokenCache?: boolean;
 };
 
 export const sharedOptions = {
@@ -82,4 +83,8 @@ export const sharedOptions = {
     description: 'Custom Sentry DSN for error tracking',
     hidden: true,
   } as const,
+  tokenCache: {
+    type: 'boolean',
+    description: 'Enable token caching for LLM API calls',
+  } as const,
 };

packages/cli/src/settings/config.ts

@@ -19,6 +19,7 @@ const defaultConfig = {
   ollamaBaseUrl: 'http://localhost:11434/api',
   customPrompt: '',
   profile: false,
+  tokenCache: true,
 };
 
 export type Config = typeof defaultConfig;

packages/cli/tests/settings/config.test.ts

@@ -48,6 +48,7 @@ describe('Config', () => {
         ollamaBaseUrl: 'http://localhost:11434/api',
         profile: false,
         customPrompt: '',
+        tokenCache: true,
       });
       expect(fs.existsSync).toHaveBeenCalledWith(mockConfigFile);
     });
@@ -84,6 +85,7 @@ describe('Config', () => {
         ollamaBaseUrl: 'http://localhost:11434/api',
         profile: false,
         customPrompt: '',
+        tokenCache: true,
       });
     });
   });