Skip to content

Commit 3b7a93d

Browse files
bhoustonbhouston
committed
simplify anthropic context window determination
1 parent cd51c96 commit 3b7a93d

File tree

1 file changed

+83
-113
lines changed

1 file changed

+83
-113
lines changed

‎packages/agent/src/core/llm/providers/anthropic.ts‎

Lines changed: 83 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ function addCacheControlToMessages(
8787
function tokenUsageFromMessage(
8888
message: Anthropic.Message,
8989
model: string,
90-
contextWindow?: number,
90+
contextWindow: number,
9191
) {
9292
const usage = new TokenUsage();
9393
usage.input = message.usage.input_tokens;
@@ -97,19 +97,10 @@ function tokenUsageFromMessage(
9797

9898
const totalTokens = usage.input + usage.output;
9999

100-
// Use provided context window or fallback to cached value
101-
const maxTokens = contextWindow || modelContextWindowCache[model];
102-
103-
if (!maxTokens) {
104-
throw new Error(
105-
`Context window size not available for model: ${model}. Make sure to initialize the model properly.`,
106-
);
107-
}
108-
109100
return {
110101
usage,
111102
totalTokens,
112-
maxTokens,
103+
maxTokens: contextWindow,
113104
};
114105
}
115106

@@ -123,7 +114,6 @@ export class AnthropicProvider implements LLMProvider {
123114
private client: Anthropic;
124115
private apiKey: string;
125116
private baseUrl?: string;
126-
private modelContextWindow?: number;
127117

128118
constructor(model: string, options: AnthropicOptions = {}) {
129119
this.model = model;
@@ -139,15 +129,6 @@ export class AnthropicProvider implements LLMProvider {
139129
apiKey: this.apiKey,
140130
...(this.baseUrl && { baseURL: this.baseUrl }),
141131
});
142-
143-
// Initialize model context window detection
144-
// This is async but we don't need to await it here
145-
// If it fails, an error will be thrown when the model is used
146-
this.initializeModelContextWindow().catch((error) => {
147-
console.error(
148-
`Failed to initialize model context window: ${error.message}. The model will not work until context window information is available.`,
149-
);
150-
});
151132
}
152133

153134
/**
@@ -156,54 +137,49 @@ export class AnthropicProvider implements LLMProvider {
156137
* @returns The context window size
157138
* @throws Error if the context window size cannot be determined
158139
*/
159-
private async initializeModelContextWindow(): Promise<number> {
160-
try {
161-
const response = await this.client.models.list();
140+
private async getModelContextWindow(): Promise<number> {
141+
const cachedContextWindow = modelContextWindowCache[this.model];
142+
if (cachedContextWindow !== undefined) {
143+
return cachedContextWindow;
144+
}
145+
const response = await this.client.models.list();
162146

163-
if (!response?.data || !Array.isArray(response.data)) {
164-
throw new Error(
165-
`Invalid response from models.list() for ${this.model}`,
166-
);
167-
}
147+
if (!response?.data || !Array.isArray(response.data)) {
148+
throw new Error(`Invalid response from models.list() for ${this.model}`);
149+
}
168150

169-
// Try to find the exact model
170-
let model = response.data.find((m) => m.id === this.model);
151+
// Try to find the exact model
152+
let model = response.data.find((m) => m.id === this.model);
171153

172-
// If not found, try to find a model that starts with the same name
173-
// This helps with model aliases like 'claude-3-sonnet-latest'
174-
if (!model) {
175-
// Split by '-latest' or '-20' to get the base model name
176-
const parts = this.model.split('-latest');
177-
const modelPrefix =
178-
parts.length > 1 ? parts[0] : this.model.split('-20')[0];
154+
// If not found, try to find a model that starts with the same name
155+
// This helps with model aliases like 'claude-3-sonnet-latest'
156+
if (!model) {
157+
// Split by '-latest' or '-20' to get the base model name
158+
const parts = this.model.split('-latest');
159+
const modelPrefix =
160+
parts.length > 1 ? parts[0] : this.model.split('-20')[0];
179161

180-
if (modelPrefix) {
181-
model = response.data.find((m) => m.id.startsWith(modelPrefix));
162+
if (modelPrefix) {
163+
model = response.data.find((m) => m.id.startsWith(modelPrefix));
182164

183-
if (model) {
184-
console.info(
185-
`Model ${this.model} not found, using ${model.id} for context window size`,
186-
);
187-
}
165+
if (model) {
166+
console.info(
167+
`Model ${this.model} not found, using ${model.id} for context window size`,
168+
);
188169
}
189170
}
171+
}
190172

191-
// Using type assertion to access context_window property
192-
// The Anthropic API returns context_window but it may not be in the TypeScript definitions
193-
if (model && 'context_window' in model) {
194-
const contextWindow = (model as any).context_window;
195-
this.modelContextWindow = contextWindow;
196-
// Cache the result for future use
197-
modelContextWindowCache[this.model] = contextWindow;
198-
return contextWindow;
199-
} else {
200-
throw new Error(
201-
`No context window information found for model: ${this.model}`,
202-
);
203-
}
204-
} catch (error) {
173+
// Using type assertion to access context_window property
174+
// The Anthropic API returns context_window but it may not be in the TypeScript definitions
175+
if (model && 'context_window' in model) {
176+
const contextWindow = (model as any).context_window;
177+
// Cache the result for future use
178+
modelContextWindowCache[this.model] = contextWindow;
179+
return contextWindow;
180+
} else {
205181
throw new Error(
206-
`Failed to determine context window size for model ${this.model}: ${(error as Error).message}`,
182+
`No context window information found for model: ${this.model}`,
207183
);
208184
}
209185
}
@@ -212,6 +188,7 @@ export class AnthropicProvider implements LLMProvider {
212188
* Generate text using Anthropic API
213189
*/
214190
async generateText(options: GenerateOptions): Promise<LLMResponse> {
191+
const modelContextWindow = await this.getModelContextWindow();
215192
const { messages, functions, temperature = 0.7, maxTokens, topP } = options;
216193

217194
// Extract system message
@@ -227,63 +204,56 @@ export class AnthropicProvider implements LLMProvider {
227204
})),
228205
);
229206

230-
try {
231-
const requestOptions: Anthropic.MessageCreateParams = {
232-
model: this.model,
233-
messages: addCacheControlToMessages(formattedMessages),
234-
temperature,
235-
max_tokens: maxTokens || 1024,
236-
system: systemMessage?.content
237-
? [
238-
{
239-
type: 'text',
240-
text: systemMessage?.content,
241-
cache_control: { type: 'ephemeral' },
242-
},
243-
]
244-
: undefined,
245-
top_p: topP,
246-
tools,
247-
stream: false,
248-
};
207+
const requestOptions: Anthropic.MessageCreateParams = {
208+
model: this.model,
209+
messages: addCacheControlToMessages(formattedMessages),
210+
temperature,
211+
max_tokens: maxTokens || 1024,
212+
system: systemMessage?.content
213+
? [
214+
{
215+
type: 'text',
216+
text: systemMessage?.content,
217+
cache_control: { type: 'ephemeral' },
218+
},
219+
]
220+
: undefined,
221+
top_p: topP,
222+
tools,
223+
stream: false,
224+
};
249225

250-
const response = await this.client.messages.create(requestOptions);
226+
const response = await this.client.messages.create(requestOptions);
251227

252-
// Extract content and tool calls
253-
const content =
254-
response.content.find((c) => c.type === 'text')?.text || '';
255-
const toolCalls = response.content
256-
.filter((c) => {
257-
const contentType = c.type;
258-
return contentType === 'tool_use';
259-
})
260-
.map((c) => {
261-
const toolUse = c as Anthropic.Messages.ToolUseBlock;
262-
return {
263-
id: toolUse.id,
264-
name: toolUse.name,
265-
content: JSON.stringify(toolUse.input),
266-
};
267-
});
228+
// Extract content and tool calls
229+
const content = response.content.find((c) => c.type === 'text')?.text || '';
230+
const toolCalls = response.content
231+
.filter((c) => {
232+
const contentType = c.type;
233+
return contentType === 'tool_use';
234+
})
235+
.map((c) => {
236+
const toolUse = c as Anthropic.Messages.ToolUseBlock;
237+
return {
238+
id: toolUse.id,
239+
name: toolUse.name,
240+
content: JSON.stringify(toolUse.input),
241+
};
242+
});
268243

269-
const tokenInfo = tokenUsageFromMessage(
270-
response,
271-
this.model,
272-
this.modelContextWindow,
273-
);
244+
const tokenInfo = tokenUsageFromMessage(
245+
response,
246+
this.model,
247+
modelContextWindow,
248+
);
274249

275-
return {
276-
text: content,
277-
toolCalls: toolCalls,
278-
tokenUsage: tokenInfo.usage,
279-
totalTokens: tokenInfo.totalTokens,
280-
maxTokens: tokenInfo.maxTokens,
281-
};
282-
} catch (error) {
283-
throw new Error(
284-
`Error calling Anthropic API: ${(error as Error).message}`,
285-
);
286-
}
250+
return {
251+
text: content,
252+
toolCalls: toolCalls,
253+
tokenUsage: tokenInfo.usage,
254+
totalTokens: tokenInfo.totalTokens,
255+
maxTokens: tokenInfo.maxTokens,
256+
};
287257
}
288258

289259
/**

0 commit comments

Comments
 (0)