Skip to content

Commit 3581ebf

Browse files
committed
🤖 fix: disable extended thinking for workspace name generation
When generating workspace names with reasoning models (e.g., claude-opus-4-5), the API sometimes returns thinking content that breaks JSON parsing, causing 'Invalid JSON response' errors. Fix: Explicitly pass providerOptions with thinking disabled to generateObject(). - Add buildProviderOptions(modelString, 'off') to ensure no thinking content - Add ModelProvider interface for easier testing without full AIService - Add unit/integration tests for workspace title generation - Add testUtils.ts with minimal model creation for tests
1 parent 478ee1f commit 3581ebf

File tree

3 files changed

+141
-3
lines changed

3 files changed

+141
-3
lines changed

src/node/services/testUtils.ts

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/**
2+
* Test utilities for AI service tests
3+
*
4+
* Provides minimal implementations for testing without full service infrastructure.
5+
*/
6+
7+
import type { LanguageModel } from "ai";
8+
import type { Result } from "@/common/types/result";
9+
import { Ok, Err } from "@/common/types/result";
10+
import type { SendMessageError } from "@/common/types/errors";
11+
12+
/**
13+
* Create a model for testing - minimal implementation that uses the AI SDK directly.
14+
* Supports Anthropic models only (add more providers as needed).
15+
*/
16+
export async function createModelForTest(
17+
modelString: string
18+
): Promise<Result<LanguageModel, SendMessageError>> {
19+
const [provider, modelId] = modelString.split(":");
20+
21+
if (!provider || !modelId) {
22+
return Err({
23+
type: "invalid_model_string",
24+
message: `Invalid model string: ${modelString}`,
25+
});
26+
}
27+
28+
if (provider === "anthropic") {
29+
const apiKey = process.env.ANTHROPIC_API_KEY ?? process.env.ANTHROPIC_AUTH_TOKEN;
30+
if (!apiKey) {
31+
return Err({ type: "api_key_not_found", provider: "anthropic" });
32+
}
33+
34+
// Dynamic import is acceptable here - test utility only, not production code
35+
// eslint-disable-next-line no-restricted-syntax
36+
const { createAnthropic } = await import("@ai-sdk/anthropic");
37+
const anthropic = createAnthropic({ apiKey });
38+
return Ok(anthropic(modelId));
39+
}
40+
41+
return Err({ type: "provider_not_supported", provider });
42+
}
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/**
2+
* Tests for workspace title generator
3+
*
4+
* Uses real API calls (TEST_INTEGRATION=1) to verify:
5+
* - Small prompts work correctly
6+
* - Reasoning/thinking models don't break JSON parsing
7+
*/
8+
9+
import { generateWorkspaceName, generatePlaceholderName } from "./workspaceTitleGenerator";
10+
import { createModelForTest } from "./testUtils";
11+
12+
// Unit tests - always run
13+
describe("generatePlaceholderName", () => {
14+
it("should generate git-safe placeholder from message", () => {
15+
expect(generatePlaceholderName("Fix the login bug")).toBe("fix-the-login-bug");
16+
});
17+
18+
it("should handle empty message", () => {
19+
expect(generatePlaceholderName("")).toBe("new-workspace");
20+
});
21+
22+
it("should truncate long messages", () => {
23+
const longMessage =
24+
"This is a very long message that should be truncated to fit within the limit";
25+
const result = generatePlaceholderName(longMessage);
26+
expect(result.length).toBeLessThanOrEqual(30);
27+
});
28+
29+
it("should sanitize special characters", () => {
30+
expect(generatePlaceholderName("Fix: the @login #bug!")).toBe("fix-the-login-bug");
31+
});
32+
});
33+
34+
// Integration tests - require TEST_INTEGRATION=1
35+
const describeIntegration = process.env.TEST_INTEGRATION === "1" ? describe : describe.skip;
36+
37+
describeIntegration("generateWorkspaceName - integration", () => {
38+
// Minimal AIService-like object that only provides createModel
39+
const aiService = { createModel: createModelForTest };
40+
41+
// Test with a small prompt that triggered the bug
42+
it("should handle small prompts with opus-4-5 (reasoning model)", async () => {
43+
const message = "Solve https://github.com/coder/registry/issues/42";
44+
const model = "anthropic:claude-opus-4-5";
45+
46+
const result = await generateWorkspaceName(message, model, aiService);
47+
48+
// The result should be successful, not fail with "Invalid JSON response"
49+
expect(result.success).toBe(true);
50+
if (result.success) {
51+
expect(result.data).toMatch(/^[a-z0-9-]+$/);
52+
expect(result.data.length).toBeGreaterThanOrEqual(3);
53+
expect(result.data.length).toBeLessThanOrEqual(50);
54+
}
55+
}, 30000);
56+
57+
// Test with very short prompt
58+
it("should handle very short prompts", async () => {
59+
const message = "fix bug";
60+
const model = "anthropic:claude-opus-4-5";
61+
62+
const result = await generateWorkspaceName(message, model, aiService);
63+
64+
expect(result.success).toBe(true);
65+
if (result.success) {
66+
expect(result.data).toMatch(/^[a-z0-9-]+$/);
67+
}
68+
}, 30000);
69+
70+
// Test with claude-sonnet-4-5 (thinking model but different config)
71+
it("should handle sonnet-4-5 with small prompts", async () => {
72+
const message = "update README";
73+
const model = "anthropic:claude-sonnet-4-5";
74+
75+
const result = await generateWorkspaceName(message, model, aiService);
76+
77+
expect(result.success).toBe(true);
78+
if (result.success) {
79+
expect(result.data).toMatch(/^[a-z0-9-]+$/);
80+
}
81+
}, 30000);
82+
});

src/node/services/workspaceTitleGenerator.ts

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
import { generateObject } from "ai";
1+
import { generateObject, type LanguageModel } from "ai";
22
import { z } from "zod";
3-
import type { AIService } from "./aiService";
43
import { log } from "./log";
54
import type { Result } from "@/common/types/result";
65
import { Ok, Err } from "@/common/types/result";
76
import type { SendMessageError } from "@/common/types/errors";
7+
import { buildProviderOptions } from "@/common/utils/ai/providerOptions";
88

99
const workspaceNameSchema = z.object({
1010
name: z
@@ -15,26 +15,40 @@ const workspaceNameSchema = z.object({
1515
.describe("Git-safe branch/workspace name: lowercase, hyphens only"),
1616
});
1717

18+
/** Minimal interface for model creation - allows testing without full AIService */
19+
interface ModelProvider {
20+
createModel(modelString: string): Promise<Result<LanguageModel, SendMessageError>>;
21+
}
22+
1823
/**
1924
* Generate workspace name using AI.
2025
* If AI cannot be used (e.g. missing credentials, unsupported provider, invalid model),
2126
* returns a SendMessageError so callers can surface the standard provider error UX.
27+
*
28+
* Explicitly disables extended thinking to ensure clean JSON responses.
2229
*/
2330
export async function generateWorkspaceName(
2431
message: string,
2532
modelString: string,
26-
aiService: AIService
33+
aiService: ModelProvider
2734
): Promise<Result<string, SendMessageError>> {
2835
try {
2936
const modelResult = await aiService.createModel(modelString);
3037
if (!modelResult.success) {
3138
return Err(modelResult.error);
3239
}
3340

41+
// Explicitly disable extended thinking for workspace name generation.
42+
// Reasoning models sometimes return thinking content that breaks JSON parsing.
43+
const providerOptions = buildProviderOptions(modelString, "off");
44+
3445
const result = await generateObject({
3546
model: modelResult.data,
3647
schema: workspaceNameSchema,
3748
prompt: `Generate a git-safe branch/workspace name for this development task:\n\n"${message}"\n\nRequirements:\n- Git-safe identifier (e.g., "automatic-title-generation")\n- Lowercase, hyphens only, no spaces\n- Concise (2-5 words) and descriptive of the task`,
49+
// Cast needed: our ProviderOptions type is stricter than AI SDK's generic type
50+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
51+
providerOptions: providerOptions as any,
3852
});
3953

4054
return Ok(validateBranchName(result.object.name));

0 commit comments

Comments
 (0)