Skip to content

Commit 6984a5f

Browse files
brandonkachenbrandonkachen
committed
feat(sdk): new e2e tests and cleaned up existing ones
- E2E tests: streaming, workflows, custom-agents, features - Integration tests: event-types, event-ordering, stream-chunks, connection-check - Unit tests: EventCollector utility class (25 tests) - Examples: code-reviewer, code-explainer, commit-message-generator, sdk-lint, sdk-refactor, sdk-test-gen - Add test:e2e, test:integration, test:unit:e2e scripts to package.json
1 parent 8d93c93 commit 6984a5f

27 files changed

+2926
-0
lines changed

‎sdk/e2e/README.md‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
# SDK Tests & Examples
2+
3+
This folder contains end-to-end tests, integration tests, unit tests, and runnable examples for the Codebuff SDK.
4+
5+
## Directory Structure
6+
7+
```
8+
sdk/e2e/
9+
├── streaming/ # E2E tests for streaming behavior
10+
├── workflows/ # E2E tests for multi-step workflows
11+
├── custom-agents/ # E2E tests for custom agent & tool integration
12+
├── features/ # E2E tests for SDK feature coverage
13+
├── integration/ # Integration tests (SDK + API mechanics)
14+
├── examples/ # Runnable example scripts (not tests)
15+
└── utils/ # Shared utilities
16+
├── __tests__/ # Unit tests for utilities
17+
├── event-collector.ts
18+
├── get-api-key.ts
19+
└── test-fixtures.ts
20+
```
21+
22+
## Test Categories
23+
24+
### E2E Tests (`test:e2e`)
25+
Full end-to-end tests that exercise complete user workflows with real API calls.
26+
27+
```bash
28+
bun run test:e2e
29+
```
30+
31+
| Category | Description |
32+
|----------|-------------|
33+
| `streaming/` | Subagent streaming, concurrent streams |
34+
| `workflows/` | Multi-turn conversations, error recovery |
35+
| `custom-agents/` | Custom agents with custom tools |
36+
| `features/` | projectFiles, knowledgeFiles, maxAgentSteps |
37+
38+
### Integration Tests (`test:integration`)
39+
Tests that verify SDK + API integration mechanics.
40+
41+
```bash
42+
bun run test:integration
43+
```
44+
45+
| Test | Description |
46+
|------|-------------|
47+
| `event-types` | Validates all PrintModeEvent types are emitted |
48+
| `event-ordering` | Validates event sequence (start → content → finish) |
49+
| `stream-chunks` | Tests handleStreamChunk callback |
50+
| `connection-check` | Tests checkConnection() method |
51+
52+
### Unit Tests (`test:unit:e2e`)
53+
Pure unit tests with no external dependencies.
54+
55+
```bash
56+
bun run test:unit:e2e
57+
```
58+
59+
| Test | Description |
60+
|------|-------------|
61+
| `event-collector.test.ts` | Tests EventCollector utility class |
62+
63+
### Examples
64+
Runnable scripts demonstrating SDK usage patterns. Not tests - just examples!
65+
66+
```bash
67+
# Run an example
68+
bun run sdk/e2e/examples/code-reviewer.example.ts
69+
```
70+
71+
| Example | Description |
72+
|---------|-------------|
73+
| `code-reviewer.example.ts` | AI code review |
74+
| `code-explainer.example.ts` | Explain code in plain English |
75+
| `commit-message-generator.example.ts` | Generate commit messages from diffs |
76+
| `sdk-lint.example.ts` | AI-powered linter |
77+
| `sdk-refactor.example.ts` | Code refactoring |
78+
| `sdk-test-gen.example.ts` | Unit test generation |
79+
80+
## Running All Tests
81+
82+
```bash
83+
# All E2E tests
84+
bun run test:e2e
85+
86+
# All integration tests
87+
bun run test:integration
88+
89+
# All unit tests
90+
bun run test:unit:e2e
91+
92+
# Everything
93+
bun run test:e2e && bun run test:integration && bun run test:unit:e2e
94+
```
95+
96+
## Prerequisites
97+
98+
- **API Key**: Set `CODEBUFF_API_KEY` environment variable for E2E and integration tests
99+
- Tests skip gracefully if API key is not set
100+
101+
## Writing Tests
102+
103+
### E2E Test Pattern
104+
```typescript
105+
import { describe, test, expect, beforeAll } from 'bun:test'
106+
import { CodebuffClient } from '../../src/client'
107+
import { EventCollector, getApiKey, skipIfNoApiKey, isAuthError, DEFAULT_AGENT, DEFAULT_TIMEOUT } from '../utils'
108+
109+
describe('E2E: My Test', () => {
110+
let client: CodebuffClient
111+
112+
beforeAll(() => {
113+
if (skipIfNoApiKey()) return
114+
client = new CodebuffClient({ apiKey: getApiKey() })
115+
})
116+
117+
test('does something', async () => {
118+
if (skipIfNoApiKey()) return
119+
const collector = new EventCollector()
120+
121+
const result = await client.run({
122+
agent: DEFAULT_AGENT,
123+
prompt: 'Test prompt',
124+
handleEvent: collector.handleEvent,
125+
})
126+
127+
if (isAuthError(result.output)) return
128+
129+
expect(result.output.type).not.toBe('error')
130+
}, DEFAULT_TIMEOUT)
131+
})
132+
```
133+
134+
### Unit Test Pattern
135+
```typescript
136+
import { describe, test, expect, beforeEach } from 'bun:test'
137+
import { EventCollector } from '../event-collector'
138+
139+
describe('Unit: EventCollector', () => {
140+
let collector: EventCollector
141+
142+
beforeEach(() => {
143+
collector = new EventCollector()
144+
})
145+
146+
test('collects events', () => {
147+
collector.handleEvent({ type: 'start', messageHistoryLength: 0 })
148+
expect(collector.events).toHaveLength(1)
149+
})
150+
})
151+
```

‎sdk/e2e/custom-agents/api-integration-agent.e2e.test.ts‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/**
2+
* E2E Test: API Integration Agent
3+
*
4+
* Agent that fetches from external APIs demonstrating API integration patterns.
5+
*/
6+
7+
import { describe, test, expect, beforeAll } from 'bun:test'
8+
import { z } from 'zod/v4'
9+
10+
import { CodebuffClient, getCustomToolDefinition } from '../../src'
11+
import { EventCollector, getApiKey, skipIfNoApiKey, isAuthError, DEFAULT_TIMEOUT } from '../utils'
12+
13+
import type { AgentDefinition } from '../../src'
14+
15+
describe('Custom Agents: API Integration Agent', () => {
16+
let client: CodebuffClient
17+
18+
const apiAgent: AgentDefinition = {
19+
id: 'api-agent',
20+
model: 'anthropic/claude-sonnet-4-20250514',
21+
displayName: 'API Integration Agent',
22+
toolNames: ['fetch_api'],
23+
instructionsPrompt: `You are an API integration assistant.
24+
Use the fetch_api tool to make HTTP requests.
25+
Summarize the response data clearly.`,
26+
}
27+
28+
const fetchTool = getCustomToolDefinition({
29+
toolName: 'fetch_api',
30+
description: 'Fetch data from an API endpoint',
31+
inputSchema: z.object({
32+
url: z.string().describe('URL to fetch'),
33+
method: z.enum(['GET', 'POST']).default('GET'),
34+
}),
35+
exampleInputs: [{ url: 'https://api.example.com/data', method: 'GET' }],
36+
execute: async ({ url, method }) => {
37+
// Mock responses for common test URLs
38+
if (url.includes('jsonplaceholder') || url.includes('example')) {
39+
return [
40+
{
41+
type: 'json' as const,
42+
value: {
43+
status: 200,
44+
data: {
45+
id: 1,
46+
title: 'Mock Response',
47+
body: 'This is mock API data for testing',
48+
},
49+
},
50+
},
51+
]
52+
}
53+
54+
// Try real fetch for other URLs (with timeout)
55+
try {
56+
const controller = new AbortController()
57+
const timeout = setTimeout(() => controller.abort(), 5000)
58+
59+
const response = await fetch(url, {
60+
method,
61+
signal: controller.signal,
62+
})
63+
clearTimeout(timeout)
64+
65+
const text = await response.text()
66+
return [
67+
{
68+
type: 'json' as const,
69+
value: {
70+
status: response.status,
71+
data: text.slice(0, 1000),
72+
},
73+
},
74+
]
75+
} catch {
76+
return [
77+
{
78+
type: 'json' as const,
79+
value: {
80+
error: 'Failed to fetch',
81+
status: 0,
82+
},
83+
},
84+
]
85+
}
86+
},
87+
})
88+
89+
beforeAll(() => {
90+
if (skipIfNoApiKey()) return
91+
client = new CodebuffClient({ apiKey: getApiKey() })
92+
})
93+
94+
test(
95+
'fetches mock API data and summarizes response',
96+
async () => {
97+
if (skipIfNoApiKey()) return
98+
99+
const collector = new EventCollector()
100+
101+
const result = await client.run({
102+
agent: 'api-agent',
103+
prompt: 'Fetch data from https://jsonplaceholder.typicode.com/posts/1',
104+
agentDefinitions: [apiAgent],
105+
customToolDefinitions: [fetchTool],
106+
handleEvent: collector.handleEvent,
107+
})
108+
109+
if (isAuthError(result.output)) return
110+
111+
expect(result.output.type).not.toBe('error')
112+
113+
const toolCalls = collector.getEventsByType('tool_call')
114+
expect(toolCalls.some((c) => c.toolName === 'fetch_api')).toBe(true)
115+
116+
expect(collector.hasEventType('finish')).toBe(true)
117+
},
118+
DEFAULT_TIMEOUT,
119+
)
120+
121+
test(
122+
'handles API errors gracefully',
123+
async () => {
124+
if (skipIfNoApiKey()) return
125+
126+
const collector = new EventCollector()
127+
128+
const result = await client.run({
129+
agent: 'api-agent',
130+
prompt: 'Try to fetch data from https://nonexistent-domain-12345.invalid/api',
131+
agentDefinitions: [apiAgent],
132+
customToolDefinitions: [fetchTool],
133+
handleEvent: collector.handleEvent,
134+
})
135+
136+
if (isAuthError(result.output)) return
137+
138+
// Should complete without crashing
139+
expect(collector.hasEventType('finish')).toBe(true)
140+
},
141+
DEFAULT_TIMEOUT,
142+
)
143+
})

0 commit comments

Comments
 (0)