diff --git a/README.md b/README.md index 741b0b7..d3e1890 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,11 @@ A framework for testing MCP (Model Context Protocol) client and server implement ### Testing Clients ```bash -npx @modelcontextprotocol/conformance client --command "tsx examples/clients/typescript/test1.ts" --scenario initialize +# Using the everything-client (recommended) +npx @modelcontextprotocol/conformance client --command "tsx examples/clients/typescript/everything-client.ts" --scenario initialize + +# Run an entire suite of tests +npx @modelcontextprotocol/conformance client --command "tsx examples/clients/typescript/everything-client.ts" --suite auth ``` ### Testing Servers @@ -59,10 +63,11 @@ npx @modelcontextprotocol/conformance client --command "" --scen - `--command` - The command to run your MCP client (can include flags) - `--scenario` - The test scenario to run (e.g., "initialize") +- `--suite` - Run a suite of tests in parallel (e.g., "auth") - `--timeout` - Timeout in milliseconds (default: 30000) - `--verbose` - Show verbose output -The framework appends the server URL as the final argument to your command. +The framework appends ` ` as arguments to your command. Your client should accept these two positional arguments. ### Server Testing @@ -89,8 +94,9 @@ npx @modelcontextprotocol/conformance server --url [--scenario ] ## Example Clients -- `examples/clients/typescript/test1.ts` - Valid MCP client (passes all checks) -- `examples/clients/typescript/test-broken.ts` - Invalid client missing required fields (fails checks) +- `examples/clients/typescript/everything-client.ts` - Single client that handles all scenarios based on scenario name (recommended) +- `examples/clients/typescript/test1.ts` - Simple MCP client (for reference) +- `examples/clients/typescript/auth-test.ts` - Well-behaved OAuth client (for reference) ## Available Scenarios diff --git a/examples/clients/typescript/everything-client.ts b/examples/clients/typescript/everything-client.ts new file mode 100644 index 0000000..1ecb790 --- /dev/null +++ b/examples/clients/typescript/everything-client.ts @@ -0,0 +1,214 @@ +#!/usr/bin/env node + +/** + * Everything client - a single conformance test client that handles all scenarios. + * + * Usage: everything-client + * + * This client routes to the appropriate behavior based on the scenario name, + * consolidating all the individual test clients into one. + */ + +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { ElicitRequestSchema } from '@modelcontextprotocol/sdk/types.js'; +import { withOAuthRetry } from './helpers/withOAuthRetry.js'; +import { logger } from './helpers/logger.js'; + +// Scenario handler type +type ScenarioHandler = (serverUrl: string) => Promise; + +// Registry of scenario handlers +const scenarioHandlers: Record = {}; + +// Helper to register a scenario handler +function registerScenario(name: string, handler: ScenarioHandler): void { + scenarioHandlers[name] = handler; +} + +// Helper to register multiple scenarios with the same handler +function registerScenarios(names: string[], handler: ScenarioHandler): void { + for (const name of names) { + scenarioHandlers[name] = handler; + } +} + +// ============================================================================ +// Basic scenarios (initialize, tools-call) +// ============================================================================ + +async function runBasicClient(serverUrl: string): Promise { + const client = new Client( + { name: 'test-client', version: '1.0.0' }, + { capabilities: {} } + ); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + await client.listTools(); + logger.debug('Successfully listed tools'); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenarios(['initialize', 'tools-call'], runBasicClient); + +// ============================================================================ +// Auth scenarios - well-behaved client +// ============================================================================ + +async function runAuthClient(serverUrl: string): Promise { + const client = new Client( + { name: 'test-auth-client', version: '1.0.0' }, + { capabilities: {} } + ); + + const oauthFetch = withOAuthRetry( + 'test-auth-client', + new URL(serverUrl) + )(fetch); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + fetch: oauthFetch + }); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + await client.listTools(); + logger.debug('Successfully listed tools'); + + await client.callTool({ name: 'test-tool', arguments: {} }); + logger.debug('Successfully called tool'); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +// Register all auth scenarios that should use the well-behaved auth client +registerScenarios( + [ + 'auth/basic-dcr', + 'auth/basic-metadata-var1', + 'auth/basic-metadata-var2', + 'auth/basic-metadata-var3', + 'auth/2025-03-26-oauth-metadata-backcompat', + 'auth/2025-03-26-oauth-endpoint-fallback', + 'auth/scope-from-www-authenticate', + 'auth/scope-from-scopes-supported', + 'auth/scope-omitted-when-undefined', + 'auth/scope-step-up' + ], + runAuthClient +); + +// ============================================================================ +// Elicitation defaults scenario +// ============================================================================ + +async function runElicitationDefaultsClient(serverUrl: string): Promise { + const client = new Client( + { name: 'elicitation-defaults-test-client', version: '1.0.0' }, + { + capabilities: { + elicitation: { + applyDefaults: true + } + } + } + ); + + // Register elicitation handler that returns empty content + // The SDK should fill in defaults for all omitted fields + client.setRequestHandler(ElicitRequestSchema, async (request) => { + logger.debug( + 'Received elicitation request:', + JSON.stringify(request.params, null, 2) + ); + logger.debug('Accepting with empty content - SDK should apply defaults'); + + // Return empty content - SDK should merge in defaults + return { + action: 'accept' as const, + content: {} + }; + }); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + // List available tools + const tools = await client.listTools(); + logger.debug( + 'Available tools:', + tools.tools.map((t) => t.name) + ); + + // Call the test tool which will trigger elicitation + const testTool = tools.tools.find( + (t) => t.name === 'test_client_elicitation_defaults' + ); + if (!testTool) { + throw new Error('Test tool not found: test_client_elicitation_defaults'); + } + + logger.debug('Calling test_client_elicitation_defaults tool...'); + const result = await client.callTool({ + name: 'test_client_elicitation_defaults', + arguments: {} + }); + + logger.debug('Tool result:', JSON.stringify(result, null, 2)); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenario('elicitation-defaults', runElicitationDefaultsClient); + +// ============================================================================ +// Main entry point +// ============================================================================ + +async function main(): Promise { + const scenarioName = process.argv[2]; + const serverUrl = process.argv[3]; + + if (!scenarioName || !serverUrl) { + console.error('Usage: everything-client '); + console.error('\nAvailable scenarios:'); + for (const name of Object.keys(scenarioHandlers).sort()) { + console.error(` - ${name}`); + } + process.exit(1); + } + + const handler = scenarioHandlers[scenarioName]; + if (!handler) { + console.error(`Unknown scenario: ${scenarioName}`); + console.error('\nAvailable scenarios:'); + for (const name of Object.keys(scenarioHandlers).sort()) { + console.error(` - ${name}`); + } + process.exit(1); + } + + try { + await handler(serverUrl); + process.exit(0); + } catch (error) { + console.error('Error:', error); + process.exit(1); + } +} + +main().catch((error) => { + console.error('Unhandled error:', error); + process.exit(1); +}); diff --git a/src/runner/client.ts b/src/runner/client.ts index cafb7c3..615abf5 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -14,13 +14,14 @@ export interface ClientExecutionResult { async function executeClient( command: string, + scenarioName: string, serverUrl: string, timeout: number = 30000, context?: Record ): Promise { const commandParts = command.split(' '); const executable = commandParts[0]; - const args = [...commandParts.slice(1), serverUrl]; + const args = [...commandParts.slice(1), scenarioName, serverUrl]; let stdout = ''; let stderr = ''; @@ -97,7 +98,9 @@ export async function runConformanceTest( console.error(`Starting scenario: ${scenarioName}`); const urls = await scenario.start(); - console.error(`Executing client: ${clientCommand} ${urls.serverUrl}`); + console.error( + `Executing client: ${clientCommand} ${scenarioName} ${urls.serverUrl}` + ); if (urls.context) { console.error(`With context: ${JSON.stringify(urls.context)}`); } @@ -105,6 +108,7 @@ export async function runConformanceTest( try { const clientOutput = await executeClient( clientCommand, + scenarioName, urls.serverUrl, timeout, urls.context