diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 000000000..4c7a477f1 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,24 @@ +name: Conformance Tests + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + client-conformance: + runs-on: ubuntu-latest + continue-on-error: true # Non-blocking initially + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + - uses: pnpm/action-setup@v4 + with: + version: 10.24.0 + - run: pnpm install + - run: pnpm run build:all + - run: pnpm run conformance:client diff --git a/package.json b/package.json index 2633d5ef2..63a832207 100644 --- a/package.json +++ b/package.json @@ -29,9 +29,11 @@ "lint:all": "pnpm -r lint", "lint:fix:all": "pnpm -r lint:fix", "check:all": "pnpm -r typecheck && pnpm -r lint", - "test:all": "pnpm -r test" + "test:all": "pnpm -r test", + "conformance:client": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts' --scenario initialize" }, "devDependencies": { + "@modelcontextprotocol/client": "workspace:^", "@cfworker/json-schema": "catalog:runtimeShared", "@changesets/changelog-github": "^0.5.2", "@changesets/cli": "^2.29.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 92dbf8253..d5148215f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -149,6 +149,9 @@ importers: '@eslint/js': specifier: catalog:devTools version: 9.39.1 + '@modelcontextprotocol/client': + specifier: workspace:^ + version: link:packages/client '@types/content-type': specifier: catalog:devTools version: 1.1.9 diff --git a/src/conformance/everything-client.ts b/src/conformance/everything-client.ts new file mode 100644 index 000000000..f8a8ae581 --- /dev/null +++ b/src/conformance/everything-client.ts @@ -0,0 +1,224 @@ +#!/usr/bin/env node + +/** + * Everything client - a single conformance test client that handles all scenarios. + * + * Usage: everything-client + * + * The scenario name is read from the MCP_CONFORMANCE_SCENARIO environment variable, + * which is set by the conformance test runner. + * + * This client routes to the appropriate behavior based on the scenario name, + * consolidating all the individual test clients into one. + */ + +import { + Client, + StreamableHTTPClientTransport, + ElicitRequestSchema +} from '@modelcontextprotocol/client'; +import { withOAuthRetry } from './helpers/withOAuthRetry.js'; +import { logger } from './helpers/logger.js'; + +// Scenario handler type +type ScenarioHandler = (serverUrl: string) => Promise; + +// Registry of scenario handlers +const scenarioHandlers: Record = {}; + +// Helper to register a scenario handler +function registerScenario(name: string, handler: ScenarioHandler): void { + scenarioHandlers[name] = handler; +} + +// Helper to register multiple scenarios with the same handler +function registerScenarios(names: string[], handler: ScenarioHandler): void { + for (const name of names) { + scenarioHandlers[name] = handler; + } +} + +// ============================================================================ +// Basic scenarios (initialize, tools-call) +// ============================================================================ + +async function runBasicClient(serverUrl: string): Promise { + const client = new Client( + { name: 'test-client', version: '1.0.0' }, + { capabilities: {} } + ); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + await client.listTools(); + logger.debug('Successfully listed tools'); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenarios(['initialize', 'tools-call'], runBasicClient); + +// ============================================================================ +// Auth scenarios - well-behaved client +// ============================================================================ + +async function runAuthClient(serverUrl: string): Promise { + const client = new Client( + { name: 'test-auth-client', version: '1.0.0' }, + { capabilities: {} } + ); + + const oauthFetch = withOAuthRetry( + 'test-auth-client', + new URL(serverUrl) + )(fetch); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + fetch: oauthFetch + }); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + await client.listTools(); + logger.debug('Successfully listed tools'); + + await client.callTool({ name: 'test-tool', arguments: {} }); + logger.debug('Successfully called tool'); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +// Register all auth scenarios that should use the well-behaved auth client +registerScenarios( + [ + 'auth/basic-dcr', + 'auth/basic-metadata-var1', + 'auth/basic-metadata-var2', + 'auth/basic-metadata-var3', + 'auth/2025-03-26-oauth-metadata-backcompat', + 'auth/2025-03-26-oauth-endpoint-fallback', + 'auth/scope-from-www-authenticate', + 'auth/scope-from-scopes-supported', + 'auth/scope-omitted-when-undefined', + 'auth/scope-step-up' + ], + runAuthClient +); + +// ============================================================================ +// Elicitation defaults scenario +// ============================================================================ + +async function runElicitationDefaultsClient(serverUrl: string): Promise { + const client = new Client( + { name: 'elicitation-defaults-test-client', version: '1.0.0' }, + { + capabilities: { + elicitation: { + applyDefaults: true + } + } + } + ); + + // Register elicitation handler that returns empty content + // The SDK should fill in defaults for all omitted fields + client.setRequestHandler(ElicitRequestSchema, async (request) => { + logger.debug( + 'Received elicitation request:', + JSON.stringify(request.params, null, 2) + ); + logger.debug('Accepting with empty content - SDK should apply defaults'); + + // Return empty content - SDK should merge in defaults + return { + action: 'accept' as const, + content: {} + }; + }); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + logger.debug('Successfully connected to MCP server'); + + // List available tools + const tools = await client.listTools(); + logger.debug( + 'Available tools:', + tools.tools.map((t) => t.name) + ); + + // Call the test tool which will trigger elicitation + const testTool = tools.tools.find( + (t) => t.name === 'test_client_elicitation_defaults' + ); + if (!testTool) { + throw new Error('Test tool not found: test_client_elicitation_defaults'); + } + + logger.debug('Calling test_client_elicitation_defaults tool...'); + const result = await client.callTool({ + name: 'test_client_elicitation_defaults', + arguments: {} + }); + + logger.debug('Tool result:', JSON.stringify(result, null, 2)); + + await transport.close(); + logger.debug('Connection closed successfully'); +} + +registerScenario('elicitation-defaults', runElicitationDefaultsClient); + +// ============================================================================ +// Main entry point +// ============================================================================ + +async function main(): Promise { + const scenarioName = process.env.MCP_CONFORMANCE_SCENARIO; + const serverUrl = process.argv[2]; + + if (!scenarioName || !serverUrl) { + console.error( + 'Usage: MCP_CONFORMANCE_SCENARIO= everything-client ' + ); + console.error( + '\nThe MCP_CONFORMANCE_SCENARIO env var is set automatically by the conformance runner.' + ); + console.error('\nAvailable scenarios:'); + for (const name of Object.keys(scenarioHandlers).sort()) { + console.error(` - ${name}`); + } + process.exit(1); + } + + const handler = scenarioHandlers[scenarioName]; + if (!handler) { + console.error(`Unknown scenario: ${scenarioName}`); + console.error('\nAvailable scenarios:'); + for (const name of Object.keys(scenarioHandlers).sort()) { + console.error(` - ${name}`); + } + process.exit(1); + } + + try { + await handler(serverUrl); + process.exit(0); + } catch (error) { + console.error('Error:', error); + process.exit(1); + } +} + +main().catch((error) => { + console.error('Unhandled error:', error); + process.exit(1); +}); diff --git a/src/conformance/helpers/ConformanceOAuthProvider.ts b/src/conformance/helpers/ConformanceOAuthProvider.ts new file mode 100644 index 000000000..b29b95796 --- /dev/null +++ b/src/conformance/helpers/ConformanceOAuthProvider.ts @@ -0,0 +1,95 @@ +import type { + OAuthClientProvider, + OAuthClientInformation, + OAuthClientInformationFull, + OAuthClientMetadata, + OAuthTokens +} from '@modelcontextprotocol/client'; + +export class ConformanceOAuthProvider implements OAuthClientProvider { + private _clientInformation?: OAuthClientInformationFull; + private _tokens?: OAuthTokens; + private _codeVerifier?: string; + private _authCode?: string; + private _authCodePromise?: Promise; + + constructor( + private readonly _redirectUrl: string | URL, + private readonly _clientMetadata: OAuthClientMetadata, + private readonly _clientMetadataUrl?: string | URL + ) {} + + get redirectUrl(): string | URL { + return this._redirectUrl; + } + + get clientMetadata(): OAuthClientMetadata { + return this._clientMetadata; + } + + get clientMetadataUrl(): string | undefined { + return this._clientMetadataUrl?.toString(); + } + + clientInformation(): OAuthClientInformation | undefined { + return this._clientInformation; + } + + saveClientInformation(clientInformation: OAuthClientInformationFull): void { + this._clientInformation = clientInformation; + } + + tokens(): OAuthTokens | undefined { + return this._tokens; + } + + saveTokens(tokens: OAuthTokens): void { + this._tokens = tokens; + } + + async redirectToAuthorization(authorizationUrl: URL): Promise { + try { + const response = await fetch(authorizationUrl.toString(), { + redirect: 'manual' // Don't follow redirects automatically + }); + + // Get the Location header which contains the redirect with auth code + const location = response.headers.get('location'); + if (location) { + const redirectUrl = new URL(location); + const code = redirectUrl.searchParams.get('code'); + if (code) { + this._authCode = code; + return; + } else { + throw new Error('No auth code in redirect URL'); + } + } else { + throw new Error( + `No redirect location received, from '${authorizationUrl.toString()}'` + ); + } + } catch (error) { + console.error('Failed to fetch authorization URL:', error); + throw error; + } + } + + async getAuthCode(): Promise { + if (this._authCode) { + return this._authCode; + } + throw new Error('No authorization code'); + } + + saveCodeVerifier(codeVerifier: string): void { + this._codeVerifier = codeVerifier; + } + + codeVerifier(): string { + if (!this._codeVerifier) { + throw new Error('No code verifier saved'); + } + return this._codeVerifier; + } +} diff --git a/src/conformance/helpers/logger.ts b/src/conformance/helpers/logger.ts new file mode 100644 index 000000000..3b1ff5c39 --- /dev/null +++ b/src/conformance/helpers/logger.ts @@ -0,0 +1,27 @@ +/** + * Simple logger with configurable log levels. + * Set to 'error' in tests to suppress debug output. + */ + +export type LogLevel = 'debug' | 'error'; + +let currentLogLevel: LogLevel = 'debug'; + +export function setLogLevel(level: LogLevel): void { + currentLogLevel = level; +} + +export function getLogLevel(): LogLevel { + return currentLogLevel; +} + +export const logger = { + debug: (...args: unknown[]): void => { + if (currentLogLevel === 'debug') { + console.log(...args); + } + }, + error: (...args: unknown[]): void => { + console.error(...args); + } +}; diff --git a/src/conformance/helpers/withOAuthRetry.ts b/src/conformance/helpers/withOAuthRetry.ts new file mode 100644 index 000000000..bba21ef73 --- /dev/null +++ b/src/conformance/helpers/withOAuthRetry.ts @@ -0,0 +1,111 @@ +import { + auth, + extractWWWAuthenticateParams, + UnauthorizedError +} from '@modelcontextprotocol/client'; +import type { FetchLike, Middleware } from '@modelcontextprotocol/client'; +import { ConformanceOAuthProvider } from './ConformanceOAuthProvider.js'; + +export const handle401 = async ( + response: Response, + provider: ConformanceOAuthProvider, + next: FetchLike, + serverUrl: string | URL +): Promise => { + const { resourceMetadataUrl, scope } = extractWWWAuthenticateParams(response); + let result = await auth(provider, { + serverUrl, + resourceMetadataUrl, + scope, + fetchFn: next + }); + + if (result === 'REDIRECT') { + // Ordinarily, we'd wait for the callback to be handled here, + // but in our conformance provider, we get the authorization code + // during the redirect handling, so we can go straight to + // retrying the auth step. + // await provider.waitForCallback(); + + const authorizationCode = await provider.getAuthCode(); + + // TODO: this retry logic should be incorporated into the typescript SDK + result = await auth(provider, { + serverUrl, + resourceMetadataUrl, + scope, + authorizationCode, + fetchFn: next + }); + if (result !== 'AUTHORIZED') { + throw new UnauthorizedError( + `Authentication failed with result: ${result}` + ); + } + } +}; +/** + * Creates a fetch wrapper that handles OAuth authentication with retry logic. + * + * Unlike the SDK's withOAuth, this version: + * - Automatically handles authorization redirects by retrying with fresh tokens + * - Does not throw UnauthorizedError on redirect, but instead retries + * - Calls next() instead of throwing for redirect-based auth + * + * @param provider - OAuth client provider for authentication + * @param baseUrl - Base URL for OAuth server discovery (defaults to request URL domain) + * @returns A fetch middleware function + */ +export const withOAuthRetry = ( + clientName: string, + baseUrl?: string | URL, + handle401Fn: typeof handle401 = handle401, + clientMetadataUrl?: string +): Middleware => { + const provider = new ConformanceOAuthProvider( + 'http://localhost:3000/callback', + { + client_name: clientName, + redirect_uris: ['http://localhost:3000/callback'] + }, + clientMetadataUrl + ); + return (next: FetchLike) => { + return async ( + input: string | URL, + init?: RequestInit + ): Promise => { + const makeRequest = async (): Promise => { + const headers = new Headers(init?.headers); + + // Add authorization header if tokens are available + const tokens = await provider.tokens(); + if (tokens) { + headers.set('Authorization', `Bearer ${tokens.access_token}`); + } + + return await next(input, { ...init, headers }); + }; + + let response = await makeRequest(); + + // Handle 401 responses by attempting re-authentication + if (response.status === 401 || response.status === 403) { + const serverUrl = + baseUrl || + (typeof input === 'string' ? new URL(input).origin : input.origin); + await handle401Fn(response, provider, next, serverUrl); + + response = await makeRequest(); + } + + // If we still have a 401 after re-auth attempt, throw an error + if (response.status === 401 || response.status === 403) { + const url = typeof input === 'string' ? input : input.toString(); + throw new UnauthorizedError(`Authentication failed for ${url}`); + } + + return response; + }; + }; +};