feat: add client conformance testing CI

.github/workflows/conformance.yml

@@ -0,0 +1,24 @@
+name: Conformance Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  client-conformance:
+    runs-on: ubuntu-latest
+    continue-on-error: true  # Non-blocking initially
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: npm
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 10.24.0
+      - run: pnpm install
+      - run: pnpm run build:all
+      - run: pnpm run conformance:client

package.json

@@ -29,9 +29,11 @@
         "lint:all": "pnpm -r lint",
         "lint:fix:all": "pnpm -r lint:fix",
         "check:all": "pnpm -r typecheck && pnpm -r lint",
-        "test:all": "pnpm -r test"
+        "test:all": "pnpm -r test",
+        "conformance:client": "npx @modelcontextprotocol/conformance client --command 'npx tsx src/conformance/everything-client.ts' --scenario initialize"
     },
     "devDependencies": {
+        "@modelcontextprotocol/client": "workspace:^",
         "@cfworker/json-schema": "catalog:runtimeShared",
         "@changesets/changelog-github": "^0.5.2",
         "@changesets/cli": "^2.29.8",

src/conformance/everything-client.ts

@@ -0,0 +1,224 @@
+#!/usr/bin/env node
+
+/**
+ * Everything client - a single conformance test client that handles all scenarios.
+ *
+ * Usage: everything-client <server-url>
+ *
+ * The scenario name is read from the MCP_CONFORMANCE_SCENARIO environment variable,
+ * which is set by the conformance test runner.
+ *
+ * This client routes to the appropriate behavior based on the scenario name,
+ * consolidating all the individual test clients into one.
+ */
+
+import {
+  Client,
+  StreamableHTTPClientTransport,
+  ElicitRequestSchema
+} from '@modelcontextprotocol/client';
+import { withOAuthRetry } from './helpers/withOAuthRetry.js';
+import { logger } from './helpers/logger.js';
+
+// Scenario handler type
+type ScenarioHandler = (serverUrl: string) => Promise<void>;
+
+// Registry of scenario handlers
+const scenarioHandlers: Record<string, ScenarioHandler> = {};
+
+// Helper to register a scenario handler
+function registerScenario(name: string, handler: ScenarioHandler): void {
+  scenarioHandlers[name] = handler;
+}
+
+// Helper to register multiple scenarios with the same handler
+function registerScenarios(names: string[], handler: ScenarioHandler): void {
+  for (const name of names) {
+    scenarioHandlers[name] = handler;
+  }
+}
+
+// ============================================================================
+// Basic scenarios (initialize, tools-call)
+// ============================================================================
+
+async function runBasicClient(serverUrl: string): Promise<void> {
+  const client = new Client(
+    { name: 'test-client', version: '1.0.0' },
+    { capabilities: {} }
+  );
+
+  const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
+
+  await client.connect(transport);
+  logger.debug('Successfully connected to MCP server');
+
+  await client.listTools();
+  logger.debug('Successfully listed tools');
+
+  await transport.close();
+  logger.debug('Connection closed successfully');
+}
+
+registerScenarios(['initialize', 'tools-call'], runBasicClient);
+
+// ============================================================================
+// Auth scenarios - well-behaved client
+// ============================================================================
+
+async function runAuthClient(serverUrl: string): Promise<void> {
+  const client = new Client(
+    { name: 'test-auth-client', version: '1.0.0' },
+    { capabilities: {} }
+  );
+
+  const oauthFetch = withOAuthRetry(
+    'test-auth-client',
+    new URL(serverUrl)
+  )(fetch);
+
+  const transport = new StreamableHTTPClientTransport(new URL(serverUrl), {
+    fetch: oauthFetch
+  });
+
+  await client.connect(transport);
+  logger.debug('Successfully connected to MCP server');
+
+  await client.listTools();
+  logger.debug('Successfully listed tools');
+
+  await client.callTool({ name: 'test-tool', arguments: {} });
+  logger.debug('Successfully called tool');
+
+  await transport.close();
+  logger.debug('Connection closed successfully');
+}
+
+// Register all auth scenarios that should use the well-behaved auth client
+registerScenarios(
+  [
+    'auth/basic-dcr',
+    'auth/basic-metadata-var1',
+    'auth/basic-metadata-var2',
+    'auth/basic-metadata-var3',
+    'auth/2025-03-26-oauth-metadata-backcompat',
+    'auth/2025-03-26-oauth-endpoint-fallback',
+    'auth/scope-from-www-authenticate',
+    'auth/scope-from-scopes-supported',
+    'auth/scope-omitted-when-undefined',
+    'auth/scope-step-up'
+  ],
+  runAuthClient
+);
+
+// ============================================================================
+// Elicitation defaults scenario
+// ============================================================================
+
+async function runElicitationDefaultsClient(serverUrl: string): Promise<void> {
+  const client = new Client(
+    { name: 'elicitation-defaults-test-client', version: '1.0.0' },
+    {
+      capabilities: {
+        elicitation: {
+          applyDefaults: true
+        }
+      }
+    }
+  );
+
+  // Register elicitation handler that returns empty content
+  // The SDK should fill in defaults for all omitted fields
+  client.setRequestHandler(ElicitRequestSchema, async (request) => {
+    logger.debug(
+      'Received elicitation request:',
+      JSON.stringify(request.params, null, 2)
+    );
+    logger.debug('Accepting with empty content - SDK should apply defaults');
+
+    // Return empty content - SDK should merge in defaults
+    return {
+      action: 'accept' as const,
+      content: {}
+    };
+  });
+
+  const transport = new StreamableHTTPClientTransport(new URL(serverUrl));
+
+  await client.connect(transport);
+  logger.debug('Successfully connected to MCP server');
+
+  // List available tools
+  const tools = await client.listTools();
+  logger.debug(
+    'Available tools:',
+    tools.tools.map((t) => t.name)
+  );
+
+  // Call the test tool which will trigger elicitation
+  const testTool = tools.tools.find(
+    (t) => t.name === 'test_client_elicitation_defaults'
+  );
+  if (!testTool) {
+    throw new Error('Test tool not found: test_client_elicitation_defaults');
+  }
+
+  logger.debug('Calling test_client_elicitation_defaults tool...');
+  const result = await client.callTool({
+    name: 'test_client_elicitation_defaults',
+    arguments: {}
+  });
+
+  logger.debug('Tool result:', JSON.stringify(result, null, 2));
+
+  await transport.close();
+  logger.debug('Connection closed successfully');
+}
+
+registerScenario('elicitation-defaults', runElicitationDefaultsClient);
+
+// ============================================================================
+// Main entry point
+// ============================================================================
+
+async function main(): Promise<void> {
+  const scenarioName = process.env.MCP_CONFORMANCE_SCENARIO;
+  const serverUrl = process.argv[2];
+
+  if (!scenarioName || !serverUrl) {
+    console.error(
+      'Usage: MCP_CONFORMANCE_SCENARIO=<scenario> everything-client <server-url>'
+    );
+    console.error(
+      '\nThe MCP_CONFORMANCE_SCENARIO env var is set automatically by the conformance runner.'
+    );
+    console.error('\nAvailable scenarios:');
+    for (const name of Object.keys(scenarioHandlers).sort()) {
+      console.error(`  - ${name}`);
+    }
+    process.exit(1);
+  }
+
+  const handler = scenarioHandlers[scenarioName];
+  if (!handler) {
+    console.error(`Unknown scenario: ${scenarioName}`);
+    console.error('\nAvailable scenarios:');
+    for (const name of Object.keys(scenarioHandlers).sort()) {
+      console.error(`  - ${name}`);
+    }
+    process.exit(1);
+  }
+
+  try {
+    await handler(serverUrl);
+    process.exit(0);
+  } catch (error) {
+    console.error('Error:', error);
+    process.exit(1);
+  }
+}
+
+main().catch((error) => {
+  console.error('Unhandled error:', error);
+  process.exit(1);
+});

src/conformance/helpers/ConformanceOAuthProvider.ts

@@ -0,0 +1,95 @@
+import type {
+  OAuthClientProvider,
+  OAuthClientInformation,
+  OAuthClientInformationFull,
+  OAuthClientMetadata,
+  OAuthTokens
+} from '@modelcontextprotocol/client';
+
+export class ConformanceOAuthProvider implements OAuthClientProvider {
+  private _clientInformation?: OAuthClientInformationFull;
+  private _tokens?: OAuthTokens;
+  private _codeVerifier?: string;
+  private _authCode?: string;
+  private _authCodePromise?: Promise<string>;
+
+  constructor(
+    private readonly _redirectUrl: string | URL,
+    private readonly _clientMetadata: OAuthClientMetadata,
+    private readonly _clientMetadataUrl?: string | URL
+  ) {}
+
+  get redirectUrl(): string | URL {
+    return this._redirectUrl;
+  }
+
+  get clientMetadata(): OAuthClientMetadata {
+    return this._clientMetadata;
+  }
+
+  get clientMetadataUrl(): string | undefined {
+    return this._clientMetadataUrl?.toString();
+  }
+
+  clientInformation(): OAuthClientInformation | undefined {
+    return this._clientInformation;
+  }
+
+  saveClientInformation(clientInformation: OAuthClientInformationFull): void {
+    this._clientInformation = clientInformation;
+  }
+
+  tokens(): OAuthTokens | undefined {
+    return this._tokens;
+  }
+
+  saveTokens(tokens: OAuthTokens): void {
+    this._tokens = tokens;
+  }
+
+  async redirectToAuthorization(authorizationUrl: URL): Promise<void> {
+    try {
+      const response = await fetch(authorizationUrl.toString(), {
+        redirect: 'manual' // Don't follow redirects automatically
+      });
+
+      // Get the Location header which contains the redirect with auth code
+      const location = response.headers.get('location');
+      if (location) {
+        const redirectUrl = new URL(location);
+        const code = redirectUrl.searchParams.get('code');
+        if (code) {
+          this._authCode = code;
+          return;
+        } else {
+          throw new Error('No auth code in redirect URL');
+        }
+      } else {
+        throw new Error(
+          `No redirect location received, from '${authorizationUrl.toString()}'`
+        );
+      }
+    } catch (error) {
+      console.error('Failed to fetch authorization URL:', error);
+      throw error;
+    }
+  }
+
+  async getAuthCode(): Promise<string> {
+    if (this._authCode) {
+      return this._authCode;
+    }
+    throw new Error('No authorization code');
+  }
+
+  saveCodeVerifier(codeVerifier: string): void {
+    this._codeVerifier = codeVerifier;
+  }
+
+  codeVerifier(): string {
+    if (!this._codeVerifier) {
+      throw new Error('No code verifier saved');
+    }
+    return this._codeVerifier;
+  }
+}

src/conformance/helpers/logger.ts

@@ -0,0 +1,27 @@
+/**
+ * Simple logger with configurable log levels.
+ * Set to 'error' in tests to suppress debug output.
+ */
+
+export type LogLevel = 'debug' | 'error';
+
+let currentLogLevel: LogLevel = 'debug';
+
+export function setLogLevel(level: LogLevel): void {
+  currentLogLevel = level;
+}
+
+export function getLogLevel(): LogLevel {
+  return currentLogLevel;
+}
+
+export const logger = {
+  debug: (...args: unknown[]): void => {
+    if (currentLogLevel === 'debug') {
+      console.log(...args);
+    }
+  },
+  error: (...args: unknown[]): void => {
+    console.error(...args);
+  }
+};