Merge remote-tracking branch 'origin/main' into pcarleton/move-metadata-around

Author: pcarleton

.github/workflows/pr-publish.yml

@@ -24,4 +24,4 @@ jobs:
       - name: Build
         run: npm run build
       - name: Publish
-        run: npx pkg-pr-new publish
+        run: npx pkg-pr-new publish --bin

package.json

@@ -36,6 +36,7 @@
     "cors": "^2.8.5",
     "eslint": "^9.8.0",
     "eslint-config-prettier": "^10.1.8",
+    "lefthook": "^2.0.2",
     "prettier": "3.6.2",
     "tsdown": "^0.15.12",
     "tsx": "^4.7.0",
@@ -47,7 +48,6 @@
     "@modelcontextprotocol/sdk": "^1.22.0",
     "commander": "^14.0.2",
     "express": "^5.1.0",
-    "lefthook": "^2.0.2",
     "zod": "^3.25.76"
   }
 }

src/index.ts

@@ -13,7 +13,8 @@ import {
 import {
   listScenarios,
   listClientScenarios,
-  listActiveClientScenarios
+  listActiveClientScenarios,
+  listAuthScenarios
 } from './scenarios';
 import { ConformanceCheck } from './types';
 import { ClientOptionsSchema, ServerOptionsSchema } from './schemas';
@@ -33,34 +34,140 @@ program
     'Run conformance tests against a client implementation or start interactive mode'
   )
   .option('--command <command>', 'Command to run the client')
-  .requiredOption('--scenario <scenario>', 'Scenario to test')
+  .option('--scenario <scenario>', 'Scenario to test')
+  .option('--suite <suite>', 'Run a suite of tests in parallel (e.g., "auth")')
   .option('--timeout <ms>', 'Timeout in milliseconds', '30000')
   .option('--verbose', 'Show verbose output')
   .action(async (options) => {
     try {
-      // Validate options with Zod
+      const timeout = parseInt(options.timeout, 10);
+      const verbose = options.verbose ?? false;
+
+      // Handle suite mode
+      if (options.suite) {
+        if (!options.command) {
+          console.error('--command is required when using --suite');
+          process.exit(1);
+        }
+
+        const suites: Record<string, () => string[]> = {
+          auth: listAuthScenarios
+        };
+
+        const suiteName = options.suite.toLowerCase();
+        if (!suites[suiteName]) {
+          console.error(`Unknown suite: ${suiteName}`);
+          console.error(`Available suites: ${Object.keys(suites).join(', ')}`);
+          process.exit(1);
+        }
+
+        const scenarios = suites[suiteName]();
+        console.log(
+          `Running ${suiteName} suite (${scenarios.length} scenarios) in parallel...\n`
+        );
+
+        const results = await Promise.all(
+          scenarios.map(async (scenarioName) => {
+            try {
+              const result = await runConformanceTest(
+                options.command,
+                scenarioName,
+                timeout
+              );
+              return {
+                scenario: scenarioName,
+                checks: result.checks,
+                error: null
+              };
+            } catch (error) {
+              return {
+                scenario: scenarioName,
+                checks: [
+                  {
+                    id: scenarioName,
+                    name: scenarioName,
+                    description: 'Failed to run scenario',
+                    status: 'FAILURE' as const,
+                    timestamp: new Date().toISOString(),
+                    errorMessage:
+                      error instanceof Error ? error.message : String(error)
+                  }
+                ],
+                error
+              };
+            }
+          })
+        );
+
+        console.log('\n=== SUITE SUMMARY ===\n');
+
+        let totalPassed = 0;
+        let totalFailed = 0;
+        let totalWarnings = 0;
+
+        for (const result of results) {
+          const passed = result.checks.filter(
+            (c) => c.status === 'SUCCESS'
+          ).length;
+          const failed = result.checks.filter(
+            (c) => c.status === 'FAILURE'
+          ).length;
+          const warnings = result.checks.filter(
+            (c) => c.status === 'WARNING'
+          ).length;
+
+          totalPassed += passed;
+          totalFailed += failed;
+          totalWarnings += warnings;
+
+          const status = failed === 0 ? '✓' : '✗';
+          console.log(
+            `${status} ${result.scenario}: ${passed} passed, ${failed} failed`
+          );
+
+          if (verbose && failed > 0) {
+            result.checks
+              .filter((c) => c.status === 'FAILURE')
+              .forEach((c) => {
+                console.log(
+                  `    - ${c.name}: ${c.errorMessage || c.description}`
+                );
+              });
+          }
+        }
+
+        console.log(
+          `\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings`
+        );
+        process.exit(totalFailed > 0 ? 1 : 0);
+      }
+
+      // Require either --scenario or --suite
+      if (!options.scenario) {
+        console.error('Either --scenario or --suite is required');
+        console.error('\nAvailable client scenarios:');
+        listScenarios().forEach((s) => console.error(`  - ${s}`));
+        console.error('\nAvailable suites: auth');
+        process.exit(1);
+      }
+
+      // Validate options with Zod for single scenario mode
       const validated = ClientOptionsSchema.parse(options);
 
       // If no command provided, run in interactive mode
       if (!validated.command) {
-        await runInteractiveMode(
-          validated.scenario,
-          validated.verbose ?? false
-        );
+        await runInteractiveMode(validated.scenario, verbose);
         process.exit(0);
       }
 
       // Otherwise run conformance test
       const result = await runConformanceTest(
         validated.command,
         validated.scenario,
-        validated.timeout ?? 30000
+        timeout
       );
 
-      const { failed } = printClientResults(
-        result.checks,
-        validated.verbose ?? false
-      );
+      const { failed } = printClientResults(result.checks, verbose);
       process.exit(failed > 0 ? 1 : 0);
     } catch (error) {
       if (error instanceof ZodError) {

src/scenarios/index.ts

@@ -147,3 +147,7 @@ export function listClientScenarios(): string[] {
 export function listActiveClientScenarios(): string[] {
   return activeClientScenariosList.map((scenario) => scenario.name);
 }
+
+export function listAuthScenarios(): string[] {
+  return authScenariosList.map((scenario) => scenario.name);
+}

src/scenarios/server/tools.ts

@@ -388,6 +388,9 @@ Implement tool \`test_tool_with_logging\` with no arguments.
       const connection = await connectToServer(serverUrl);
       const notifications = new NotificationCollector(connection.client);
 
+      // Set logging level to debug
+      await connection.client.setLoggingLevel('debug');
+
       await connection.client.callTool({
         name: 'test_tool_with_logging',
         arguments: {}