fix: validate scope checks ran and report overall test status

- Add FAILURE checks to scope scenarios when authorization flow doesn't
  complete (ScopeFromWwwAuthenticateScenario, ScopeFromScopesSupportedScenario,
  ScopeOmittedWhenUndefinedScenario)
- Report client timeout and exit errors in test results
- Add OVERALL: PASSED/FAILED summary at end of test output
- Exit with code 1 on any failure (check failures, client timeout, or exit error)

Previously, if the OAuth authorization flow didn't complete, the scope
checks were never triggered and the test would incorrectly report success.
Now these scenarios emit a FAILURE if the expected scope check wasn't recorded.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: pcarleton

src/index.ts

@@ -173,8 +173,12 @@ program
         timeout
       );
 
-      const { failed } = printClientResults(result.checks, verbose);
-      process.exit(failed > 0 ? 1 : 0);
+      const { overallFailure } = printClientResults(
+        result.checks,
+        verbose,
+        result.clientOutput
+      );
+      process.exit(overallFailure ? 1 : 0);
     } catch (error) {
       if (error instanceof ZodError) {
         console.error('Validation error:');

src/runner/client.ts

@@ -150,15 +150,29 @@ export async function runConformanceTest(
 
 export function printClientResults(
   checks: ConformanceCheck[],
-  verbose: boolean = false
-): { passed: number; failed: number; denominator: number; warnings: number } {
+  verbose: boolean = false,
+  clientOutput?: ClientExecutionResult
+): {
+  passed: number;
+  failed: number;
+  denominator: number;
+  warnings: number;
+  overallFailure: boolean;
+} {
   const denominator = checks.filter(
     (c) => c.status === 'SUCCESS' || c.status === 'FAILURE'
   ).length;
   const passed = checks.filter((c) => c.status === 'SUCCESS').length;
   const failed = checks.filter((c) => c.status === 'FAILURE').length;
   const warnings = checks.filter((c) => c.status === 'WARNING').length;
 
+  // Determine if there's an overall failure (client timeout or exit failure)
+  const clientTimedOut = clientOutput?.timedOut ?? false;
+  const clientExitedWithError = clientOutput
+    ? clientOutput.exitCode !== 0
+    : false;
+  const overallFailure = failed > 0 || clientTimedOut || clientExitedWithError;
+
   if (verbose) {
     // Verbose mode: JSON goes to stdout for piping to jq/jless
     console.log(JSON.stringify(checks, null, 2));
@@ -173,6 +187,16 @@ export function printClientResults(
     `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings`
   );
 
+  if (clientTimedOut) {
+    console.error(`\n⚠️  CLIENT TIMED OUT - Test incomplete`);
+  }
+
+  if (clientExitedWithError && !clientTimedOut) {
+    console.error(
+      `\n⚠️  CLIENT EXITED WITH ERROR (code ${clientOutput?.exitCode}) - Test may be incomplete`
+    );
+  }
+
   if (failed > 0) {
     console.error('\nFailed Checks:');
     checks
@@ -185,7 +209,13 @@ export function printClientResults(
       });
   }
 
-  return { passed, failed, denominator, warnings };
+  if (overallFailure) {
+    console.error('\n❌ OVERALL: FAILED');
+  } else {
+    console.error('\n✅ OVERALL: PASSED');
+  }
+
+  return { passed, failed, denominator, warnings, overallFailure };
 }
 
 export async function runInteractiveMode(

src/scenarios/client/auth/scope-handling.ts

@@ -73,6 +73,21 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario {
   }
 
   getChecks(): ConformanceCheck[] {
+    // Emit failure check if expected scope check didn't run
+    const hasScopeCheck = this.checks.some(
+      (c) => c.id === 'scope-from-www-authenticate'
+    );
+    if (!hasScopeCheck) {
+      this.checks.push({
+        id: 'scope-from-www-authenticate',
+        name: 'Client scope selection from WWW-Authenticate header',
+        description:
+          'Client did not complete authorization flow - scope check could not be performed',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY]
+      });
+    }
     return this.checks;
   }
 }
@@ -153,6 +168,21 @@ export class ScopeFromScopesSupportedScenario implements Scenario {
   }
 
   getChecks(): ConformanceCheck[] {
+    // Emit failure check if expected scope check didn't run
+    const hasScopeCheck = this.checks.some(
+      (c) => c.id === 'scope-from-scopes-supported'
+    );
+    if (!hasScopeCheck) {
+      this.checks.push({
+        id: 'scope-from-scopes-supported',
+        name: 'Client scope selection from scopes_supported',
+        description:
+          'Client did not complete authorization flow - scope check could not be performed',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY]
+      });
+    }
     return this.checks;
   }
 }
@@ -221,6 +251,21 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario {
   }
 
   getChecks(): ConformanceCheck[] {
+    // Emit failure check if expected scope check didn't run
+    const hasScopeCheck = this.checks.some(
+      (c) => c.id === 'scope-omitted-when-undefined'
+    );
+    if (!hasScopeCheck) {
+      this.checks.push({
+        id: 'scope-omitted-when-undefined',
+        name: 'Client scope omission when scopes_supported undefined',
+        description:
+          'Client did not complete authorization flow - scope check could not be performed',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY]
+      });
+    }
     return this.checks;
   }
 }