From b0b4e24cf6084078c128e05e71788368a14b83d1 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Mon, 15 Dec 2025 16:59:26 +0000 Subject: [PATCH 1/3] fix: validate scope checks ran and report overall test status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add FAILURE checks to scope scenarios when authorization flow doesn't complete (ScopeFromWwwAuthenticateScenario, ScopeFromScopesSupportedScenario, ScopeOmittedWhenUndefinedScenario) - Report client timeout and exit errors in test results - Add OVERALL: PASSED/FAILED summary at end of test output - Exit with code 1 on any failure (check failures, client timeout, or exit error) Previously, if the OAuth authorization flow didn't complete, the scope checks were never triggered and the test would incorrectly report success. Now these scenarios emit a FAILURE if the expected scope check wasn't recorded. šŸ¤– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/index.ts | 8 +++- src/runner/client.ts | 36 +++++++++++++++-- src/scenarios/client/auth/scope-handling.ts | 45 +++++++++++++++++++++ 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index 6128a16..b54c2f3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -173,8 +173,12 @@ program timeout ); - const { failed } = printClientResults(result.checks, verbose); - process.exit(failed > 0 ? 1 : 0); + const { overallFailure } = printClientResults( + result.checks, + verbose, + result.clientOutput + ); + process.exit(overallFailure ? 1 : 0); } catch (error) { if (error instanceof ZodError) { console.error('Validation error:'); diff --git a/src/runner/client.ts b/src/runner/client.ts index cafb7c3..d444630 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -150,8 +150,15 @@ export async function runConformanceTest( export function printClientResults( checks: ConformanceCheck[], - verbose: boolean = false -): { passed: number; failed: number; denominator: number; warnings: number } { + verbose: boolean = false, + clientOutput?: ClientExecutionResult +): { + passed: number; + failed: number; + denominator: number; + warnings: number; + overallFailure: boolean; +} { const denominator = checks.filter( (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' ).length; @@ -159,6 +166,13 @@ export function printClientResults( const failed = checks.filter((c) => c.status === 'FAILURE').length; const warnings = checks.filter((c) => c.status === 'WARNING').length; + // Determine if there's an overall failure (client timeout or exit failure) + const clientTimedOut = clientOutput?.timedOut ?? false; + const clientExitedWithError = clientOutput + ? clientOutput.exitCode !== 0 + : false; + const overallFailure = failed > 0 || clientTimedOut || clientExitedWithError; + if (verbose) { // Verbose mode: JSON goes to stdout for piping to jq/jless console.log(JSON.stringify(checks, null, 2)); @@ -173,6 +187,16 @@ export function printClientResults( `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` ); + if (clientTimedOut) { + console.error(`\nāš ļø CLIENT TIMED OUT - Test incomplete`); + } + + if (clientExitedWithError && !clientTimedOut) { + console.error( + `\nāš ļø CLIENT EXITED WITH ERROR (code ${clientOutput?.exitCode}) - Test may be incomplete` + ); + } + if (failed > 0) { console.error('\nFailed Checks:'); checks @@ -185,7 +209,13 @@ export function printClientResults( }); } - return { passed, failed, denominator, warnings }; + if (overallFailure) { + console.error('\nāŒ OVERALL: FAILED'); + } else { + console.error('\nāœ… OVERALL: PASSED'); + } + + return { passed, failed, denominator, warnings, overallFailure }; } export async function runInteractiveMode( diff --git a/src/scenarios/client/auth/scope-handling.ts b/src/scenarios/client/auth/scope-handling.ts index 137371a..d94760b 100644 --- a/src/scenarios/client/auth/scope-handling.ts +++ b/src/scenarios/client/auth/scope-handling.ts @@ -73,6 +73,21 @@ export class ScopeFromWwwAuthenticateScenario implements Scenario { } getChecks(): ConformanceCheck[] { + // Emit failure check if expected scope check didn't run + const hasScopeCheck = this.checks.some( + (c) => c.id === 'scope-from-www-authenticate' + ); + if (!hasScopeCheck) { + this.checks.push({ + id: 'scope-from-www-authenticate', + name: 'Client scope selection from WWW-Authenticate header', + description: + 'Client did not complete authorization flow - scope check could not be performed', + status: 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY] + }); + } return this.checks; } } @@ -153,6 +168,21 @@ export class ScopeFromScopesSupportedScenario implements Scenario { } getChecks(): ConformanceCheck[] { + // Emit failure check if expected scope check didn't run + const hasScopeCheck = this.checks.some( + (c) => c.id === 'scope-from-scopes-supported' + ); + if (!hasScopeCheck) { + this.checks.push({ + id: 'scope-from-scopes-supported', + name: 'Client scope selection from scopes_supported', + description: + 'Client did not complete authorization flow - scope check could not be performed', + status: 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY] + }); + } return this.checks; } } @@ -221,6 +251,21 @@ export class ScopeOmittedWhenUndefinedScenario implements Scenario { } getChecks(): ConformanceCheck[] { + // Emit failure check if expected scope check didn't run + const hasScopeCheck = this.checks.some( + (c) => c.id === 'scope-omitted-when-undefined' + ); + if (!hasScopeCheck) { + this.checks.push({ + id: 'scope-omitted-when-undefined', + name: 'Client scope omission when scopes_supported undefined', + description: + 'Client did not complete authorization flow - scope check could not be performed', + status: 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [SpecReferences.MCP_SCOPE_SELECTION_STRATEGY] + }); + } return this.checks; } } From 8d9ce83348df2b3f14aa83ce77ecdbb30fba4659 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Mon, 15 Dec 2025 17:25:42 +0000 Subject: [PATCH 2/3] Count warnings as failures in overall test result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Warnings now contribute to the overall failure status, causing the test to exit with code 1 when any warnings are present. Also displays warning checks in the output summary, similar to how failures are displayed. šŸ¤– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/runner/client.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/runner/client.ts b/src/runner/client.ts index d444630..11a6115 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -166,12 +166,13 @@ export function printClientResults( const failed = checks.filter((c) => c.status === 'FAILURE').length; const warnings = checks.filter((c) => c.status === 'WARNING').length; - // Determine if there's an overall failure (client timeout or exit failure) + // Determine if there's an overall failure (failures, warnings, client timeout, or exit failure) const clientTimedOut = clientOutput?.timedOut ?? false; const clientExitedWithError = clientOutput ? clientOutput.exitCode !== 0 : false; - const overallFailure = failed > 0 || clientTimedOut || clientExitedWithError; + const overallFailure = + failed > 0 || warnings > 0 || clientTimedOut || clientExitedWithError; if (verbose) { // Verbose mode: JSON goes to stdout for piping to jq/jless @@ -209,6 +210,18 @@ export function printClientResults( }); } + if (warnings > 0) { + console.error('\nWarning Checks:'); + checks + .filter((c) => c.status === 'WARNING') + .forEach((c) => { + console.error(` - ${c.name}: ${c.description}`); + if (c.errorMessage) { + console.error(` Warning: ${c.errorMessage}`); + } + }); + } + if (overallFailure) { console.error('\nāŒ OVERALL: FAILED'); } else { From 1c42539739e179f67d092863b0e552c26b6d7272 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Mon, 15 Dec 2025 17:31:25 +0000 Subject: [PATCH 3/3] Count warnings as failures in suite runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the suite summary to show āœ— for scenarios with warnings, and exit with code 1 if any warnings are present across the suite. šŸ¤– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index b54c2f3..edcf042 100644 --- a/src/index.ts +++ b/src/index.ts @@ -125,7 +125,7 @@ program totalFailed += failed; totalWarnings += warnings; - const status = failed === 0 ? 'āœ“' : 'āœ—'; + const status = failed === 0 && warnings === 0 ? 'āœ“' : 'āœ—'; const warningStr = warnings > 0 ? `, ${warnings} warnings` : ''; console.log( `${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}` @@ -145,7 +145,7 @@ program console.log( `\nTotal: ${totalPassed} passed, ${totalFailed} failed, ${totalWarnings} warnings` ); - process.exit(totalFailed > 0 ? 1 : 0); + process.exit(totalFailed > 0 || totalWarnings > 0 ? 1 : 0); } // Require either --scenario or --suite