From da695eb56c105334b7dddd50f01072dea221a748 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Thu, 30 Oct 2025 13:26:21 +0000 Subject: [PATCH 1/4] fix runner --- src/runner/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/runner/index.ts b/src/runner/index.ts index 582072f..4c90a12 100644 --- a/src/runner/index.ts +++ b/src/runner/index.ts @@ -146,12 +146,14 @@ async function main(): Promise { try { const result = await runConformanceTest(command, scenario); + const denominator = result.checks.filter(c => c.status === 'SUCCESS' || c.status == 'FAILURE').length; const passed = result.checks.filter(c => c.status === 'SUCCESS').length; const failed = result.checks.filter(c => c.status === 'FAILURE').length; + console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`); + console.log(`\nTest Results:`); - console.log(`Passed: ${passed}/${result.checks.length}`); - console.log(`Failed: ${failed}/${result.checks.length}`); + console.log(`Passed: ${passed}/${denominator}, ${failed} failed`); if (failed > 0) { console.log('\nFailed Checks:'); From 1fa165df3df600ffaa592b744d7b1a0265414306 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Thu, 30 Oct 2025 13:26:29 +0000 Subject: [PATCH 2/4] add tools-call to index --- src/scenarios/index.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index f105ae4..ca671d1 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -1,7 +1,11 @@ import { Scenario } from '../types.js'; import { InitializeScenario } from './initialize.js'; +import { ToolsCallScenario } from './tools_call.js'; -export const scenarios = new Map([['initialize', new InitializeScenario()]]); +export const scenarios = new Map([ + ['initialize', new InitializeScenario()], + ['tools-call', new ToolsCallScenario()] +]); export function registerScenario(name: string, scenario: Scenario): void { scenarios.set(name, scenario); From cff9669f8a19fe800f8a7c5e7a834745d28ea8d9 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Thu, 30 Oct 2025 13:23:22 +0000 Subject: [PATCH 3/4] Add tools_call scenario and test --- examples/clients/typescript/test2.ts | 51 ++++++++ src/scenarios/tools_call.ts | 172 +++++++++++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 examples/clients/typescript/test2.ts create mode 100644 src/scenarios/tools_call.ts diff --git a/examples/clients/typescript/test2.ts b/examples/clients/typescript/test2.ts new file mode 100644 index 0000000..117a376 --- /dev/null +++ b/examples/clients/typescript/test2.ts @@ -0,0 +1,51 @@ +#!/usr/bin/env node + +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; + +async function main(): Promise { + const serverUrl = process.argv[2]; + + if (!serverUrl) { + console.error('Usage: test-client '); + process.exit(1); + } + + console.log(`Connecting to MCP server at: ${serverUrl}`); + + try { + const client = new Client( + { + name: 'test-client', + version: '1.0.0' + }, + { + capabilities: {} + } + ); + + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + + await client.connect(transport); + console.log('✅ Successfully connected to MCP server'); + + await client.listTools(); + console.log('✅ Successfully listed tools'); + + await client.callTool({ name: 'add_numbers', arguments: { a: 5, b: 10 } }); + console.log('✅ Successfully called add_numbers tool'); + + await transport.close(); + console.log('✅ Connection closed successfully'); + + process.exit(0); + } catch (error) { + console.error('❌ Failed to connect to MCP server:', error); + process.exit(1); + } +} + +main().catch(error => { + console.error('Unhandled error:', error); + process.exit(1); +}); diff --git a/src/scenarios/tools_call.ts b/src/scenarios/tools_call.ts new file mode 100644 index 0000000..f2b71f6 --- /dev/null +++ b/src/scenarios/tools_call.ts @@ -0,0 +1,172 @@ +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; +import type { Scenario, ConformanceCheck } from './types.js'; +import express from 'express'; +import { ScenarioUrls } from '../types.js'; + +function createServer(checks: ConformanceCheck[]): express.Application { + const server = new Server( + { + name: 'add-numbers-server', + version: '1.0.0' + }, + { + capabilities: { + tools: {} + } + } + ); + + server.setRequestHandler(ListToolsRequestSchema, async () => { + return { + tools: [ + { + name: 'add_numbers', + description: 'Add two numbers together', + inputSchema: { + type: 'object', + properties: { + a: { + type: 'number', + description: 'First number' + }, + b: { + type: 'number', + description: 'Second number' + } + }, + required: ['a', 'b'] + } + } + ] + }; + }); + + server.setRequestHandler(CallToolRequestSchema, async request => { + if (request.params.name === 'add_numbers') { + const { a, b } = request.params.arguments as { a: number; b: number }; + const result = a + b; + + checks.push({ + id: 'tool-add-numbers', + name: 'ToolAddNumbers', + description: 'Validates that the add_numbers tool works correctly', + status: result === a + b ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'MCP-Tools', + url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' + } + ], + details: { + a, + b, + result + } + }); + + return { + content: [ + { + type: 'text', + text: `The sum of ${a} and ${b} is ${result}` + } + ] + }; + } + + throw new Error(`Unknown tool: ${request.params.name}`); + }); + + const app = express(); + app.use(express.json()); + + app.use((req, res, next) => { + // Log incoming requests for debugging + // console.log(`Incoming request: ${req.method} ${req.url}`); + checks.push({ + id: 'incoming-request', + name: 'IncomingRequest', + description: `Received ${req.method} request for ${req.url}`, + status: 'INFO', + timestamp: new Date().toISOString(), + details: { + body: JSON.stringify(req.body) + } + }); + next(); + checks.push({ + id: 'outgoing-response', + name: 'OutgoingResponse', + // TODO: include MCP method? + description: `Sent ${res.statusCode} response`, + status: 'INFO', + timestamp: new Date().toISOString(), + details: { + // TODO: this isn't working + body: JSON.stringify(res.body) + } + }); + }); + + app.post('/mcp', async (req, res) => { + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined + }); + await server.connect(transport); + + await transport.handleRequest(req, res, req.body); + }); + + return app; +} + +export class ToolsCallScenario implements Scenario { + name = 'tools_call'; + private app: express.Application | null = null; + private httpServer: any = null; + private checks: ConformanceCheck[] = []; + + async start(): Promise { + this.checks = []; + this.app = createServer(this.checks); + this.httpServer = this.app.listen(0); + const port = this.httpServer.address().port; + return { serverUrl: `http://localhost:${port}/mcp` }; + } + + async stop() { + if (this.httpServer) { + await new Promise(resolve => this.httpServer.close(resolve)); + this.httpServer = null; + } + this.app = null; + } + + getChecks(): ConformanceCheck[] { + const expectedSlugs = ['tool-add-numbers']; + // add a failure if not in there already + for (const slug of expectedSlugs) { + if (!this.checks.find(c => c.id === slug)) { + // TODO: this is duplicated from above, refactor + this.checks.push({ + id: slug, + name: `ToolAddNumbers`, + description: `Validates that the add_numbers tool works correctly`, + status: 'FAILURE', + timestamp: new Date().toISOString(), + details: { message: 'Tool was not called by client' }, + specReferences: [ + { + id: 'MCP-Tools', + url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/tools#calling-tools' + } + ] + }); + } + } + return this.checks; + } +} From 8bce5d194bce20720568100dd14ec127d6d3f5d9 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Fri, 31 Oct 2025 17:07:42 +0000 Subject: [PATCH 4/4] fix status --- src/scenarios/tools_call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scenarios/tools_call.ts b/src/scenarios/tools_call.ts index f2b71f6..96dcf99 100644 --- a/src/scenarios/tools_call.ts +++ b/src/scenarios/tools_call.ts @@ -52,7 +52,7 @@ function createServer(checks: ConformanceCheck[]): express.Application { id: 'tool-add-numbers', name: 'ToolAddNumbers', description: 'Validates that the add_numbers tool works correctly', - status: result === a + b ? 'SUCCESS' : 'FAILURE', + status: 'SUCCESS', timestamp: new Date().toISOString(), specReferences: [ {