Add JSON Schema 2020-12 conformance test (SEP-1613) (#45)

felixweinberger · web-flow · commit f087e44bdb6c · 2025-11-20T18:20:06.000Z
Add conformance test to validate that MCP servers correctly preserve
JSON Schema 2020-12 keywords ($schema, $defs, additionalProperties)
in tool definitions.

Changes:
- Add JsonSchema2020_12Scenario in src/scenarios/server/json-schema-2020-12.ts
- Add json_schema_2020_12_tool to everything-server with tools/list override
  to return raw JSON Schema instead of Zod-converted schema
- Register scenario in pendingClientScenariosList (requires SDK PR #1135)

The test is added to pending because the client SDK currently strips
the JSON Schema 2020-12 keywords. Once PR #1135 is merged into the SDK,
this scenario can be moved to the active list.
diff --git a/examples/servers/typescript/everything-server.ts b/examples/servers/typescript/everything-server.ts
@@ -13,7 +13,13 @@ import {
   ResourceTemplate
 } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
-import { ElicitResultSchema } from '@modelcontextprotocol/sdk/types.js';
+import {
+  ElicitResultSchema,
+  ListToolsRequestSchema,
+  type ListToolsResult,
+  type Tool
+} from '@modelcontextprotocol/sdk/types.js';
+import { zodToJsonSchema } from 'zod-to-json-schema';
 import { z } from 'zod';
 import express from 'express';
 import cors from 'cors';
@@ -35,6 +41,28 @@ const TEST_IMAGE_BASE64 =
 const TEST_AUDIO_BASE64 =
   'UklGRiYAAABXQVZFZm10IBAAAAABAAEAQB8AAAB9AAACABAAZGF0YQIAAAA=';
 
+// SEP-1613: Raw JSON Schema 2020-12 definition for conformance testing
+// This schema includes $schema, $defs, and additionalProperties to test
+// that SDKs correctly preserve these fields
+const JSON_SCHEMA_2020_12_INPUT_SCHEMA = {
+  $schema: 'https://json-schema.org/draft/2020-12/schema',
+  type: 'object' as const,
+  $defs: {
+    address: {
+      type: 'object',
+      properties: {
+        street: { type: 'string' },
+        city: { type: 'string' }
+      }
+    }
+  },
+  properties: {
+    name: { type: 'string' },
+    address: { $ref: '#/$defs/address' }
+  },
+  additionalProperties: false
+};
+
 // Function to create a new MCP server instance (one per session)
 function createMcpServer() {
   const mcpServer = new McpServer(
@@ -566,6 +594,40 @@ function createMcpServer() {
     }
   );
 
+  // SEP-1613: JSON Schema 2020-12 conformance test tool
+  // This tool is registered with a Zod schema for tools/call validation,
+  // but the tools/list handler (below) returns the raw JSON Schema 2020-12
+  // definition to test that SDKs preserve $schema, $defs, additionalProperties
+  mcpServer.registerTool(
+    'json_schema_2020_12_tool',
+    {
+      description:
+        'Tool with JSON Schema 2020-12 features for conformance testing (SEP-1613)',
+      inputSchema: {
+        name: z.string().optional(),
+        address: z
+          .object({
+            street: z.string().optional(),
+            city: z.string().optional()
+          })
+          .optional()
+      }
+    },
+    async (args: {
+      name?: string;
+      address?: { street?: string; city?: string };
+    }) => {
+      return {
+        content: [
+          {
+            type: 'text',
+            text: `JSON Schema 2020-12 tool called with: ${JSON.stringify(args)}`
+          }
+        ]
+      };
+    }
+  );
+
   // Dynamic tool (registered later via timer)
 
   // ===== RESOURCES =====
@@ -829,6 +891,80 @@ function createMcpServer() {
     }
   );
 
+  // ===== SEP-1613: Override tools/list to return raw JSON Schema 2020-12 =====
+  // This override is necessary because registerTool converts Zod schemas to
+  // JSON Schema without preserving $schema, $defs, and additionalProperties.
+  // We need to return the raw JSON Schema for our test tool while using the
+  // SDK's conversion for other tools.
+  mcpServer.server.setRequestHandler(
+    ListToolsRequestSchema,
+    (): ListToolsResult => {
+      // Access internal registered tools (this is internal SDK API but stable)
+      const registeredTools = (mcpServer as any)._registeredTools as Record<
+        string,
+        {
+          enabled: boolean;
+          title?: string;
+          description?: string;
+          inputSchema?: any;
+          outputSchema?: any;
+          annotations?: any;
+          _meta?: any;
+        }
+      >;
+
+      return {
+        tools: Object.entries(registeredTools)
+          .filter(([, tool]) => tool.enabled)
+          .map(([name, tool]): Tool => {
+            // For our SEP-1613 test tool, return raw JSON Schema 2020-12
+            if (name === 'json_schema_2020_12_tool') {
+              return {
+                name,
+                description: tool.description,
+                inputSchema: JSON_SCHEMA_2020_12_INPUT_SCHEMA
+              };
+            }
+
+            // For other tools, convert Zod to JSON Schema
+            // Handle different inputSchema formats:
+            // - undefined/null: use empty object schema
+            // - Zod schema (has _def): convert directly
+            // - Raw shape (object with Zod values): wrap in z.object first
+            let inputSchema: Tool['inputSchema'];
+            if (!tool.inputSchema) {
+              inputSchema = { type: 'object' as const, properties: {} };
+            } else if ('_def' in tool.inputSchema) {
+              // Already a Zod schema
+              inputSchema = zodToJsonSchema(tool.inputSchema, {
+                strictUnions: true
+              }) as Tool['inputSchema'];
+            } else if (
+              typeof tool.inputSchema === 'object' &&
+              Object.keys(tool.inputSchema).length > 0
+            ) {
+              // Raw shape with Zod values
+              inputSchema = zodToJsonSchema(z.object(tool.inputSchema), {
+                strictUnions: true
+              }) as Tool['inputSchema'];
+            } else {
+              // Empty object or unknown format
+              inputSchema = { type: 'object' as const, properties: {} };
+            }
+
+            return {
+              name,
+              title: tool.title,
+              description: tool.description,
+              inputSchema,
+              annotations: tool.annotations,
+              _meta: tool._meta
+            };
+          })
+      };
+    }
+  );
+
   return mcpServer;
 }
 
diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts
@@ -25,6 +25,8 @@ import {
   ToolsCallEmbeddedResourceScenario
 } from './server/tools.js';
 
+import { JsonSchema2020_12Scenario } from './server/json-schema-2020-12.js';
+
 import { ElicitationDefaultsScenario } from './server/elicitation-defaults.js';
 import { ElicitationEnumsScenario } from './server/elicitation-enums.js';
 
@@ -51,7 +53,12 @@ import { listMetadataScenarios } from './client/auth/discovery-metadata.js';
 // Pending client scenarios (not yet fully tested/implemented)
 const pendingClientScenariosList: ClientScenario[] = [
   // Elicitation scenarios (SEP-1330)
-  new ElicitationEnumsScenario()
+  new ElicitationEnumsScenario(),
+
+  // JSON Schema 2020-12 (SEP-1613)
+  // This test is pending until the SDK includes PR #1135 which preserves
+  // $schema, $defs, and additionalProperties fields in tool schemas.
+  new JsonSchema2020_12Scenario()
 ];
 
 // All client scenarios
@@ -76,6 +83,9 @@ const allClientScenariosList: ClientScenario[] = [
   new ToolsCallSamplingScenario(),
   new ToolsCallElicitationScenario(),
 
+  // JSON Schema 2020-12 support (SEP-1613)
+  new JsonSchema2020_12Scenario(),
+
   // Elicitation scenarios (SEP-1034)
   new ElicitationDefaultsScenario(),
 
diff --git a/src/scenarios/server/json-schema-2020-12.ts b/src/scenarios/server/json-schema-2020-12.ts
@@ -0,0 +1,181 @@
+/**
+ * JSON Schema 2020-12 conformance test scenario (SEP-1613)
+ *
+ * Validates that MCP servers correctly preserve JSON Schema 2020-12 keywords
+ * in tool definitions, ensuring implementations don't strip $schema, $defs,
+ * or additionalProperties fields.
+ */
+
+import { ClientScenario, ConformanceCheck } from '../../types.js';
+import { connectToServer } from './client-helper.js';
+
+const EXPECTED_TOOL_NAME = 'json_schema_2020_12_tool';
+const EXPECTED_SCHEMA_DIALECT = 'https://json-schema.org/draft/2020-12/schema';
+
+export class JsonSchema2020_12Scenario implements ClientScenario {
+  name = 'json-schema-2020-12';
+  description = `Validates JSON Schema 2020-12 keyword preservation (SEP-1613).
+
+**Server Implementation Requirements:**
+
+Implement tool \`${EXPECTED_TOOL_NAME}\` with inputSchema containing JSON Schema 2020-12 features:
+
+\`\`\`json
+{
+  "name": "${EXPECTED_TOOL_NAME}",
+  "description": "Tool with JSON Schema 2020-12 features",
+  "inputSchema": {
+    "$schema": "${EXPECTED_SCHEMA_DIALECT}",
+    "type": "object",
+    "$defs": {
+      "address": {
+        "type": "object",
+        "properties": {
+          "street": { "type": "string" },
+          "city": { "type": "string" }
+        }
+      }
+    },
+    "properties": {
+      "name": { "type": "string" },
+      "address": { "$ref": "#/$defs/address" }
+    },
+    "additionalProperties": false
+  }
+}
+\`\`\`
+
+**Verification**: The test verifies that \`$schema\`, \`$defs\`, and \`additionalProperties\` are preserved in the tool listing response.`;
+
+  async run(serverUrl: string): Promise<ConformanceCheck[]> {
+    const checks: ConformanceCheck[] = [];
+    const specReferences = [
+      {
+        id: 'SEP-1613',
+        url: 'https://github.com/modelcontextprotocol/specification/pull/655'
+      }
+    ];
+
+    try {
+      const connection = await connectToServer(serverUrl);
+      const result = await connection.client.listTools();
+
+      // Find the test tool
+      const tool = result.tools?.find((t) => t.name === EXPECTED_TOOL_NAME);
+
+      // Check 1: Tool exists
+      checks.push({
+        id: 'json-schema-2020-12-tool-found',
+        name: 'JsonSchema2020_12ToolFound',
+        description: `Server advertises tool '${EXPECTED_TOOL_NAME}'`,
+        status: tool ? 'SUCCESS' : 'FAILURE',
+        timestamp: new Date().toISOString(),
+        errorMessage: tool
+          ? undefined
+          : `Tool '${EXPECTED_TOOL_NAME}' not found. Available tools: ${result.tools?.map((t) => t.name).join(', ') || 'none'}`,
+        specReferences,
+        details: {
+          toolFound: !!tool,
+          availableTools: result.tools?.map((t) => t.name) || []
+        }
+      });
+
+      if (!tool) {
+        await connection.close();
+        return checks;
+      }
+
+      const inputSchema = tool.inputSchema as Record<string, unknown>;
+
+      // Check 2: $schema field preserved
+      const hasSchema = '$schema' in inputSchema;
+      const schemaValue = inputSchema['$schema'];
+      const schemaCorrect = schemaValue === EXPECTED_SCHEMA_DIALECT;
+
+      checks.push({
+        id: 'json-schema-2020-12-$schema',
+        name: 'JsonSchema2020_12$Schema',
+        description: `inputSchema.$schema field preserved with value '${EXPECTED_SCHEMA_DIALECT}'`,
+        status: hasSchema && schemaCorrect ? 'SUCCESS' : 'FAILURE',
+        timestamp: new Date().toISOString(),
+        errorMessage: !hasSchema
+          ? '$schema field missing from inputSchema - field was likely stripped'
+          : !schemaCorrect
+            ? `$schema has unexpected value: ${JSON.stringify(schemaValue)}`
+            : undefined,
+        specReferences,
+        details: {
+          hasSchema,
+          schemaValue,
+          expected: EXPECTED_SCHEMA_DIALECT
+        }
+      });
+
+      // Check 3: $defs field preserved
+      const hasDefs = '$defs' in inputSchema;
+      const defsValue = inputSchema['$defs'] as
+        | Record<string, unknown>
+        | undefined;
+      const defsHasAddress = defsValue && 'address' in defsValue;
+
+      checks.push({
+        id: 'json-schema-2020-12-$defs',
+        name: 'JsonSchema2020_12$Defs',
+        description:
+          'inputSchema.$defs field preserved with expected structure',
+        status: hasDefs && defsHasAddress ? 'SUCCESS' : 'FAILURE',
+        timestamp: new Date().toISOString(),
+        errorMessage: !hasDefs
+          ? '$defs field missing from inputSchema - field was likely stripped'
+          : !defsHasAddress
+            ? '$defs exists but missing expected "address" definition'
+            : undefined,
+        specReferences,
+        details: {
+          hasDefs,
+          defsKeys: defsValue ? Object.keys(defsValue) : [],
+          defsValue
+        }
+      });
+
+      // Check 4: additionalProperties field preserved
+      const hasAdditionalProps = 'additionalProperties' in inputSchema;
+      const additionalPropsValue = inputSchema['additionalProperties'];
+      const additionalPropsCorrect = additionalPropsValue === false;
+
+      checks.push({
+        id: 'json-schema-2020-12-additionalProperties',
+        name: 'JsonSchema2020_12AdditionalProperties',
+        description: 'inputSchema.additionalProperties field preserved',
+        status:
+          hasAdditionalProps && additionalPropsCorrect ? 'SUCCESS' : 'FAILURE',
+        timestamp: new Date().toISOString(),
+        errorMessage: !hasAdditionalProps
+          ? 'additionalProperties field missing from inputSchema - field was likely stripped'
+          : !additionalPropsCorrect
+            ? `additionalProperties has unexpected value: ${JSON.stringify(additionalPropsValue)}, expected: false`
+            : undefined,
+        specReferences,
+        details: {
+          hasAdditionalProps,
+          additionalPropsValue,
+          expected: false
+        }
+      });
+
+      await connection.close();
+    } catch (error) {
+      checks.push({
+        id: 'json-schema-2020-12-error',
+        name: 'JsonSchema2020_12Error',
+        description: 'JSON Schema 2020-12 conformance test',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
+        specReferences
+      });
+    }
+
+    return checks;
+  }
+}
