Add client conformance tests (#1102)

Co-authored-by: Stephen Halter <halter73@gmail.com>

Author: mikekistler

ModelContextProtocol.slnx

@@ -70,6 +70,7 @@
   <Folder Name="/tests/">
     <Project Path="tests/ModelContextProtocol.Analyzers.Tests/ModelContextProtocol.Analyzers.Tests.csproj" />
     <Project Path="tests/ModelContextProtocol.AspNetCore.Tests/ModelContextProtocol.AspNetCore.Tests.csproj" />
+    <Project Path="tests/ModelContextProtocol.ConformanceClient/ModelContextProtocol.ConformanceClient.csproj" />
     <Project Path="tests/ModelContextProtocol.ConformanceServer/ModelContextProtocol.ConformanceServer.csproj" />
     <Project Path="tests/ModelContextProtocol.TestOAuthServer/ModelContextProtocol.TestOAuthServer.csproj" />
     <Project Path="tests/ModelContextProtocol.Tests/ModelContextProtocol.Tests.csproj" />

tests/Common/Utils/NodeHelpers.cs

@@ -0,0 +1,70 @@
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+
+namespace ModelContextProtocol.Tests.Utils;
+
+/// <summary>
+/// Helper utilities for Node.js and npm operations.
+/// </summary>
+public static class NodeHelpers
+{
+    /// <summary>
+    /// Creates a ProcessStartInfo configured to run npx with the specified arguments.
+    /// </summary>
+    /// <param name="arguments">The arguments to pass to npx.</param>
+    /// <returns>A configured ProcessStartInfo for running npx.</returns>
+    public static ProcessStartInfo NpxStartInfo(string arguments)
+    {
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+        {
+            // On Windows, npx is a PowerShell script, so we need to use cmd.exe to invoke it
+            return new ProcessStartInfo
+            {
+                FileName = "cmd.exe",
+                Arguments = $"/c npx {arguments}",
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            };
+        }
+        else
+        {
+            // On Unix-like systems, npx is typically a shell script that can be executed directly
+            return new ProcessStartInfo
+            {
+                FileName = "npx",
+                Arguments = arguments,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                CreateNoWindow = true
+            };
+        }
+    }
+
+    /// <summary>
+    /// Checks if Node.js and npx are installed and available on the system.
+    /// </summary>
+    /// <returns>True if npx is available, false otherwise.</returns>
+    public static bool IsNpxInstalled()
+    {
+        try
+        {
+            var startInfo = NpxStartInfo("--version");
+
+            using var process = Process.Start(startInfo);
+            if (process == null)
+            {
+                return false;
+            }
+
+            process.WaitForExit(5000);
+            return process.ExitCode == 0;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+}

tests/ModelContextProtocol.AspNetCore.Tests/ClientConformanceTests.cs

@@ -0,0 +1,99 @@
+using System.Diagnostics;
+using System.Text;
+using ModelContextProtocol.Tests.Utils;
+
+namespace ModelContextProtocol.ConformanceTests;
+
+/// <summary>
+/// Runs the official MCP conformance tests against the ConformanceClient.
+/// This test runs the Node.js-based conformance test suite for the client
+/// and reports the results.
+/// </summary>
+public class ClientConformanceTests //: IAsyncLifetime
+{
+    private readonly ITestOutputHelper _output;
+
+    // Public static property required for SkipUnless attribute
+    public static bool IsNpxInstalled => NodeHelpers.IsNpxInstalled();
+
+    public ClientConformanceTests(ITestOutputHelper output)
+    {
+        _output = output;
+    }
+
+    [Theory(Skip = "npx is not installed. Skipping client conformance tests.", SkipUnless = nameof(IsNpxInstalled))]
+    [InlineData("initialize")]
+    [InlineData("tools_call")]
+    [InlineData("auth/metadata-default")]
+    [InlineData("auth/metadata-var1")]
+    [InlineData("auth/metadata-var2")]
+    [InlineData("auth/metadata-var3")]
+    [InlineData("auth/basic-cimd")]
+    // [InlineData("auth/2025-03-26-oauth-metadata-backcompat")]
+    // [InlineData("auth/2025-03-26-oauth-endpoint-fallback")]
+    [InlineData("auth/scope-from-www-authenticate")]
+    [InlineData("auth/scope-from-scopes-supported")]
+    [InlineData("auth/scope-omitted-when-undefined")]
+    [InlineData("auth/scope-step-up")]
+    public async Task RunConformanceTest(string scenario)
+    {
+        // Run the conformance test suite
+        var result = await RunClientConformanceScenario(scenario);
+
+        // Report the results
+        Assert.True(result.Success,
+            $"Conformance test failed.\n\nStdout:\n{result.Output}\n\nStderr:\n{result.Error}");
+    }
+
+    private async Task<(bool Success, string Output, string Error)> RunClientConformanceScenario(string scenario)
+    {
+        // Construct an absolute path to the conformance client executable
+        var exeSuffix = OperatingSystem.IsWindows() ? ".exe" : "";
+        var conformanceClientPath = Path.GetFullPath($"./ModelContextProtocol.ConformanceClient{exeSuffix}");
+        // Replace AspNetCore.Tests with ConformanceClient in the path
+        conformanceClientPath = conformanceClientPath.Replace("AspNetCore.Tests", "ConformanceClient");
+
+        if (!File.Exists(conformanceClientPath))
+        {
+            throw new FileNotFoundException(
+                $"ConformanceClient executable not found at: {conformanceClientPath}");
+        }
+
+        var startInfo = NodeHelpers.NpxStartInfo($"-y @modelcontextprotocol/conformance client --scenario {scenario} --command \"{conformanceClientPath} {scenario}\"");
+
+        var outputBuilder = new StringBuilder();
+        var errorBuilder = new StringBuilder();
+
+        var process = new Process { StartInfo = startInfo };
+
+        process.OutputDataReceived += (sender, e) =>
+        {
+            if (e.Data != null)
+            {
+                _output.WriteLine(e.Data);
+                outputBuilder.AppendLine(e.Data);
+            }
+        };
+
+        process.ErrorDataReceived += (sender, e) =>
+        {
+            if (e.Data != null)
+            {
+                _output.WriteLine(e.Data);
+                errorBuilder.AppendLine(e.Data);
+            }
+        };
+
+        process.Start();
+        process.BeginOutputReadLine();
+        process.BeginErrorReadLine();
+
+        await process.WaitForExitAsync();
+
+        return (
+            Success: process.ExitCode == 0,
+            Output: outputBuilder.ToString(),
+            Error: errorBuilder.ToString()
+        );
+    }
+}

tests/ModelContextProtocol.AspNetCore.Tests/ModelContextProtocol.AspNetCore.Tests.csproj

@@ -57,6 +57,7 @@
     <ProjectReference Include="..\..\src\ModelContextProtocol.AspNetCore\ModelContextProtocol.AspNetCore.csproj" />
     <ProjectReference Include="..\ModelContextProtocol.TestSseServer\ModelContextProtocol.TestSseServer.csproj" />
     <ProjectReference Include="..\ModelContextProtocol.TestOAuthServer\ModelContextProtocol.TestOAuthServer.csproj" />
+    <ProjectReference Include="..\ModelContextProtocol.ConformanceClient\ModelContextProtocol.ConformanceClient.csproj" />
     <ProjectReference Include="..\ModelContextProtocol.ConformanceServer\ModelContextProtocol.ConformanceServer.csproj" />
   </ItemGroup>
 

tests/ModelContextProtocol.AspNetCore.Tests/ServerConformanceTests.cs

@@ -98,7 +98,7 @@ public async ValueTask DisposeAsync()
     public async Task RunConformanceTests()
     {
         // Check if Node.js is installed
-        Assert.SkipWhen(!IsNodeInstalled(), "Node.js is not installed. Skipping conformance tests.");
+        Assert.SkipWhen(!NodeHelpers.IsNpxInstalled(), "Node.js is not installed. Skipping conformance tests.");
 
         // Run the conformance test suite
         var result = await RunNpxConformanceTests();
@@ -117,15 +117,7 @@ private void StartConformanceServer()
 
     private async Task<(bool Success, string Output, string Error)> RunNpxConformanceTests()
     {
-        var startInfo = new ProcessStartInfo
-        {
-            FileName = "npx",
-            Arguments = $"-y @modelcontextprotocol/conformance server --url {_serverUrl}",
-            RedirectStandardOutput = true,
-            RedirectStandardError = true,
-            UseShellExecute = false,
-            CreateNoWindow = true
-        };
+        var startInfo = NodeHelpers.NpxStartInfo($"-y @modelcontextprotocol/conformance server --url {_serverUrl}");
 
         var outputBuilder = new StringBuilder();
         var errorBuilder = new StringBuilder();
@@ -162,33 +154,4 @@ private void StartConformanceServer()
             Error: errorBuilder.ToString()
         );
     }
-
-    private static bool IsNodeInstalled()
-    {
-        try
-        {
-            var startInfo = new ProcessStartInfo
-            {
-                FileName = "npx",   // Check specifically for npx because windows seems unable to find it
-                Arguments = "--version",
-                RedirectStandardOutput = true,
-                RedirectStandardError = true,
-                UseShellExecute = false,
-                CreateNoWindow = true
-            };
-
-            using var process = Process.Start(startInfo);
-            if (process == null)
-            {
-                return false;
-            }
-
-            process.WaitForExit(5000);
-            return process.ExitCode == 0;
-        }
-        catch
-        {
-            return false;
-        }
-    }
 }

tests/ModelContextProtocol.ConformanceClient/ModelContextProtocol.ConformanceClient.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>net10.0;net9.0;net8.0</TargetFrameworks>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(TargetFramework)' == 'net9.0'">
+    <!-- For better test coverage, only disable reflection in one of the targets -->
+    <JsonSerializerIsReflectionEnabledByDefault>false</JsonSerializerIsReflectionEnabledByDefault>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../../src/ModelContextProtocol.Core/ModelContextProtocol.Core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" />
+  </ItemGroup>
+
+</Project>