Draft

Author: kylelai1

scripts/accuracy/runAccuracyTests.sh

@@ -8,6 +8,11 @@ export MDB_ACCURACY_RUN_ID=$(npx uuid v4)
 # export MDB_AZURE_OPEN_AI_API_KEY=""
 # export MDB_AZURE_OPEN_AI_API_URL=""
 
+# For providing Atlas API credentials (required for Atlas tools)
+# Set dummy values for testing (allows Atlas tools to be registered for mocking)
+export MDB_MCP_API_CLIENT_ID=${MDB_MCP_API_CLIENT_ID:-"test-atlas-client-id"}
+export MDB_MCP_API_CLIENT_SECRET=${MDB_MCP_API_CLIENT_SECRET:-"test-atlas-client-secret"}
+
 # For providing a mongodb based storage to store accuracy result
 # export MDB_ACCURACY_MDB_URL=""
 # export MDB_ACCURACY_MDB_DB=""

tests/accuracy/listPerformanceAdvisor.test.ts

@@ -0,0 +1,64 @@
+import { PerformanceAdvisorOperation } from "../../src/common/atlas/performanceAdvisorUtils.js";
+import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+
+describeAccuracyTests([
+    {
+        prompt: "Can you give me index suggestions for the database 'mflix' in the project 'mflix' and cluster 'mflix-cluster'?",
+        expectedToolCalls: [
+            {
+                toolName: "atlas-list-projects",
+                parameters: {},
+            },
+            {
+                toolName: "atlas-list-clusters",
+                parameters: {
+                    projectId: "mflix",
+                },
+            },
+            {
+                toolName: "atlas-list-performance-advisor",
+                parameters: {
+                    projectId: "mflix",
+                    clusterName: "mflix-cluster",
+                    operations: [PerformanceAdvisorOperation.SUGGESTED_INDEXES],
+                },
+            },
+        ],
+        mockedTools: {
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            "atlas-list-performance-advisor": (..._parameters): CallToolResult => {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: "Found 2 performance advisor recommendations\n\n## Suggested Indexes\n# | Namespace | Weight | Avg Obj Size | Index Keys\n---|-----------|--------|--------------|------------\n1 | mflix.movies | 0.8 | 1024 | title, year\n2 | mflix.shows | 0.6 | 512 | genre, rating",
+                        },
+                    ],
+                };
+            },
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            "atlas-list-projects": (..._parameters): CallToolResult => {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: "Found 1 project\n\n# | Name | ID\n---|------|----\n1 | mflix | mflix",
+                        },
+                    ],
+                };
+            },
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            "atlas-list-clusters": (..._parameters): CallToolResult => {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: "Found 1 cluster\n\n# | Name | Type | State\n---|------|------|-----\n1 | mflix-cluster | REPLICASET | IDLE",
+                        },
+                    ],
+                };
+            },
+        },
+    },
+]);

tests/accuracy/sdk/accuracyTestingClient.ts

@@ -79,10 +79,24 @@ export class AccuracyTestingClient {
         this.llmToolCalls = [];
     }
 
-    static async initializeClient(mdbConnectionString: string): Promise<AccuracyTestingClient> {
+    static async initializeClient(
+        mdbConnectionString: string,
+        atlasApiClientId?: string,
+        atlasApiClientSecret?: string
+    ): Promise<AccuracyTestingClient> {
+        const args = [MCP_SERVER_CLI_SCRIPT, "--connectionString", mdbConnectionString];
+
+        // Add Atlas API credentials if provided
+        if (atlasApiClientId) {
+            args.push("--apiClientId", atlasApiClientId);
+        }
+        if (atlasApiClientSecret) {
+            args.push("--apiClientSecret", atlasApiClientSecret);
+        }
+
         const clientTransport = new StdioClientTransport({
             command: process.execPath,
-            args: [MCP_SERVER_CLI_SCRIPT, "--connectionString", mdbConnectionString],
+            args,
         });
 
         const client = await createMCPClient({

tests/accuracy/sdk/describeAccuracyTests.ts

@@ -66,6 +66,9 @@ export function describeAccuracyTests(accuracyTestConfigs: AccuracyTestConfig[])
         const mdbIntegration = setupMongoDBIntegrationTest({}, []);
         const { populateTestData, cleanupTestDatabases } = prepareTestData(mdbIntegration);
 
+        const atlasApiClientId = process.env.MDB_MCP_API_CLIENT_ID;
+        const atlasApiClientSecret = process.env.MDB_MCP_API_CLIENT_SECRET;
+
         let commitSHA: string;
         let accuracyResultStorage: AccuracyResultStorage;
         let testMCPClient: AccuracyTestingClient;
@@ -79,7 +82,11 @@ export function describeAccuracyTests(accuracyTestConfigs: AccuracyTestConfig[])
             commitSHA = retrievedCommitSHA;
 
             accuracyResultStorage = getAccuracyResultStorage();
-            testMCPClient = await AccuracyTestingClient.initializeClient(mdbIntegration.connectionString());
+            testMCPClient = await AccuracyTestingClient.initializeClient(
+                mdbIntegration.connectionString(),
+                atlasApiClientId,
+                atlasApiClientSecret
+            );
             agent = getVercelToolCallingAgent();
         });