From ca636adfb740146209b884c788cf0c843474a4d1 Mon Sep 17 00:00:00 2001 From: Luke Sanderson Date: Fri, 12 Sep 2025 17:09:58 +0100 Subject: [PATCH 1/4] Add accuracy tests --- tests/accuracy/createDeployment.test.ts | 57 ++++++++++++++++++++ tests/accuracy/deleteDeployment.test.ts | 69 +++++++++++++++++++++++++ tests/accuracy/listDeployments.test.ts | 40 ++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 tests/accuracy/createDeployment.test.ts create mode 100644 tests/accuracy/deleteDeployment.test.ts create mode 100644 tests/accuracy/listDeployments.test.ts diff --git a/tests/accuracy/createDeployment.test.ts b/tests/accuracy/createDeployment.test.ts new file mode 100644 index 000000000..3ba1f8c51 --- /dev/null +++ b/tests/accuracy/createDeployment.test.ts @@ -0,0 +1,57 @@ +import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js"; + +describeAccuracyTests([ + { + prompt: "Setup a local MongoDB cluster", + expectedToolCalls: [ + { + toolName: "atlas-local-create-deployment", + parameters: {}, + }, + ], + }, + { + prompt: "Create a local MongoDB instance", + expectedToolCalls: [ + { + toolName: "atlas-local-create-deployment", + parameters: {}, + }, + ], + }, + { + prompt: "Setup a local MongoDB database", + expectedToolCalls: [ + { + toolName: "atlas-local-create-deployment", + parameters: {}, + }, + ], + }, + { + prompt: "Create a local MongoDB database named 'local-mflix'", + expectedToolCalls: [ + { + toolName: "atlas-local-create-deployment", + parameters: { + deploymentName: "local-mflix", + }, + }, + ], + }, + { + prompt: "If and only if, the local MongoDB deployment 'this-database' does not exist, then create it", + expectedToolCalls: [ + { + toolName: "list-collections", + parameters: {}, + }, + { + toolName: "create-collection", + parameters: { + deploymentName: "this-database", + }, + }, + ], + }, +]); diff --git a/tests/accuracy/deleteDeployment.test.ts b/tests/accuracy/deleteDeployment.test.ts new file mode 100644 index 000000000..dc1c7e89b --- /dev/null +++ b/tests/accuracy/deleteDeployment.test.ts @@ -0,0 +1,69 @@ +import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js"; + +describeAccuracyTests([ + { + prompt: "Delete the local MongoDB cluster called 'my-database'", + expectedToolCalls: [ + { + toolName: "atlas-local-delete-deployment", + parameters: { + deploymentName: "my-database", + }, + }, + ], + }, + { + prompt: "Delete the local MongoDB database called 'my-instance'", + expectedToolCalls: [ + { + toolName: "atlas-local-delete-deployment", + parameters: { + deploymentName: "my-instance", + }, + }, + ], + }, + { + prompt: "Delete all my local MongoDB instances", + expectedToolCalls: [ + { + toolName: "atlas-local-list-deployments", + parameters: {}, + }, + { + toolName: "atlas-local-delete-deployment", + parameters: {}, + }, + ], + }, + { + prompt: "If and only if, the local MongoDB deployment 'local-mflix' exists, then delete it", + expectedToolCalls: [ + { + toolName: "atlas-local-list-deployments", + parameters: {}, + }, + { + toolName: "atlas-local-delete-deployment", + parameters: { + deploymentName: "local-mflix", + }, + }, + ], + }, + { + prompt: "If and only if, the local MongoDB deployment 'new-database' does not exist, then create it", + expectedToolCalls: [ + { + toolName: "list-collections", + parameters: {}, + }, + { + toolName: "create-collection", + parameters: { + deploymentName: "new-database", + }, + }, + ], + }, +]); diff --git a/tests/accuracy/listDeployments.test.ts b/tests/accuracy/listDeployments.test.ts new file mode 100644 index 000000000..e6b252c50 --- /dev/null +++ b/tests/accuracy/listDeployments.test.ts @@ -0,0 +1,40 @@ +import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js"; + +describeAccuracyTests([ + { + prompt: "What MongoDB clusters do I have running?", + expectedToolCalls: [ + { + toolName: "atlas-local-list-deployments", + parameters: {}, + }, + ], + }, + { + prompt: "What MongoDB databases do I have running?", + expectedToolCalls: [ + { + toolName: "atlas-local-list-deployments", + parameters: {}, + }, + ], + }, + { + prompt: "What MongoDB instances do I have running?", + expectedToolCalls: [ + { + toolName: "atlas-local-list-deployments", + parameters: {}, + }, + ], + }, + { + prompt: "How many MongoDB clusters are running?", + expectedToolCalls: [ + { + toolName: "atlas-local-list-deployments", + parameters: {}, + }, + ], + }, +]); From 62e47761e6ab51bbb7db66cc7911d7e2734f500f Mon Sep 17 00:00:00 2001 From: Luke Sanderson Date: Fri, 12 Sep 2025 17:35:26 +0100 Subject: [PATCH 2/4] Update tests --- tests/accuracy/createDeployment.test.ts | 32 ++++++++++++++----------- tests/accuracy/deleteDeployment.test.ts | 17 +------------ 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/tests/accuracy/createDeployment.test.ts b/tests/accuracy/createDeployment.test.ts index 3ba1f8c51..e23ee87d5 100644 --- a/tests/accuracy/createDeployment.test.ts +++ b/tests/accuracy/createDeployment.test.ts @@ -2,54 +2,58 @@ import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js"; describeAccuracyTests([ { - prompt: "Setup a local MongoDB cluster", + prompt: "Setup a local MongoDB cluster named 'local-cluster'", expectedToolCalls: [ { toolName: "atlas-local-create-deployment", - parameters: {}, + parameters: { + deploymentName: "local-cluster", + }, }, ], }, { - prompt: "Create a local MongoDB instance", + prompt: "Create a local MongoDB instance named 'local-cluster'", expectedToolCalls: [ { toolName: "atlas-local-create-deployment", - parameters: {}, + parameters: { + deploymentName: "local-cluster", + }, }, ], }, { - prompt: "Setup a local MongoDB database", + prompt: "Setup a local MongoDB database named 'local-cluster'", expectedToolCalls: [ { toolName: "atlas-local-create-deployment", - parameters: {}, + parameters: { + deploymentName: "local-cluster", + }, }, ], }, { - prompt: "Create a local MongoDB database named 'local-mflix'", + prompt: "Setup a local MongoDB cluster, do not specify a name", expectedToolCalls: [ { toolName: "atlas-local-create-deployment", - parameters: { - deploymentName: "local-mflix", - }, + parameters: {}, }, ], }, { - prompt: "If and only if, the local MongoDB deployment 'this-database' does not exist, then create it", + prompt: "If and only if, the local MongoDB deployment 'new-database' does not exist, then create it", expectedToolCalls: [ { - toolName: "list-collections", + toolName: "atlas-local-list-deployments", parameters: {}, }, { - toolName: "create-collection", + toolName: "atlas-local-create-deployment", parameters: { - deploymentName: "this-database", + deploymentName: "new-database", }, }, ], diff --git a/tests/accuracy/deleteDeployment.test.ts b/tests/accuracy/deleteDeployment.test.ts index dc1c7e89b..8419bd599 100644 --- a/tests/accuracy/deleteDeployment.test.ts +++ b/tests/accuracy/deleteDeployment.test.ts @@ -13,7 +13,7 @@ describeAccuracyTests([ ], }, { - prompt: "Delete the local MongoDB database called 'my-instance'", + prompt: "Delete the local MongoDB atlas database called 'my-instance'", expectedToolCalls: [ { toolName: "atlas-local-delete-deployment", @@ -51,19 +51,4 @@ describeAccuracyTests([ }, ], }, - { - prompt: "If and only if, the local MongoDB deployment 'new-database' does not exist, then create it", - expectedToolCalls: [ - { - toolName: "list-collections", - parameters: {}, - }, - { - toolName: "create-collection", - parameters: { - deploymentName: "new-database", - }, - }, - ], - }, ]); From df8201e25b9d2393319d23d66fbd7b711924bef0 Mon Sep 17 00:00:00 2001 From: Luke Sanderson Date: Mon, 15 Sep 2025 11:56:15 +0100 Subject: [PATCH 3/4] Update delete tests --- tests/accuracy/deleteDeployment.test.ts | 17 +++++++++++++---- tests/accuracy/listDeployments.test.ts | 9 --------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/accuracy/deleteDeployment.test.ts b/tests/accuracy/deleteDeployment.test.ts index 8419bd599..1a096ae4a 100644 --- a/tests/accuracy/deleteDeployment.test.ts +++ b/tests/accuracy/deleteDeployment.test.ts @@ -30,10 +30,7 @@ describeAccuracyTests([ toolName: "atlas-local-list-deployments", parameters: {}, }, - { - toolName: "atlas-local-delete-deployment", - parameters: {}, - }, + // There is none, so no delete call ], }, { @@ -43,6 +40,18 @@ describeAccuracyTests([ toolName: "atlas-local-list-deployments", parameters: {}, }, + // There doesn't exist one so no delete call + ], + }, + { + prompt: "Create a local MongoDB cluster named 'local-mflix' then delete it immediately", + expectedToolCalls: [ + { + toolName: "atlas-local-create-deployment", + parameters: { + deploymentName: "local-mflix", + }, + }, { toolName: "atlas-local-delete-deployment", parameters: { diff --git a/tests/accuracy/listDeployments.test.ts b/tests/accuracy/listDeployments.test.ts index e6b252c50..762ef4f95 100644 --- a/tests/accuracy/listDeployments.test.ts +++ b/tests/accuracy/listDeployments.test.ts @@ -10,15 +10,6 @@ describeAccuracyTests([ }, ], }, - { - prompt: "What MongoDB databases do I have running?", - expectedToolCalls: [ - { - toolName: "atlas-local-list-deployments", - parameters: {}, - }, - ], - }, { prompt: "What MongoDB instances do I have running?", expectedToolCalls: [ From b25848402695e035dfb207c2f7eb998096cfe761 Mon Sep 17 00:00:00 2001 From: Luke Sanderson Date: Mon, 15 Sep 2025 12:40:09 +0100 Subject: [PATCH 4/4] Add mocked tests --- tests/accuracy/createDeployment.test.ts | 21 ++++++++++++ tests/accuracy/deleteDeployment.test.ts | 43 +++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/tests/accuracy/createDeployment.test.ts b/tests/accuracy/createDeployment.test.ts index e23ee87d5..559206a46 100644 --- a/tests/accuracy/createDeployment.test.ts +++ b/tests/accuracy/createDeployment.test.ts @@ -1,4 +1,5 @@ import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; describeAccuracyTests([ { @@ -58,4 +59,24 @@ describeAccuracyTests([ }, ], }, + { + prompt: "If and only if, the local MongoDB deployment 'existing-database' does not exist, then create it", + mockedTools: { + "atlas-local-list-deployments": (): CallToolResult => ({ + content: [ + { type: "text", text: "Found 1 deployment:" }, + { + type: "text", + text: "Deployment Name | State | MongoDB Version\n----------------|----------------|----------------\nexisting-database | Running | 6.0", + }, + ], + }), + }, + expectedToolCalls: [ + { + toolName: "atlas-local-list-deployments", + parameters: {}, + }, + ], + }, ]); diff --git a/tests/accuracy/deleteDeployment.test.ts b/tests/accuracy/deleteDeployment.test.ts index 1a096ae4a..dcd4d8249 100644 --- a/tests/accuracy/deleteDeployment.test.ts +++ b/tests/accuracy/deleteDeployment.test.ts @@ -1,4 +1,5 @@ import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; describeAccuracyTests([ { @@ -25,22 +26,60 @@ describeAccuracyTests([ }, { prompt: "Delete all my local MongoDB instances", + mockedTools: { + "atlas-local-list-deployments": (): CallToolResult => ({ + content: [ + { type: "text", text: "Found 1 deployment:" }, + { + type: "text", + text: "Deployment Name | State | MongoDB Version\n----------------|----------------|----------------\nlocal-mflix | Running | 6.0\nlocal-comics | Running | 6.0", + }, + ], + }), + }, expectedToolCalls: [ { toolName: "atlas-local-list-deployments", parameters: {}, }, - // There is none, so no delete call + { + toolName: "atlas-local-delete-deployment", + parameters: { + deploymentName: "local-mflix", + }, + }, + { + toolName: "atlas-local-delete-deployment", + parameters: { + deploymentName: "local-comics", + }, + }, ], }, { prompt: "If and only if, the local MongoDB deployment 'local-mflix' exists, then delete it", + mockedTools: { + "atlas-local-list-deployments": (): CallToolResult => ({ + content: [ + { type: "text", text: "Found 1 deployment:" }, + { + type: "text", + text: "Deployment Name | State | MongoDB Version\n----------------|----------------|----------------\nlocal-mflix | Running | 6.0", + }, + ], + }), + }, expectedToolCalls: [ { toolName: "atlas-local-list-deployments", parameters: {}, }, - // There doesn't exist one so no delete call + { + toolName: "atlas-local-delete-deployment", + parameters: { + deploymentName: "local-mflix", + }, + }, ], }, {