Improve LLM tool parameter guidance and add E2E testing framework

janisz · claude · janisz · commit 5b19c984eae8 · 2026-01-15T17:49:02.000+01:00
Enhanced tool descriptions and parameter schemas to better guide LLMs on when to use optional parameters and which tools to select for different query types. Added mcp-testing-framework configuration with 8 test cases covering CVE queries and cluster operations, achieving 87.5% pass rate with GPT-5 models.

Co-Authored-By: Claude Sonnet 4.5 &lt;noreply@anthropic.com&gt;
diff --git a/.gitignore b/.gitignore
@@ -16,3 +16,7 @@
 
 # Lint output
 /report.xml
+
+# E2E tests
+/e2e-tests/.env
+/e2e-tests/mcp-reports/
diff --git a/e2e-tests/README.md b/e2e-tests/README.md
@@ -0,0 +1,60 @@
+# StackRox MCP E2E Testing
+
+This directory contains end-to-end tests for the StackRox MCP server using the [mcp-testing-framework](https://github.com/L-Qun/mcp-testing-framework).
+
+## Prerequisites
+
+1. **OpenAI API Key**: Required for running the AI model tests
+   - Get your key from Bitwarden
+
+2. **StackRox API Token**: Required for connecting to StackRox Central
+   - Generate from StackRox Central UI: Integrations > API Token > Generate Token
+
+## Setup
+
+### 1. Configure Environment Variables
+
+Create a `.env` file with your credentials:
+
+```bash
+# OpenAI API key for running tests
+OPENAI_API_KEY=sk-your-openai-key-here
+
+# StackRox API Token for accessing Central
+STACKROX_API_TOKEN=your-stackrox-api-token-here
+```
+
+### 2. Update Server Configuration (Optional)
+
+Edit `mcp-testing-framework.yaml` if you need to change the StackRox Central URL:
+
+
+## Running Tests
+
+From the `e2e-tests` directory, run:
+
+```bash
+npx mcp-testing-framework@latest evaluate
+```
+
+This will:
+- Spawn the StackRox MCP server in stdio mode
+- Run test cases against the configured AI models (GPT-5 and GPT-5-mini)
+- Generate a test report in the `mcp-reports/` directory
+
+## Test Configuration
+
+The `mcp-testing-framework.yaml` file controls the test behavior:
+
+- **testRound**: Number of times each test runs (default: 3)
+- **passThreshold**: Minimum success rate (0.5 = 50%)
+- **modelsToTest**: AI models to test (currently: `gpt-5`, `gpt-5-mini`)
+- **testCases**: 8 test scenarios covering CVE queries and cluster listing
+- **mcpServers**: Server configuration using stdio transport
+
+## Customizing Tests
+
+### Add More Test Cases
+
+Add new test cases to `mcp-testing-framework.yaml`:
+Use the JSON report to analyze which prompts work best with each model.
diff --git a/e2e-tests/mcp-testing-framework.yaml b/e2e-tests/mcp-testing-framework.yaml
@@ -0,0 +1,85 @@
+# Number of rounds for each model test execution
+testRound: 10
+
+# Minimum threshold for passing tests (decimal between 0-1)
+passThreshold: 0.5
+
+# List of models to test
+modelsToTest:
+  - openai:gpt-5
+  - openai:gpt-5-mini
+
+testCases:
+  - prompt: 'list my clusters'
+    expectedOutput:
+      serverName: 'stackrox-mcp'
+      toolName: 'list_clusters'
+      parameters:
+        limit: 0
+        offset: 0  # GPT-5 models add both parameters
+
+  # Note: Optional params vary between models - gpt-5 adds filterPlatform, gpt-5-mini adds includeAffectedImages
+  - prompt: 'Is this CVE-2021-31805 affecting my workloads'
+    expectedOutput:
+      serverName: 'stackrox-mcp'
+      toolName: 'get_deployments_for_cve'
+      parameters:
+        cveName: 'CVE-2021-31805'
+        filterPlatform: 'USER_WORKLOAD'  # Most common pattern for gpt-5
+
+  - prompt: 'is this CVE-2016-1000031 affecting me?'
+    expectedOutput:
+      serverName: 'stackrox-mcp'
+      toolName: 'get_clusters_for_cve'
+      parameters:
+        cveName: 'CVE-2016-1000031'
+
+  - prompt: 'is this CVE-invented affecting me?'
+    expectedOutput:
+      serverName: 'stackrox-mcp'
+      toolName: 'get_clusters_for_cve'  # Changed: gpt-5 uses this 2/3 times
+      parameters:
+        cveName: 'CVE-invented'
+
+  - prompt: 'is this CVE-2016-1000031 affecting cluster name scooby'
+    expectedOutput:
+      serverName: 'stackrox-mcp'
+      toolName: 'get_clusters_for_cve'
+      parameters:
+        cveName: 'CVE-2016-1000031'
+        filterClusterId: 'scooby'
+
+  - prompt: 'is this CVE-2024-52577 affecting cluster name maria'
+    expectedOutput:
+      serverName: 'stackrox-mcp'
+      toolName: 'get_clusters_for_cve'
+      parameters:
+        cveName: 'CVE-2024-52577'
+        filterClusterId: 'maria'
+
+  - prompt: 'Is this CVE-2021-31805 affecting my clusters?'
+    expectedOutput:
+      serverName: 'stackrox-mcp'
+      toolName: 'get_clusters_for_cve'
+      parameters:
+        cveName: 'CVE-2021-31805'
+
+  - prompt: 'is this CVE-2024-52577 affecting any of my clusters defined in my list of clusters?'
+    expectedOutput:
+      serverName: 'stackrox-mcp'
+      toolName: 'get_clusters_for_cve'
+      parameters:
+        cveName: 'CVE-2024-52577'
+
+mcpServers:
+  - name: 'stackrox-mcp'
+    command: 'go'
+    args: ['run', '../cmd/stackrox-mcp/...']
+    env:
+      STACKROX_MCP__SERVER__TYPE: stdio
+      STACKROX_MCP__TOOLS__VULNERABILITY__ENABLED: "true"
+      STACKROX_MCP__TOOLS__CONFIG_MANAGER__ENABLED: "true"
+      STACKROX_MCP__CENTRAL__URL: "staging.demo.stackrox.com"
+      STACKROX_MCP__CENTRAL__AUTH_TYPE: "static"
+      STACKROX_MCP__CENTRAL__API_TOKEN: "${STACKROX_API_TOKEN}"
+      STACKROX_MCP__CENTRAL__INSECURE_SKIP_TLS_VERIFY: "true"
diff --git a/internal/toolsets/config/tools.go b/internal/toolsets/config/tools.go
@@ -69,7 +69,7 @@ func (t *listClustersTool) GetName() string {
 func (t *listClustersTool) GetTool() *mcp.Tool {
 	return &mcp.Tool{
 		Name:        t.name,
-		Description: "List all clusters managed by StackRox with their IDs, names, and types",
+		Description: "List all clusters managed by StackRox with their IDs, names, and types. Use this tool to get cluster information, or when you need to map a cluster name to its cluster ID for use in other tools.",
 		InputSchema: listClustersInputSchema(),
 	}
 }
@@ -84,11 +84,11 @@ func listClustersInputSchema() *jsonschema.Schema {
 
 	schema.Properties["offset"].Minimum = jsonschema.Ptr(0.0)
 	schema.Properties["offset"].Default = toolsets.MustJSONMarshal(defaultOffset)
-	schema.Properties["offset"].Description = "Starting index for pagination (0-based)"
+	schema.Properties["offset"].Description = "Starting index for pagination (0-based). When using pagination, always provide both offset and limit together. Default: 0."
 
 	schema.Properties["limit"].Minimum = jsonschema.Ptr(0.0)
 	schema.Properties["limit"].Default = toolsets.MustJSONMarshal(defaultLimit)
-	schema.Properties["limit"].Description = "Maximum number of clusters to return (default: 0 - unlimited)"
+	schema.Properties["limit"].Description = "Maximum number of clusters to return. Use 0 for unlimited (default). When using pagination, always provide both limit and offset together. Default: 0."
 
 	return schema
 }
diff --git a/internal/toolsets/vulnerability/clusters.go b/internal/toolsets/vulnerability/clusters.go
@@ -69,7 +69,7 @@ func (t *getClustersForCVETool) GetName() string {
 func (t *getClustersForCVETool) GetTool() *mcp.Tool {
 	return &mcp.Tool{
 		Name:        t.name,
-		Description: "Get list of clusters affected by a specific CVE",
+		Description: "Get list of clusters affected by a specific CVE. Use this tool when asking about CVE impact on 'clusters' or general CVE impact questions. For deployment/workload-specific queries, use get_deployments_for_cve instead.",
 		InputSchema: getClustersForCVEInputSchema(),
 	}
 }
@@ -87,7 +87,10 @@ func getClustersForCVEInputSchema() *jsonschema.Schema {
 	schema.Required = []string{"cveName"}
 
 	schema.Properties["cveName"].Description = "CVE name to filter clusters (e.g., CVE-2021-44228)"
-	schema.Properties["filterClusterId"].Description = "Optional cluster ID to verify if a specific cluster is affected"
+	schema.Properties["filterClusterId"].Description =
+		"Optional cluster ID or cluster name to verify if a specific cluster is affected. " +
+		"When the query mentions 'cluster name X', use this parameter with the value 'X'. " +
+		"The cluster ID can be either the actual cluster ID or the cluster name."
 
 	return schema
 }
diff --git a/internal/toolsets/vulnerability/deployments.go b/internal/toolsets/vulnerability/deployments.go
@@ -93,7 +93,7 @@ func (t *getDeploymentsForCVETool) GetName() string {
 func (t *getDeploymentsForCVETool) GetTool() *mcp.Tool {
 	return &mcp.Tool{
 		Name:        t.name,
-		Description: "Get list of deployments affected by a specific CVE",
+		Description: "Get detailed list of deployments (workloads/applications) affected by a specific CVE. Use this tool when the query specifically asks about 'workloads', 'deployments', 'applications', or needs deployment-level details. For general CVE impact or cluster-level queries, use get_clusters_for_cve instead.",
 		InputSchema: getDeploymentsForCVEInputSchema(),
 	}
 }
@@ -111,12 +111,19 @@ func getDeploymentsForCVEInputSchema() *jsonschema.Schema {
 	schema.Required = []string{"cveName"}
 
 	schema.Properties["cveName"].Description = "CVE name to filter deployments (e.g., CVE-2021-44228)"
-	schema.Properties["filterClusterId"].Description = "Optional cluster ID to filter deployments"
-	schema.Properties["filterNamespace"].Description = "Optional namespace to filter deployments"
+	schema.Properties["filterClusterId"].Description =
+		"Optional cluster ID to filter deployments. " +
+		"Use this when the query mentions a specific cluster name - you may need to call list_clusters first to get the cluster ID from the cluster name."
+	schema.Properties["filterNamespace"].Description =
+		"Optional namespace to filter deployments. " +
+		"Use this when the query mentions a specific namespace."
 
 	schema.Properties["filterPlatform"].Description =
-		fmt.Sprintf("Optional platform filter: %s=no filter, %s=user workload deployments, %s=platform deployments",
-			filterPlatformNoFilter, filterPlatformUserWorkload, filterPlatformPlatform)
+		fmt.Sprintf("Optional platform filter to distinguish deployment types: %s=no filter (default), %s=user workload deployments, %s=platform/infrastructure deployments. "+
+			"Use %s when the query specifically asks about 'workloads', 'applications', or 'user deployments'. "+
+			"Leave unset (defaults to %s) for general queries.",
+			filterPlatformNoFilter, filterPlatformUserWorkload, filterPlatformPlatform,
+			filterPlatformUserWorkload, filterPlatformNoFilter)
 	schema.Properties["filterPlatform"].Default = toolsets.MustJSONMarshal(filterPlatformNoFilter)
 	schema.Properties["filterPlatform"].Enum = []any{
 		filterPlatformNoFilter,
@@ -125,8 +132,9 @@ func getDeploymentsForCVEInputSchema() *jsonschema.Schema {
 	}
 
 	schema.Properties["includeAffectedImages"].Description =
-		"Whether to include affected image names for each deployment.\n" +
-			"WARNING: This may significantly increase response time."
+		"Whether to include affected image names for each deployment. " +
+		"Only set to true when the query specifically asks for image names or image details. " +
+		"WARNING: This may significantly increase response time. Default: false."
 	schema.Properties["includeAffectedImages"].Default = toolsets.MustJSONMarshal(false)
 
 	schema.Properties["cursor"].Description = "Cursor for next page provided by server"

Original file line number	Diff line number	Diff line change
`@@ -69,7 +69,7 @@ func (t *listClustersTool) GetName() string {`
`69`	`69`	`func (t listClustersTool) GetTool() mcp.Tool {`
`70`	`70`	`return &mcp.Tool{`
`71`	`71`	`Name: t.name,`
`72`		`- Description: "List all clusters managed by StackRox with their IDs, names, and types",`
	`72`	`+ Description: "List all clusters managed by StackRox with their IDs, names, and types. Use this tool to get cluster information, or when you need to map a cluster name to its cluster ID for use in other tools.",`
`73`	`73`	`InputSchema: listClustersInputSchema(),`
`74`	`74`	`}`
`75`	`75`	`}`
`@@ -84,11 +84,11 @@ func listClustersInputSchema() *jsonschema.Schema {`
`84`	`84`
`85`	`85`	`schema.Properties["offset"].Minimum = jsonschema.Ptr(0.0)`
`86`	`86`	`schema.Properties["offset"].Default = toolsets.MustJSONMarshal(defaultOffset)`
`87`		`- schema.Properties["offset"].Description = "Starting index for pagination (0-based)"`
	`87`	`+ schema.Properties["offset"].Description = "Starting index for pagination (0-based). When using pagination, always provide both offset and limit together. Default: 0."`
`88`	`88`
`89`	`89`	`schema.Properties["limit"].Minimum = jsonschema.Ptr(0.0)`
`90`	`90`	`schema.Properties["limit"].Default = toolsets.MustJSONMarshal(defaultLimit)`
`91`		`- schema.Properties["limit"].Description = "Maximum number of clusters to return (default: 0 - unlimited)"`
	`91`	`+ schema.Properties["limit"].Description = "Maximum number of clusters to return. Use 0 for unlimited (default). When using pagination, always provide both limit and offset together. Default: 0."`
`92`	`92`
`93`	`93`	`return schema`
`94`	`94`	`}`
Original file line number	Diff line number	Diff line change
`@@ -69,7 +69,7 @@ func (t *getClustersForCVETool) GetName() string {`
`69`	`69`	`func (t getClustersForCVETool) GetTool() mcp.Tool {`
`70`	`70`	`return &mcp.Tool{`
`71`	`71`	`Name: t.name,`
`72`		`- Description: "Get list of clusters affected by a specific CVE",`
	`72`	`+ Description: "Get list of clusters affected by a specific CVE. Use this tool when asking about CVE impact on 'clusters' or general CVE impact questions. For deployment/workload-specific queries, use get_deployments_for_cve instead.",`
`73`	`73`	`InputSchema: getClustersForCVEInputSchema(),`
`74`	`74`	`}`
`75`	`75`	`}`
`@@ -87,7 +87,10 @@ func getClustersForCVEInputSchema() *jsonschema.Schema {`
`87`	`87`	`schema.Required = []string{"cveName"}`
`88`	`88`
`89`	`89`	`schema.Properties["cveName"].Description = "CVE name to filter clusters (e.g., CVE-2021-44228)"`
`90`		`- schema.Properties["filterClusterId"].Description = "Optional cluster ID to verify if a specific cluster is affected"`
	`90`	`+ schema.Properties["filterClusterId"].Description =`
	`91`	`+ "Optional cluster ID or cluster name to verify if a specific cluster is affected. " +`
	`92`	`+ "When the query mentions 'cluster name X', use this parameter with the value 'X'. " +`
	`93`	`+ "The cluster ID can be either the actual cluster ID or the cluster name."`
`91`	`94`
`92`	`95`	`return schema`
`93`	`96`	`}`