Skip to content

Commit 98bebab

Browse files
SamMorrowDrumsSamMorrowDrums
committed
ci: use mcp-conformance-action for conformance tests
Replace custom script/conformance-test with the reusable mcp-conformance-action GitHub Action. This provides: - Standardized MCP conformance testing across all MCP servers - Automatic comparison between PR branch and merge-base - Support for multiple configurations with different flags - Custom message support for dynamic toolset testing - Detailed conformance reports with timing comparisons - Artifact upload for test results All 16 original test configurations are preserved including the dynamic tool call tests for toolset management. Action: https://github.com/marketplace/actions/mcp-conformance-test Related: #1826
1 parent a663a6b commit 98bebab

File tree

1 file changed

+45
-46
lines changed

1 file changed

+45
-46
lines changed

‎.github/workflows/conformance.yml‎

Lines changed: 45 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -12,58 +12,57 @@ jobs:
1212

1313
steps:
1414
- name: Check out code
15-
uses: actions/checkout@v6
15+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
1616
with:
17-
# Fetch full history to access merge-base
1817
fetch-depth: 0
1918

20-
- name: Set up Go
21-
uses: actions/setup-go@v6
19+
- name: Run MCP Conformance Test
20+
uses: SamMorrowDrums/mcp-conformance-action@v1
2221
with:
23-
go-version-file: "go.mod"
22+
setup_go: "true"
23+
go_version_file: go.mod
24+
install_command: go mod download
25+
server_command: go run ./cmd/github-mcp-server stdio
26+
server_env: |
27+
GITHUB_PERSONAL_ACCESS_TOKEN=test-token
28+
configurations: |
29+
[
30+
{"name": "default", "args": ""},
31+
{"name": "read-only", "args": "--read-only"},
32+
{"name": "dynamic-toolsets", "args": "--dynamic-toolsets"},
33+
{"name": "read-only+dynamic", "args": "--read-only --dynamic-toolsets"},
34+
{"name": "toolsets-repos", "args": "--toolsets=repos"},
35+
{"name": "toolsets-issues", "args": "--toolsets=issues"},
36+
{"name": "toolsets-pull_requests", "args": "--toolsets=pull_requests"},
37+
{"name": "toolsets-repos,issues", "args": "--toolsets=repos,issues"},
38+
{"name": "toolsets-all", "args": "--toolsets=all"},
39+
{"name": "tools-get_me", "args": "--tools=get_me"},
40+
{"name": "tools-get_me,list_issues", "args": "--tools=get_me,list_issues"},
41+
{"name": "toolsets-repos+read-only", "args": "--toolsets=repos --read-only"},
42+
{"name": "toolsets-all+dynamic", "args": "--toolsets=all --dynamic-toolsets"},
43+
{"name": "toolsets-repos+dynamic", "args": "--toolsets=repos --dynamic-toolsets"},
44+
{"name": "toolsets-repos,issues+dynamic", "args": "--toolsets=repos,issues --dynamic-toolsets"},
45+
{
46+
"name": "dynamic-tool-calls",
47+
"args": "--dynamic-toolsets",
48+
"custom_messages": [
49+
{"id": 10, "name": "list_toolsets_before", "message": {"jsonrpc": "2.0", "id": 10, "method": "tools/call", "params": {"name": "list_available_toolsets", "arguments": {}}}},
50+
{"id": 11, "name": "get_toolset_tools", "message": {"jsonrpc": "2.0", "id": 11, "method": "tools/call", "params": {"name": "get_toolset_tools", "arguments": {"toolset": "repos"}}}},
51+
{"id": 12, "name": "enable_toolset", "message": {"jsonrpc": "2.0", "id": 12, "method": "tools/call", "params": {"name": "enable_toolset", "arguments": {"toolset": "repos"}}}},
52+
{"id": 13, "name": "list_toolsets_after", "message": {"jsonrpc": "2.0", "id": 13, "method": "tools/call", "params": {"name": "list_available_toolsets", "arguments": {}}}}
53+
]
54+
}
55+
]
2456
25-
- name: Download dependencies
26-
run: go mod download
27-
28-
- name: Run conformance test
29-
id: conformance
30-
run: |
31-
# Run conformance test, capture stdout for summary
32-
script/conformance-test > conformance-summary.txt 2>&1 || true
33-
34-
# Output the summary
35-
cat conformance-summary.txt
36-
37-
# Check result
38-
if grep -q "RESULT: ALL TESTS PASSED" conformance-summary.txt; then
39-
echo "status=passed" >> $GITHUB_OUTPUT
40-
else
41-
echo "status=differences" >> $GITHUB_OUTPUT
42-
fi
43-
44-
- name: Generate Job Summary
57+
- name: Add interpretation note
58+
if: always()
4559
run: |
46-
# Add the full markdown report to the job summary
47-
echo "# MCP Server Conformance Report" >> $GITHUB_STEP_SUMMARY
48-
echo "" >> $GITHUB_STEP_SUMMARY
49-
echo "Comparing PR branch against merge-base with \`origin/main\`" >> $GITHUB_STEP_SUMMARY
50-
echo "" >> $GITHUB_STEP_SUMMARY
51-
52-
# Extract and append the report content (skip the header since we added our own)
53-
tail -n +5 conformance-report/CONFORMANCE_REPORT.md >> $GITHUB_STEP_SUMMARY
54-
5560
echo "" >> $GITHUB_STEP_SUMMARY
5661
echo "---" >> $GITHUB_STEP_SUMMARY
5762
echo "" >> $GITHUB_STEP_SUMMARY
58-
59-
# Add interpretation note
60-
if [ "${{ steps.conformance.outputs.status }}" = "passed" ]; then
61-
echo "✅ **All conformance tests passed** - No behavioral differences detected." >> $GITHUB_STEP_SUMMARY
62-
else
63-
echo "⚠️ **Differences detected** - Review the diffs above to ensure changes are intentional." >> $GITHUB_STEP_SUMMARY
64-
echo "" >> $GITHUB_STEP_SUMMARY
65-
echo "Common expected differences:" >> $GITHUB_STEP_SUMMARY
66-
echo "- New tools/toolsets added" >> $GITHUB_STEP_SUMMARY
67-
echo "- Tool descriptions updated" >> $GITHUB_STEP_SUMMARY
68-
echo "- Capability changes (intentional improvements)" >> $GITHUB_STEP_SUMMARY
69-
fi
63+
echo "ℹ️ **Note:** Differences may be intentional improvements." >> $GITHUB_STEP_SUMMARY
64+
echo "" >> $GITHUB_STEP_SUMMARY
65+
echo "Common expected differences:" >> $GITHUB_STEP_SUMMARY
66+
echo "- New tools/toolsets added" >> $GITHUB_STEP_SUMMARY
67+
echo "- Tool descriptions updated" >> $GITHUB_STEP_SUMMARY
68+
echo "- Capability changes (intentional improvements)" >> $GITHUB_STEP_SUMMARY

0 commit comments

Comments
 (0)