diff --git a/.github/actions/setup-playwright/action.yml b/.github/actions/setup-playwright/action.yml new file mode 100644 index 000000000..6cb8fca39 --- /dev/null +++ b/.github/actions/setup-playwright/action.yml @@ -0,0 +1,47 @@ +name: 'Setup Playwright' +description: 'Install dependencies and setup Playwright' +inputs: + working-directory: + description: 'Directory where the project is located' + required: false + default: '.' +runs: + using: 'composite' + steps: + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libwoff1 + shell: bash + + - uses: actions/setup-node@v4 + id: setup_node + with: + node-version-file: ${{ inputs.working-directory }}/package.json + cache: npm + cache-dependency-path: ${{ inputs.working-directory }} + + - name: Install dependencies + run: npm ci + shell: bash + working-directory: ${{ inputs.working-directory }} + + - name: Cache Playwright browsers + id: cache-playwright + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles(format('{0}/package-lock.json', inputs.working-directory)) }} + restore-keys: | + ${{ runner.os }}-playwright- + + - name: Install Playwright dependencies + run: npx playwright install-deps + shell: bash + working-directory: ${{ inputs.working-directory }} + + - name: Install Playwright and browsers unless cached + run: npx playwright install --with-deps + shell: bash + working-directory: ${{ inputs.working-directory }} + if: steps.cache-playwright.outputs.cache-hit != 'true' diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index 573672cc8..59474b80b 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -2,49 +2,21 @@ name: Playwright Tests on: push: - branches: [main] + branches: [main, e2e-tests-everything-server] pull_request: branches: [main] jobs: test: - # Installing Playright dependencies can take quite awhile, and also depends on GitHub CI load. + # Installing Playwright dependencies can take quite awhile, and also depends on GitHub CI load. timeout-minutes: 15 runs-on: ubuntu-latest steps: - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libwoff1 - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - id: setup_node - with: - node-version-file: package.json - cache: npm - - # Cache Playwright browsers - - name: Cache Playwright browsers - id: cache-playwright - uses: actions/cache@v4 - with: - path: ~/.cache/ms-playwright # The default Playwright cache path - key: ${{ runner.os }}-playwright-${{ hashFiles('package-lock.json') }} # Cache key based on OS and package-lock.json - restore-keys: | - ${{ runner.os }}-playwright- - - - name: Install dependencies - run: npm ci - - - name: Install Playwright dependencies - run: npx playwright install-deps - - - name: Install Playwright and browsers unless cached - run: npx playwright install --with-deps - if: steps.cache-playwright.outputs.cache-hit != 'true' + # FIXME: Explicit path through .github is really necessary? + - uses: ./.github/actions/setup-playwright - name: Run Playwright tests id: playwright-tests @@ -70,9 +42,93 @@ jobs: comment-title: "🎭 Playwright E2E Test Results" job-summary: true icon-style: "emojis" + # FIXME: Node version should come from the composite setup-playwright action custom-info: | **Test Environment:** Ubuntu Latest, Node.js ${{ steps.setup_node.outputs.node-version }} **Browsers:** Chromium, Firefox 📊 [View Detailed HTML Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) (download artifacts) test-command: "npm run test:e2e" + + test-everything-server: + if: github.ref == 'refs/heads/e2e-tests-everything-server' + timeout-minutes: 15 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + path: inspector + + - uses: ./inspector/.github/actions/setup-playwright + with: + working-directory: inspector + + - uses: actions/checkout@v4 + with: + repository: richardkmichael/servers + ref: add-structured-content-tool + path: servers + + - name: Install and build everything server + working-directory: servers/src/everything + run: | + npm clean-install + npm run prepare + + - name: Run Everything Server Playwright tests + working-directory: inspector + run: npm run test:e2e:everything + env: + EVERYTHING_SERVER_PATH: ${{ github.workspace }}/servers/src/everything + + # FIXME: + # + # Revert to: always(), GitHub Actions are difficult to control with clarity. + # + # This step needs to run when the playwright-tests step ran whatsoever. These step conclusions + # seem to be: success, fail, cancelled (GitHub Action timeout; not Playwright timeout-- + # that's fail). + # + # But not if the setup action didn't finish. For example, job timeout -- which is `cancelled()`: + # https://github.com/modelcontextprotocol/inspector/actions/runs/15860838530/job/44717383789#step:7:1152 + # + # But yes, if the playwright-tests step caused the timeout -- `cancelled()`. + # + # It seems we need to know which *step* caused the overall job to be cancelled. The + # expression functions: success(), failure(), cancelled(), etc. refer to the *overall* + # job. + # + # Also, there is an implicit `success()` even with an `if: `, i.e., `if: success() && + # https://docs.github.com/en/actions/reference/evaluate-expressions-in-workflows-and-actions#failure-with-conditions + # + # Consider: cancelled() && step.[NAME].conclusion == "ANY" + # + - name: Upload Everything Server Playwright Report and Screenshots + uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report-everything-server + path: | + inspector/client/report-everything/ + inspector/client/report-everything.json + inspector/client/report-everything.xml + inspector/client/e2e/results-everything/ + retention-days: 2 + +# - name: Publish Everything Server Playwright Test Summary +# uses: daun/playwright-report-summary@v3 +# if: always() +# with: +# create-comment: ${{ github.event.pull_request.head.repo.full_name == github.repository }} +# report-file: client/results-everything.json +# comment-title: "🎭 Everything Server E2E Test Results" +# job-summary: true +# icon-style: "emojis" +# custom-info: | +# **Test Environment:** Ubuntu Latest, Node.js 20 +# **Browsers:** Chromium +# **MCP Server:** Everything Server + +# 📊 [View Detailed HTML Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) (download artifacts) +# test-command: "npm run test:e2e:everything" diff --git a/.gitignore b/.gitignore index 8e184de89..120944dfd 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ test-output client/playwright-report/ client/results.json client/test-results/ +client/e2e/test-results-everything/ diff --git a/CLAUDE.md b/CLAUDE.md index ec826b1fd..0ade07433 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,14 @@ # MCP Inspector Development Guide +## Exploring the Inspector yourself + +- The MCP Inspector is a React app in the client/ directory +- The React app might be running at http://localhost:6274, if so you can use a browser tool to explore the UI +- Most React app functionality requires a connected MCP Server to test + - If not already filled, first fill in the form values to configure the MCP Server to run + - Use the `Connect` button in the UI to connect to the MCP Server +- Explore the running Inspector UI to learn the DOM structure and develop Playwright e2e tests + ## Build Commands - Build all: `npm run build` diff --git a/client/.gitignore b/client/.gitignore index a547bf36d..23b08fea4 100644 --- a/client/.gitignore +++ b/client/.gitignore @@ -22,3 +22,9 @@ dist-ssr *.njsproj *.sln *.sw? + +# Playwright output, see: playwright.everything.config.ts +report-everything/ +report-everything.json +report-everything.xml +e2e/results-everything/ diff --git a/client/e2e/everything-server.spec.ts b/client/e2e/everything-server.spec.ts new file mode 100644 index 000000000..d9470f2a4 --- /dev/null +++ b/client/e2e/everything-server.spec.ts @@ -0,0 +1,206 @@ +import { test, expect } from "@playwright/test"; + +test.describe("Everything Server Integration", () => { + test.skip(!process.env.EVERYTHING_SERVER_PATH, 'EVERYTHING_SERVER_PATH environment variable is required'); + + let page; + + test.beforeAll(async ({ browser, baseURL }) => { + page = await browser.newPage(); + + page.on('requestfailed', request => { + const errorText = request.failure()?.errorText; + const url = request.url(); + + // Expected: MCP connection abort during disconnect + if (errorText === 'net::ERR_ABORTED' && url.includes('/stdio')) { + console.log(`Expected at Disconnect: ${request._guid} (${errorText})`); + return; + } + + // console.log(`REQ FAILED: ${request._guid}`); + // console.log(` REQ FAILED ${request._guid}: ${errorText}`); + // console.log(` REQ FAILED ${request._guid}: ${request.method()}`); + // console.log(` REQ FAILED ${request._guid}: ${url}`); + }); + + // page.on('request', async request => { + // console.log(`REQUEST: ${request._guid}`); + // console.log(` REQ METHOD ${request._guid}: ${request.method()}`); + // console.log(` REQ URL ${request._guid}: ${request.url()}`); + // console.log(` REQ HEADERS ${request._guid}: ${JSON.stringify(await request.allHeaders(), null, 2)}`); + // console.log(` REQ POST DATA ${request._guid}: ${request.postData()}`); + // }); + + // page.on('response', response => { + // console.log(`RESPONSE ${response.request()._guid}:`); + // console.log(` RESP STATUS ${response.request()._guid}: ${response.status()}`); + // console.log(` RESP URL ${response.request()._guid}: ${response.url()}`); + // console.log(` BODY: ${await response.body()}`); + // }); + + + await page.goto(baseURL); + await expect(page.getByLabel("Transport Type")).toBeVisible(); + + // Fill in the form to connect to the everything server + const everythingServerPath = process.env.EVERYTHING_SERVER_PATH!; + + // Ensure Transport Type is set to STDIO + const transportTypeCombo = page.getByLabel("Transport Type"); + await transportTypeCombo.click(); + // Select STDIO from the dropdown options (not the label) + await page.locator('[role="option"]').getByText("STDIO").click(); + + // Fill in the command field + const commandInput = page.getByRole('textbox', { name: 'Command' }); + await commandInput.fill('npm'); + + // Fill in the arguments field with the everything server startup command + const argumentsInput = page.getByRole('textbox', { name: 'Arguments' }); + await argumentsInput.fill(`--prefix ${everythingServerPath} --loglevel silent run start`); + + const connectButton = page.locator('button:text-is("Connect")'); + await expect(connectButton).toBeVisible({ timeout: 10000 }); + await connectButton.click(); + + await page.waitForTimeout(250); + + const connectedIndicator = page.locator('text="Connected"'); + const greenCircle = page.locator('.bg-green-500'); + await expect(connectedIndicator).toBeVisible({ timeout: 10000 }); + await expect(greenCircle).toBeVisible({ timeout: 10000 }); + }); + + test.afterAll(async () => { + if (page) { + const disconnectButton = page.locator('button:text-is("Disconnect")'); + if (await disconnectButton.isVisible({ timeout: 500 })) { + await disconnectButton.click(); + await page.waitForTimeout(250); + } + + await page.close(); + } + }); + + test("should list tools from the everything server", async () => { + const toolsTab = page.getByRole("tab", { name: "Tools" }); + await toolsTab.click(); + + const toolsTabPanel = page.getByRole('tabpanel', { name: 'Tools' }); + const listToolsButton = toolsTabPanel.getByRole("button", { name: "List Tools" }); + await expect(listToolsButton).toBeVisible({ timeout: 500 }); + await listToolsButton.click(); + + const knownTools = ['echo', 'add', 'longRunningOperation', 'printEnv', 'sampleLLM']; + let foundToolsCount = 0; + + for (const toolName of knownTools) { + const toolElement = toolsTabPanel.locator(`text="${toolName}"`).first(); + await expect(toolElement).toBeVisible({ timeout: 2000 }); + foundToolsCount++; + } + + expect(foundToolsCount).toBeGreaterThanOrEqual(3); + }); + + test.describe("structuredContent tool", () => { + test("should display tool details when selected", async () => { + const toolsTab = page.getByRole("tab", { name: "Tools" }); + await toolsTab.click(); + + const toolsTabPanel = page.getByRole('tabpanel', { name: 'Tools' }); + const structuredContentTool = toolsTabPanel.locator('text="structuredContent"').first(); + await expect(structuredContentTool).toBeVisible({ timeout: 2000 }); + await structuredContentTool.click(); + + const toolHeader = page.getByRole('heading', { name: 'structuredContent' }); + await expect(toolHeader).toBeVisible(); + }); + + test("should show error when executing structuredContent without required inputs", async () => { + const toolsTab = page.getByRole("tab", { name: "Tools" }); + await toolsTab.click(); + + const toolsTabPanel = page.getByRole('tabpanel', { name: 'Tools' }); + const structuredContentTool = toolsTabPanel.locator('text="structuredContent"').first(); + await expect(structuredContentTool).toBeVisible({ timeout: 2000 }); + await structuredContentTool.click(); + + const runButton = page.getByRole("button", { name: "Run Tool" }); + await expect(runButton).toBeVisible({ timeout: 500 }); + await runButton.click(); + + const errorResultHeader = page.locator('h4').filter({ hasText: 'Tool Result: Error' }); + await expect(errorResultHeader).toBeVisible({ timeout: 500 }); + + // Scope error message to the tool result area by using the error header as context + const errorMessage = errorResultHeader.locator('..').getByText(/MCP error -32603/); + await expect(errorMessage).toBeVisible({ timeout: 500 }); + }); + + test("should execute structuredContent tool with proper inputs and show structured and unstructured results", async () => { + const toolsTab = page.getByRole("tab", { name: "Tools" }); + await toolsTab.click(); + + const toolsTabPanel = page.getByRole('tabpanel', { name: 'Tools' }); + const structuredContentTool = toolsTabPanel.locator('text="structuredContent"').first(); + await expect(structuredContentTool).toBeVisible({ timeout: 2000 }); + await structuredContentTool.click(); + + const locationInput = page.getByRole('textbox', { name: 'location' }); + await expect(locationInput).toBeVisible({ timeout: 500 }); + await locationInput.fill("San Francisco"); + + const runButton = page.getByRole("button", { name: "Run Tool" }); + await expect(runButton).toBeVisible({ timeout: 500 }); + await runButton.click(); + + const successResultHeader = page.locator('h4').filter({ hasText: 'Tool Result: Success' }); + await expect(successResultHeader).toBeVisible({ timeout: 500 }); + + const structuredContentHeader = page.getByRole('heading', { name: 'Structured Content:', exact: true }); + await expect(structuredContentHeader).toBeVisible({ timeout: 500 }); + + const unstructuredContentHeader = page.getByRole('heading', { name: 'Unstructured Content:', exact: true }); + await expect(unstructuredContentHeader).toBeVisible({ timeout: 500 }); + + const schemaValidationMessage = page.locator('text="✓ Valid according to output schema"'); + await expect(schemaValidationMessage).toBeVisible({ timeout: 500 }); + + const contentMatchingMessage = page.locator('text="✓ Unstructured content matches structured content"'); + await expect(contentMatchingMessage).toBeVisible({ timeout: 500 }); + }); + + test("should show error when input is cleared after successful execution", async () => { + const toolsTab = page.getByRole("tab", { name: "Tools" }); + await toolsTab.click(); + + const toolsTabPanel = page.getByRole('tabpanel', { name: 'Tools' }); + const structuredContentTool = toolsTabPanel.locator('text="structuredContent"').first(); + await expect(structuredContentTool).toBeVisible({ timeout: 2000 }); + await structuredContentTool.click(); + + const locationInput = page.getByRole('textbox', { name: 'location' }); + await expect(locationInput).toBeVisible({ timeout: 500 }); + await locationInput.fill("New York"); + + const runButton = page.getByRole("button", { name: "Run Tool" }); + await expect(runButton).toBeVisible({ timeout: 500 }); + await runButton.click(); + + const successResultHeader = page.locator('h4').filter({ hasText: 'Tool Result: Success' }); + await expect(successResultHeader).toBeVisible({ timeout: 500 }); + + await locationInput.fill(""); + await runButton.click(); + + const errorResultHeader = page.locator('h4').filter({ hasText: 'Tool Result: Error' }); + await expect(errorResultHeader).toBeVisible({ timeout: 500 }); + + const errorMessage = errorResultHeader.locator('..').getByText(/MCP error -32603/); + await expect(errorMessage).toBeVisible({ timeout: 500 }); + }); + }); +}); diff --git a/client/package.json b/client/package.json index 59bf2288c..29f59a7dc 100644 --- a/client/package.json +++ b/client/package.json @@ -22,6 +22,7 @@ "test": "jest --config jest.config.cjs", "test:watch": "jest --config jest.config.cjs --watch", "test:e2e": "playwright test e2e && npm run cleanup:e2e", + "test:e2e:everything": "playwright test --config=playwright.everything.config.ts e2e/everything-server.spec.ts && npm run cleanup:e2e", "cleanup:e2e": "node e2e/global-teardown.js" }, "dependencies": { diff --git a/client/playwright.config.ts b/client/playwright.config.ts index 570dd054e..dee530f4c 100644 --- a/client/playwright.config.ts +++ b/client/playwright.config.ts @@ -1,5 +1,7 @@ import { defineConfig, devices } from "@playwright/test"; +const inspectorUrl = "http://localhost:6274" + /** * @see https://playwright.dev/docs/test-configuration */ @@ -8,11 +10,16 @@ export default defineConfig({ webServer: { cwd: "..", command: "npm run dev", - url: "http://localhost:6274", + url: inspectorUrl, reuseExistingServer: !process.env.CI, + stdout: "pipe", + env: { + MCP_AUTO_OPEN_ENABLED: false, + } }, testDir: "./e2e", + testIgnore: "**/everything-server.spec.ts", outputDir: "./e2e/test-results", /* Run tests in files in parallel */ fullyParallel: true, @@ -33,7 +40,7 @@ export default defineConfig({ /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: "http://localhost:6274", + baseURL: inspectorUrl, /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: "on-first-retry", diff --git a/client/playwright.everything.config.ts b/client/playwright.everything.config.ts new file mode 100644 index 000000000..471e2d988 --- /dev/null +++ b/client/playwright.everything.config.ts @@ -0,0 +1,128 @@ +import { defineConfig, devices } from "@playwright/test"; + +/** + * Avoid the default ports (6274 and 6277), to allow the Inspector to be running + * while Playwright tests run. +*/ +const CLIENT_PORT = 7274; +const SERVER_PORT = 7277; + +// FIXME: +// MCP_PROXY_PORT is required due to the client not being implicitly aware of `SERVER_PORT`; bug. +// Otherwise, need to process the *output* of `npm run dev`. +// Same with the AUTH token, hence disabling below. +// +const INSPECTOR_URL = `http://localhost:${CLIENT_PORT}/?MCP_PROXY_PORT=${SERVER_PORT}`; + +/** + * Playwright configuration for testing with the "everything" MCP server + * @see https://playwright.dev/docs/test-configuration + */ +export default defineConfig({ + /* Run the Inspector with the "everything" server before starting tests */ + webServer: { + cwd: "..", + // FIXME: + // - exit much earlier unless EVERYTHING_SERVER_PATH is defined; and, rename this var + // - npm run prod ? + // - noisy output in CI output, silence startup and request logging? + // - this works because `npm ci` (in GitHub Action) auto-invokes `npm prepare`, which is configured to `npm run build`. + // - `npm ci` lifecycle -- https://docs.npmjs.com/cli/v11/using-npm/scripts#life-cycle-operation-order + command: "npm run dev", + env: { + // FIXME: How to get the auth token from command stdout to Playwright to use as a URL param? + DANGEROUSLY_OMIT_AUTH: true, + + CLIENT_PORT: CLIENT_PORT, + SERVER_PORT: SERVER_PORT, + + // Disable auto-open. Otherwise, when local, running the tests will open the Inspector in your current browser. + // FIXME: This env wasn't inherited from the `npm run` script in `package.json`; why? + MCP_AUTO_OPEN_ENABLED: "false", + + // FIXME: Has no effect? Prob overrides the generated one, but still requires round-trip via query param. + // MCP_PROXY_FULL_ADDRESS: `http://localhost:${SERVER_PORT}`, + }, + + url: INSPECTOR_URL, + + // This would allow Playwright to use an existing running server. + // + // For example, an already running dev server in a local dev environment. + // + // These tests are for a specific MCP Server, which may not be the one running in the local + // server, so probably this isn't going to be useful. Currently starting on different ports + // anyway. + // + // If doing so, first investigate: + // - would doing so disrupt any in-progress work in that dev server? e.g., modified state (CSS, DOM, etc.) + // - would this affect the localStorage? + // - perhaps Playwright could use an anonymous/private tab? + // - headful and reuseExistingServer might cause Playwright to re-use localStorage, + // and the inspector stores settings in localstorage, so fiddling with auth and settings + // might be a problem *between* test runs + + reuseExistingServer: false, // !process.env.CI, + + timeout: 30 * 1000, // 30 seconds timeout for server startup + + // Wait for specific text that indicates the server is ready + // You may need to adjust this based on the actual startup output + stdout: "pipe", // default is ignore + // stderr: "pipe", // default is pipe + }, + + testDir: "./e2e", + outputDir: "./e2e/results-everything", + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + + /* FIXME: Revisit when there are more tests, offering parallel gains; adds complexity. */ + /* Opt out of parallel tests. */ + workers: 1, + /* Run tests in files in parallel */ + /* fullyParallel: true, */ + + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: process.env.CI + ? [ + ["line"], + ["html", { outputFolder: "report-everything" }], + ["json", { outputFile: "report-everything.json" }], + ] + : [ + ["line"], + ["html", { outputFolder: "report-everything", open: "never" }], // Generate trace viewer locally + ], + + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: INSPECTOR_URL, + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: "on-first-retry", + + /* Take screenshots on failure */ + screenshot: "only-on-failure", + + /* Record video on failure */ + video: "retain-on-failure", + + /* Action timeout */ + actionTimeout: 10000, + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: "chromium", + use: { + ...devices["Desktop Chrome"], + headless: true, + }, + }, + ], +}); diff --git a/package.json b/package.json index 5d5fe52fc..ff526fc37 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "start-client": "cd client && npm run preview", "test": "npm run prettier-check && cd client && npm test", "test-cli": "cd cli && npm run test", - "test:e2e": "MCP_AUTO_OPEN_ENABLED=false npm run test:e2e --workspace=client", + "test:e2e": "npm run test:e2e --workspace=client", + "test:e2e:everything": "npm run test:e2e:everything --workspace=client", "prettier-fix": "prettier --write .", "prettier-check": "prettier --check .", "lint": "prettier --check . && cd client && npm run lint",