diff --git a/mcp-worker/README.md b/mcp-worker/README.md index d080e3c5..41595b59 100644 --- a/mcp-worker/README.md +++ b/mcp-worker/README.md @@ -133,3 +133,27 @@ When a user completes OAuth on the hosted MCP Worker, the worker emits a single - **Channel**: `${orgId}-mcp-install` - **Event name**: `mcp-install` + +## MCP Tool Token Counts + +Measure how many AI tokens our MCP tool descriptions and schemas consume. Run the measurement script (from repo root): + +```bash +yarn install && +yarn measure:mcp-tokens +``` + +What it does: + +- Uses `gpt-tokenizer` (OpenAI-style) and `@anthropic-ai/tokenizer` (Anthropic) to count tokens in each tool's `description` and `inputSchema`. + +Current token totals: + +```json +{ + "totals": { + "anthropic": 10428, + "openai": 10746 + } +} +``` diff --git a/mcp-worker/src/projectSelectionTools.ts b/mcp-worker/src/projectSelectionTools.ts index 5c233ca5..cc512872 100644 --- a/mcp-worker/src/projectSelectionTools.ts +++ b/mcp-worker/src/projectSelectionTools.ts @@ -17,7 +17,10 @@ export const SelectProjectArgsSchema = z.object({ .string() .optional() .describe( - 'The project key to select. If not provided, will list all available projects to choose from.', + [ + 'The project key to select.', + 'If not provided, will list all available projects to choose from.', + ].join('\n'), ), }) @@ -111,6 +114,7 @@ export function registerProjectSelectionTools( 'Select a project to use for subsequent MCP operations.', 'Call without parameters to list available projects.', 'Do not automatically select a project, ask the user which project they want to select.', + 'Returns the current project, its environments, and SDK keys.', 'Include dashboard link in the response.', ].join('\n'), annotations: { diff --git a/package.json b/package.json index 9875fbde..0c9ba064 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "deploy:worker:prod": "cd mcp-worker && yarn deploy:prod", "deploy:worker:dev": "cd mcp-worker && yarn deploy:dev", "dev:worker": "cd mcp-worker && yarn dev", + "measure:mcp-tokens": "ts-node scripts/measureMcpTokens.ts", "format": "prettier --write \"src/**/*.{ts,js,json}\" \"test/**/*.{ts,js,json}\" \"test-utils/**/*.{ts,js,json}\" \"*.{ts,js,json,md}\"", "format:check": "prettier --check \"src/**/*.{ts,js,json}\" \"test/**/*.{ts,js,json}\" \"test-utils/**/*.{ts,js,json}\" \"*.{ts,js,json,md}\"", "lint": "eslint . --config eslint.config.mjs", @@ -67,6 +68,7 @@ "zod": "~3.25.76" }, "devDependencies": { + "@anthropic-ai/tokenizer": "^0.0.4", "@babel/code-frame": "^7.27.1", "@babel/core": "^7.28.0", "@babel/generator": "^7.28.0", @@ -85,6 +87,7 @@ "chai": "^5.1.2", "eslint": "^9.18.0", "eslint-config-prettier": "^9.1.0", + "gpt-tokenizer": "^3.0.1", "mocha": "^10.8.2", "mocha-chai-jest-snapshot": "^1.1.6", "nock": "^13.5.6", diff --git a/scripts/measureMcpTokens.ts b/scripts/measureMcpTokens.ts new file mode 100644 index 00000000..c1c31f45 --- /dev/null +++ b/scripts/measureMcpTokens.ts @@ -0,0 +1,135 @@ +#!/usr/bin/env ts-node +import { registerAllToolsWithServer } from '../src/mcp/tools' +import type { DevCycleMCPServerInstance } from '../src/mcp/server' +import type { IDevCycleApiClient } from '../src/mcp/api/interface' + +type Collected = { + name: string + description: string + inputSchema?: unknown + outputSchema?: unknown +} + +const collected: Collected[] = [] + +const mockServer: DevCycleMCPServerInstance = { + registerToolWithErrorHandling(name, config) { + collected.push({ + name, + description: config.description, + inputSchema: config.inputSchema, + outputSchema: config.outputSchema, + }) + }, +} + +// We do not need a real client to collect tool metadata +const fakeClient = {} as unknown as IDevCycleApiClient + +registerAllToolsWithServer(mockServer, fakeClient) + +let openAiEncoderPromise: Promise<(input: string) => number[]> | undefined +async function countOpenAI(text: string): Promise { + try { + if (!openAiEncoderPromise) { + openAiEncoderPromise = import('gpt-tokenizer').then((m) => m.encode) + } + const encode = await openAiEncoderPromise + return encode(text).length + } catch { + return 0 + } +} +let anthropicCounterPromise: Promise<(input: string) => number> | undefined +async function countAnthropic(text: string): Promise { + try { + if (!anthropicCounterPromise) { + anthropicCounterPromise = import('@anthropic-ai/tokenizer').then( + (m) => m.countTokens, + ) + } + const countTokens = await anthropicCounterPromise + return countTokens(text) + } catch { + return 0 + } +} + +type ResultRow = { + name: string + anthropic: { + description: number + inputSchema: number + outputSchema: number + total: number + } + openai: { + description: number + inputSchema: number + outputSchema: number + total: number + } +} + +const rows: ResultRow[] = [] +let grandAnthropic = 0 +let grandOpenAI = 0 + +async function main() { + for (const t of collected) { + const d = t.description ?? '' + const i = t.inputSchema ? JSON.stringify(t.inputSchema) : '' + const o = t.outputSchema ? JSON.stringify(t.outputSchema) : '' + + const [aDesc, aIn, aOut] = await Promise.all([ + countAnthropic(d), + i ? countAnthropic(i) : Promise.resolve(0), + o ? countAnthropic(o) : Promise.resolve(0), + ]) + const aTotal = aDesc + aIn + aOut + + const [oDesc, oIn, oOut] = await Promise.all([ + countOpenAI(d), + i ? countOpenAI(i) : Promise.resolve(0), + o ? countOpenAI(o) : Promise.resolve(0), + ]) + const oTotal = oDesc + oIn + oOut + + grandAnthropic += aTotal + grandOpenAI += oTotal + + rows.push({ + name: t.name, + anthropic: { + description: aDesc, + inputSchema: aIn, + outputSchema: aOut, + total: aTotal, + }, + openai: { + description: oDesc, + inputSchema: oIn, + outputSchema: oOut, + total: oTotal, + }, + }) + } + + rows.sort((a, b) => a.name.localeCompare(b.name)) + + console.log( + JSON.stringify( + { + tools: rows, + totals: { anthropic: grandAnthropic, openai: grandOpenAI }, + }, + null, + 2, + ), + ) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/src/mcp/tools/customPropertiesTools.ts b/src/mcp/tools/customPropertiesTools.ts index 0eadaa1e..c49fe094 100644 --- a/src/mcp/tools/customPropertiesTools.ts +++ b/src/mcp/tools/customPropertiesTools.ts @@ -147,6 +147,7 @@ export function registerCustomPropertiesTools( { description: [ 'Create a new custom property.', + 'Custom properties are used in feature targeting audiences as custom user-data definitions.', 'Include dashboard link in the response.', ].join('\n'), annotations: { diff --git a/src/mcp/tools/featureTools.ts b/src/mcp/tools/featureTools.ts index 49f9602b..81a55e17 100644 --- a/src/mcp/tools/featureTools.ts +++ b/src/mcp/tools/featureTools.ts @@ -457,7 +457,9 @@ export function registerFeatureTools( 'create_feature', { description: [ - 'Create a new feature flag. Include dashboard link in the response.', + 'Create a new DevCycle feature. Include dashboard link in the response.', + 'Features are the main logical container for variables and targeting rules, defining what values variables will be served to users across environments.', + 'Features can contin multiple variables, and many variations, defined by the targeting rules to determine how variable values are distributed to users.', 'If a user is creating a feature, you should follow these steps and ask users for input on these steps:', '1. create a variable and associate it with this feature. (default to creating a "boolean" variable with the same key as the feature)', '2. create variations for the feature. (default to creating an "on" and "off" variation)', @@ -499,7 +501,7 @@ export function registerFeatureTools( 'update_feature_status', { description: [ - 'Update the status of an existing feature flag.', + 'Update the status of an existing feature.', '⚠️ IMPORTANT: Changes to feature status may affect production environments.', 'Always confirm with the user before making changes to features that are active in production.', 'Include dashboard link in the response.', @@ -520,13 +522,13 @@ export function registerFeatureTools( 'delete_feature', { description: [ - 'Delete an existing feature flag.', - '⚠️ CRITICAL: Deleting a feature flag will remove it from ALL environments including production.', - 'ALWAYS confirm with the user before deleting any feature flag.', + 'Delete an existing feature.', + '⚠️ CRITICAL: Deleting a feature will remove it from ALL environments including production.', + 'ALWAYS confirm with the user before deleting any feature.', 'Include dashboard link in the response.', ].join('\n'), annotations: { - title: 'Delete Feature Flag', + title: 'Delete Feature', destructiveHint: true, }, inputSchema: DeleteFeatureArgsSchema.shape, @@ -657,7 +659,8 @@ export function registerFeatureTools( 'get_feature_audit_log_history', { description: [ - 'Get feature flag audit log history from DevCycle.', + 'Get feature audit log history from DevCycle.', + 'Returns audit log data for all changes made to a feature / variation / targeting rule ordered by date.', 'Include dashboard link in the response.', ].join('\n'), annotations: { diff --git a/src/mcp/tools/installTools.ts b/src/mcp/tools/installTools.ts index 5bb0b403..e0bf47cb 100644 --- a/src/mcp/tools/installTools.ts +++ b/src/mcp/tools/installTools.ts @@ -46,6 +46,7 @@ export function registerInstallTools( { description: [ 'Fetch DevCycle SDK installation instructions, and follow the instructions to install the DevCycle SDK.', + 'Also includes documentation and examples for using DevCycle SDK in your application.', "Choose the guide that matches the application's language/framework.", ].join('\n'), annotations: { diff --git a/src/mcp/tools/localProjectTools.ts b/src/mcp/tools/localProjectTools.ts index 7cf947ad..789d6e49 100644 --- a/src/mcp/tools/localProjectTools.ts +++ b/src/mcp/tools/localProjectTools.ts @@ -18,7 +18,10 @@ export const SelectProjectArgsSchema = z.object({ .string() .optional() .describe( - 'The project key to select. If not provided, will list all available projects to choose from.', + [ + 'The project key to select.', + 'If not provided, will list all available projects to choose from.', + ].join('\n'), ), }) @@ -113,7 +116,8 @@ export function registerLocalProjectTools( 'Select a project to use for subsequent MCP operations.', 'Call without parameters to list available projects.', 'Do not automatically select a project, ask the user which project they want to select.', - 'This will update your local DevCycle configuration (~/.config/devcycle/user.yml).', + 'This will update your local DevCycle configuration for the MCP and CLI (~/.config/devcycle/user.yml).', + 'Returns the current project, its environments, and SDK keys.', 'Include dashboard link in the response.', ].join('\n'), annotations: { diff --git a/src/mcp/tools/projectTools.ts b/src/mcp/tools/projectTools.ts index 37eabcea..aaa7ed0e 100644 --- a/src/mcp/tools/projectTools.ts +++ b/src/mcp/tools/projectTools.ts @@ -118,7 +118,7 @@ export function registerProjectTools( { description: [ 'List all projects in the current organization.', - 'Include dashboard link in the response.', + 'Can be called before "select_project"', ].join('\n'), annotations: { title: 'List Projects', @@ -139,6 +139,7 @@ export function registerProjectTools( description: [ 'Get the currently selected project.', 'Include dashboard link in the response.', + 'Returns the current project, its environments, and SDK keys.', ].join('\n'), annotations: { title: 'Get Current Project', diff --git a/src/mcp/tools/resultsTools.ts b/src/mcp/tools/resultsTools.ts index e41c40e1..d4606709 100644 --- a/src/mcp/tools/resultsTools.ts +++ b/src/mcp/tools/resultsTools.ts @@ -81,7 +81,8 @@ export function registerResultsTools( 'get_feature_total_evaluations', { description: [ - 'Get total variable evaluations per time period for a specific feature.', + 'Get total variable evaluations per time period for a feature.', + 'Useful for understanding if a feature is being used or not.', 'Include dashboard link in the response.', ].join('\n'), annotations: { @@ -105,6 +106,7 @@ export function registerResultsTools( { description: [ 'Get total variable evaluations per time period for the entire project.', + 'Useful for understanding the overall usage of variables in a project by environment or SDK type.', 'Include dashboard link in the response.', ].join('\n'), annotations: { diff --git a/src/mcp/tools/selfTargetingTools.ts b/src/mcp/tools/selfTargetingTools.ts index 6e21c56e..cf69ed08 100644 --- a/src/mcp/tools/selfTargetingTools.ts +++ b/src/mcp/tools/selfTargetingTools.ts @@ -179,7 +179,8 @@ export function registerSelfTargetingTools( 'get_self_targeting_identity', { description: [ - 'Get current DevCycle identity for self-targeting.', + 'Get current DevCycle identity for self-targeting yourself into a feature.', + 'Your applications user_id used to identify yourself with the DevCycle SDK needs to match for self-targeting to work.', 'Include dashboard link in the response.', ].join('\n'), annotations: { @@ -197,7 +198,8 @@ export function registerSelfTargetingTools( 'update_self_targeting_identity', { description: [ - 'Update DevCycle identity for self-targeting and overrides.', + 'Update DevCycle identity for self-targeting yourself into a feature.', + 'Your applications user_id used to identify yourself with the DevCycle SDK needs to match for self-targeting to work.', 'Include dashboard link in the response.', ].join('\n'), annotations: { diff --git a/src/mcp/tools/variableTools.ts b/src/mcp/tools/variableTools.ts index 4d41d316..0cb6f47d 100644 --- a/src/mcp/tools/variableTools.ts +++ b/src/mcp/tools/variableTools.ts @@ -141,6 +141,7 @@ export function registerVariableTools( { description: [ 'Create a new variable.', + 'DevCycle variables can also be referred to as "feature flags" or "flags".', 'Include dashboard link in the response.', ].join('\n'), annotations: { @@ -159,7 +160,7 @@ export function registerVariableTools( { description: [ 'Update an existing variable.', - '⚠️ IMPORTANT: Variable changes can affect feature flags in production environments.', + '⚠️ IMPORTANT: Variable changes can affect features in production environments.', 'Always confirm with the user before updating variables for features that are active in production.', 'Include dashboard link in the response.', ].join('\n'), diff --git a/yarn.lock b/yarn.lock index 19d9c755..60e7429f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -77,6 +77,16 @@ __metadata: languageName: node linkType: hard +"@anthropic-ai/tokenizer@npm:^0.0.4": + version: 0.0.4 + resolution: "@anthropic-ai/tokenizer@npm:0.0.4" + dependencies: + "@types/node": "npm:^18.11.18" + tiktoken: "npm:^1.0.10" + checksum: 10c0/fddaa82c26228b6385a0a064c145450564d0288c51d0346a70ce62a716627b26c227077aa909405313668fb5cbef91f3e396783b83decfa01d1c8777d5141220 + languageName: node + linkType: hard + "@apidevtools/json-schema-ref-parser@npm:9.0.6": version: 9.0.6 resolution: "@apidevtools/json-schema-ref-parser@npm:9.0.6" @@ -734,6 +744,7 @@ __metadata: version: 0.0.0-use.local resolution: "@devcycle/cli@workspace:." dependencies: + "@anthropic-ai/tokenizer": "npm:^0.0.4" "@babel/code-frame": "npm:^7.27.1" "@babel/core": "npm:^7.28.0" "@babel/generator": "npm:^7.28.0" @@ -769,6 +780,7 @@ __metadata: eslint-config-prettier: "npm:^9.1.0" estraverse: "npm:^5.3.0" fuzzy: "npm:^0.1.3" + gpt-tokenizer: "npm:^3.0.1" inquirer: "npm:^8.2.6" inquirer-autocomplete-prompt: "npm:^2.0.1" js-sha256: "npm:^0.11.0" @@ -2824,6 +2836,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^18.11.18": + version: 18.19.123 + resolution: "@types/node@npm:18.19.123" + dependencies: + undici-types: "npm:~5.26.4" + checksum: 10c0/8077177ee2019c4c8875784b367732813b07e533f24197d1fb3bb09e81335267de9da3e70326daaba7a6499df2410257f6099d82d15c9a903d1587a752563178 + languageName: node + linkType: hard + "@types/node@npm:^18.19.68": version: 18.19.118 resolution: "@types/node@npm:18.19.118" @@ -6115,6 +6136,13 @@ __metadata: languageName: node linkType: hard +"gpt-tokenizer@npm:^3.0.1": + version: 3.0.1 + resolution: "gpt-tokenizer@npm:3.0.1" + checksum: 10c0/e95c0825ccc13d27ff873b507c95eca1224f6c7aafdc825a49271959c1f8480368bad75728e32190d106489e25bae788c5932c7c8266ec58dc446ad0d5a26883 + languageName: node + linkType: hard + "graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.5, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": version: 4.2.11 resolution: "graceful-fs@npm:4.2.11" @@ -10524,6 +10552,13 @@ __metadata: languageName: node linkType: hard +"tiktoken@npm:^1.0.10": + version: 1.0.22 + resolution: "tiktoken@npm:1.0.22" + checksum: 10c0/4805cf957d32ee53707ea8416256b50f6b4e865fce1d36ba507bfcbf4dd31bfa69b31dbd1303ba216ae44eb680f54c46c11defa5305445ecea946f2c048fa87d + languageName: node + linkType: hard + "tinybench@npm:^2.9.0": version: 2.9.0 resolution: "tinybench@npm:2.9.0"