diff --git a/docs/patterns.md b/docs/patterns.md index 9ac20cfe..a85f6301 100644 --- a/docs/patterns.md +++ b/docs/patterns.md @@ -618,5 +618,113 @@ app.ontoolinput = (params) => { > [!IMPORTANT] > Partial arguments are "healed" JSON — the host closes unclosed brackets/braces to produce valid JSON. This means objects may be incomplete (e.g., the last item in an array may be truncated). Don't rely on partial data for critical operations; use it only for preview UI. +_See [`examples/threejs-server/`](https://github.com/modelcontextprotocol/ext-apps/tree/main/examples/threejs-server) for a full implementation of this pattern._ + +## Using UI Data in Tool Call Responses + +Sometimes an MCP server needs user interaction to form its tool response. Since MCP Apps can be displayed before the tool call completes, you can use a combination of model-visible and app-only tools to collect user input and incorporate it into the final response. + +The pattern works as follows: + +1. Register a model-visible tool (e.g., `pick_color`) that displays a UI and waits for user input +2. Register an [app-only tool](#tools-that-are-private-to-apps) (e.g., `user_picked_color`) that the UI calls when the user completes their selection +3. The UI receives tool arguments via `ontoolinput`, then calls the app-only tool when the user finishes +4. The model-visible tool's handler resolves only after the app-only tool is called + +**Server-side**: Register both tools — one that waits for user input, and one that the UI calls to submit the result: + + +```ts source="./patterns.tsx#uiDataToolResponseServer" +// Map of pending color picks waiting for user input +const pendingColorPicks = new Map< + string, + { resolve: (color: string) => void } +>(); + +// 1. Model-visible tool that displays the color picker UI +registerAppTool( + server, + "pick_color", + { + title: "Pick Color", + description: "Let the user pick a color", + inputSchema: { + requestId: z.string().describe("Unique ID for this request"), + defaultColor: z.string().optional().describe("Initial color value"), + }, + _meta: { ui: { resourceUri: "ui://colors/picker.html" } }, + }, + async ({ requestId, defaultColor }) => { + // Wait for the user to pick a color via the UI + const pickedColor = await new Promise((resolve) => { + pendingColorPicks.set(requestId, { resolve }); + }); + + return { + content: [{ type: "text", text: `User selected: ${pickedColor}` }], + structuredContent: { color: pickedColor }, + }; + }, +); + +// 2. App-only tool that the UI calls when user finishes picking +registerAppTool( + server, + "user_picked_color", + { + title: "User Picked Color", + description: "Called by the UI when user selects a color", + inputSchema: { + requestId: z.string().describe("Request ID from pick_color"), + color: z.string().describe("The color the user picked"), + }, + // Hidden from model - only callable by the App + _meta: { ui: { visibility: ["app"] } }, + }, + async ({ requestId, color }) => { + const pending = pendingColorPicks.get(requestId); + if (pending) { + pending.resolve(color); + pendingColorPicks.delete(requestId); + return { content: [{ type: "text", text: "Color submitted" }] }; + } + return { + isError: true, + content: [{ type: "text", text: "No pending request found" }], + }; + }, +); +``` + > [!NOTE] -> For full examples that implement this pattern, see: [`examples/shadertoy-server/`](https://github.com/modelcontextprotocol/ext-apps/tree/main/examples/shadertoy-server) and [`examples/threejs-server/`](https://github.com/modelcontextprotocol/ext-apps/tree/main/examples/threejs-server). +> Because it may take users a while to interact the UI, servers should consider using [tasks](https://modelcontextprotocol.io/specification/draft/basic/utilities/tasks) for reliability and network fault-tolerance. + +**Client-side**: The UI receives the request ID, displays the picker, and calls the app-only tool when the user makes a selection: + + +```ts source="./patterns.tsx#uiDataToolResponseClient" +let requestId: string | undefined; + +// Receive the request ID from the tool input +app.ontoolinput = (params) => { + requestId = params.arguments?.requestId as string; + const defaultColor = (params.arguments?.defaultColor as string) ?? "#000000"; + initializeColorPicker(defaultColor); +}; + +// When user picks a color, call the app-only tool to complete the request +async function onColorSelected(color: string) { + if (!requestId) return; + + await app.callServerTool({ + name: "user_picked_color", + arguments: { requestId, color }, + }); +} + +// Example: Wire up a color input +const colorInput = document.querySelector("#color-picker"); +colorInput?.addEventListener("change", () => { + onColorSelected(colorInput.value); +}); +``` diff --git a/docs/patterns.tsx b/docs/patterns.tsx index 6b876926..27c4d4c9 100644 --- a/docs/patterns.tsx +++ b/docs/patterns.tsx @@ -455,6 +455,108 @@ function visibilityBasedPause( //#endregion visibilityBasedPause } +/** + * Example: Server-side tools for UI-driven tool responses + */ +function uiDataToolResponseServer(server: McpServer) { + //#region uiDataToolResponseServer + // Map of pending color picks waiting for user input + const pendingColorPicks = new Map< + string, + { resolve: (color: string) => void } + >(); + + // 1. Model-visible tool that displays the color picker UI + registerAppTool( + server, + "pick_color", + { + title: "Pick Color", + description: "Let the user pick a color", + inputSchema: { + requestId: z.string().describe("Unique ID for this request"), + defaultColor: z.string().optional().describe("Initial color value"), + }, + _meta: { ui: { resourceUri: "ui://colors/picker.html" } }, + }, + async ({ requestId, defaultColor }) => { + // Wait for the user to pick a color via the UI + const pickedColor = await new Promise((resolve) => { + pendingColorPicks.set(requestId, { resolve }); + }); + + return { + content: [{ type: "text", text: `User selected: ${pickedColor}` }], + structuredContent: { color: pickedColor }, + }; + }, + ); + + // 2. App-only tool that the UI calls when user finishes picking + registerAppTool( + server, + "user_picked_color", + { + title: "User Picked Color", + description: "Called by the UI when user selects a color", + inputSchema: { + requestId: z.string().describe("Request ID from pick_color"), + color: z.string().describe("The color the user picked"), + }, + // Hidden from model - only callable by the App + _meta: { ui: { visibility: ["app"] } }, + }, + async ({ requestId, color }) => { + const pending = pendingColorPicks.get(requestId); + if (pending) { + pending.resolve(color); + pendingColorPicks.delete(requestId); + return { content: [{ type: "text", text: "Color submitted" }] }; + } + return { + isError: true, + content: [{ type: "text", text: "No pending request found" }], + }; + }, + ); + //#endregion uiDataToolResponseServer +} + +/** + * Example: Client-side color picker that submits user selection + */ +function uiDataToolResponseClient(app: App) { + //#region uiDataToolResponseClient + let requestId: string | undefined; + + // Receive the request ID from the tool input + app.ontoolinput = (params) => { + requestId = params.arguments?.requestId as string; + const defaultColor = (params.arguments?.defaultColor as string) ?? "#000000"; + initializeColorPicker(defaultColor); + }; + + // When user picks a color, call the app-only tool to complete the request + async function onColorSelected(color: string) { + if (!requestId) return; + + await app.callServerTool({ + name: "user_picked_color", + arguments: { requestId, color }, + }); + } + + // Example: Wire up a color input + const colorInput = document.querySelector("#color-picker"); + colorInput?.addEventListener("change", () => { + onColorSelected(colorInput.value); + }); + //#endregion uiDataToolResponseClient +} + +// Stubs for uiDataToolResponseClient example +declare function initializeColorPicker(defaultColor: string): void; + // Suppress unused variable warnings void pollingVanillaJs; void pollingReact; @@ -467,3 +569,5 @@ void hostContextReact; void persistViewStateServer; void persistViewState; void visibilityBasedPause; +void uiDataToolResponseServer; +void uiDataToolResponseClient;