feat: add tool logs

Author: amishne

pnpm-lock.yaml

@@ -23,6 +23,7 @@
     "@shikijs/themes": "3.13.0",
     "express": "^4.18.2",
     "jszip": "^3.10.1",
+    "ngx-json-viewer": "^3.2.1",
     "rxjs": "~7.8.0",
     "shiki": "^3.6.0",
     "tinyglobby": "^0.2.14",

report-app/package.json

@@ -411,6 +411,32 @@ <h4>Debugging Tools</h4>
                   (click)="downloadDebuggingZip(result)">
                   Download ZIP for debugging
                 </button>
+                @if (result.toolLogs.length > 0) {
+                  <expansion-panel>
+                    <expansion-panel-header>Tool Logs</expansion-panel-header>
+                    <ul class="tool-logs-list">
+                    @for (log of result.toolLogs; track $index) {
+                      <li>
+                        <details class="details mcp-log-entry">
+                          @let name = log.request.name;
+                          <summary>
+                            Log Entry #{{ $index + 1
+                            }}{{ name ? ' - ' + name : '' }}
+                          </summary>
+                          <div class="mcp-log-content">
+                            <h5>Request</h5>
+                            <ngx-json-viewer [json]="log.request" [expanded]="false"></ngx-json-viewer>
+                            <h5>Response</h5>
+                            <ngx-json-viewer [json]="log.response" [expanded]="false"></ngx-json-viewer>
+                          </div>
+                        </details>
+                      </li>
+                    } @empty {
+                      <li>No MCP logs were recorded for this run.</li>
+                    }
+                    </ul>
+                  </expansion-panel>
+                }
               </div>
 
               @if (finalBuild.runtimeErrors) {

report-app/src/app/pages/report-viewer/report-viewer.html

@@ -63,7 +63,7 @@ expansion-panel {
   padding: 0 1rem 1rem;
 }
 
-.app-details-section expansion-panel {
+.app-details-section expansion-panel, .app-details-section button {
   margin-bottom: 0.5rem;
 }
 
@@ -233,3 +233,32 @@ expansion-panel {
   padding: 0px 20px;
 }
 
+.mcp-log-entry {
+  border: 1px solid var(--border-color);
+  border-radius: var(--border-radius);
+  margin: 1rem 0;
+
+  & > summary {
+    cursor: pointer;
+    font-weight: 500;
+    padding: 1rem 1.5rem;
+
+    &:hover {
+      background-color: var(--button-active-bg-color);
+    }
+  }
+
+  & .mcp-log-content {
+    padding: 0 1.5rem 1.5rem;
+
+    h5 {
+      margin-top: 1.5rem;
+      margin-bottom: 0.5rem;
+    }
+  }
+}
+
+.tool-logs-list {
+  list-style: none;
+  padding: 0;
+}

report-app/src/app/pages/report-viewer/report-viewer.scss

@@ -11,6 +11,7 @@ import {
   signal,
   viewChild,
 } from '@angular/core';
+import { NgxJsonViewerModule } from 'ngx-json-viewer';
 import { BuildErrorType } from '../../../../../runner/builder/builder-types';
 import {
   AssessmentResult,
@@ -55,6 +56,7 @@ import { ProviderLabel } from '../../shared/provider-label';
     ExpansionPanel,
     ExpansionPanelHeader,
     ProviderLabel,
+    NgxJsonViewerModule,
   ],
   templateUrl: './report-viewer.html',
   styleUrls: ['./report-viewer.scss'],

report-app/src/app/pages/report-viewer/report-viewer.ts

@@ -89,7 +89,7 @@ export class GeminiCliRunner implements LlmRunner {
       });
     }
 
-    return { files, reasoning };
+    return { files, reasoning, toolLogs: [] };
   }
 
   generateText(): Promise<LlmGenerateTextResponse> {

runner/codegen/gemini-cli/gemini-cli-runner.ts

@@ -1,5 +1,6 @@
 import {
   DynamicResourceAction,
+  GenerateResponse,
   genkit,
   ModelReference,
   ToolAction,
@@ -27,6 +28,7 @@ import {
   GenkitModelProvider,
   PromptDataForCounting,
 } from './model-provider.js';
+import { ToolLogEntry } from '../../shared-interfaces.js';
 
 const globalLogger = new GenkitLogger();
 logger.init(globalLogger);
@@ -38,6 +40,7 @@ export class GenkitRunner implements LlmRunner {
   readonly hasBuiltInRepairLoop = false;
   private readonly genkitInstance = this.getGenkitInstance();
   private mcpHost: GenkitMcpHost | null = null;
+  private toolLogs: ToolLogEntry[] = [];
 
   async generateConstrained<T extends z.ZodTypeAny = z.ZodTypeAny>(
     options: LlmConstrainedOutputGenerateRequestOptions<T>
@@ -75,9 +78,14 @@ export class GenkitRunner implements LlmRunner {
       files: result.output.outputFiles || [],
       usage: result.usage,
       reasoning: result.reasoning,
+      toolLogs: this.flushToolLogs(),
     };
   }
 
+  flushToolLogs(): ToolLogEntry[] {
+    return this.toolLogs.splice(0);
+  }
+
   async generateText(
     options: LlmGenerateTextRequestOptions
   ): Promise<LlmGenerateTextResponse> {
@@ -87,6 +95,7 @@ export class GenkitRunner implements LlmRunner {
       text: result.text,
       usage: result.usage,
       reasoning: result.reasoning,
+      toolLogs: this.flushToolLogs(),
     };
   }
 
@@ -120,7 +129,7 @@ export class GenkitRunner implements LlmRunner {
             ]);
           }
 
-          return this.genkitInstance.generate({
+          const response = await this.genkitInstance.generate({
             prompt: options.prompt,
             model,
             output: schema
@@ -145,6 +154,10 @@ export class GenkitRunner implements LlmRunner {
             resources,
             abortSignal: options.abortSignal,
           });
+
+          this._logToolUsage(response);
+
+          return response;
         };
 
         return options.timeout
@@ -158,6 +171,42 @@ export class GenkitRunner implements LlmRunner {
     );
   }
 
+  private _logToolUsage(response: GenerateResponse<any>) {
+    const toolRequests = new Map<string, any>();
+    const toolResponses = new Map<string, any>();
+
+    if (response.request?.messages) {
+      for (const message of response.request.messages) {
+        if (!message.content) {
+          continue;
+        }
+        for (const contentPart of message.content) {
+          if (contentPart.toolRequest) {
+            toolRequests.set(
+              contentPart.toolRequest.ref || '0',
+              contentPart.toolRequest
+            );
+          } else if (contentPart.toolResponse) {
+            toolResponses.set(
+              contentPart.toolResponse.ref || '0',
+              contentPart.toolResponse
+            );
+          }
+        }
+      }
+    }
+
+    for (const [ref, toolRequest] of toolRequests.entries()) {
+      const toolResponse = toolResponses.get(ref);
+      if (toolResponse) {
+        this.toolLogs.push({
+          request: toolRequest,
+          response: toolResponse,
+        });
+      }
+    }
+  }
+
   startMcpServerHost(hostName: string, servers: McpServerOptions[]): void {
     if (this.mcpHost !== null) {
       throw new Error('MCP host is already started');

runner/codegen/genkit/genkit-runner.ts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import { LlmResponseFile, Usage } from '../shared-interfaces.js';
+import { LlmResponseFile, ToolLogEntry, Usage } from '../shared-interfaces.js';
 import { UserFacingError } from '../utils/errors.js';
 
 export function assertValidModelName(value: string, availableModels: string[]) {
@@ -141,22 +141,24 @@ export interface LlmConstrainedOutputGenerateResponse<
   reasoning: string;
 }
 
-/** File generation response from the LLM. */
-export interface LlmGenerateFilesResponse {
-  files: LlmResponseFile[];
+/** LLM response. */
+interface BaseLlmGenerateResponse {
   /** Token usage data, if available. */
   usage?: Partial<Usage>;
   /** Reasoning messages from the LLM. */
   reasoning: string;
+  /** Tool requests and responses. */
+  toolLogs: ToolLogEntry[];
+}
+
+/** File generation response from the LLM. */
+export interface LlmGenerateFilesResponse extends BaseLlmGenerateResponse {
+  files: LlmResponseFile[];
 }
 
 /** Text response from the LLM. */
-export interface LlmGenerateTextResponse {
+export interface LlmGenerateTextResponse extends BaseLlmGenerateResponse {
   text: string;
-  /** Token usage data, if available. */
-  usage?: Partial<Usage>;
-  /** Reasoning messages from the LLM. */
-  reasoning: string;
 }
 
 /** Schema for the LLM server options. */

runner/codegen/llm-runner.ts

@@ -4,7 +4,7 @@ import {
   BuildWorkerMessage,
   RepairType,
 } from '../builder/builder-types.js';
-import { LlmRunner } from '../codegen/llm-runner.js';
+import { LlmGenerateFilesResponse, LlmRunner } from '../codegen/llm-runner.js';
 import { Environment } from '../configuration/environment.js';
 import {
   AttemptDetails,
@@ -44,11 +44,7 @@ export async function attemptBuild(
   rootPromptDef: RootPromptDefinition,
   directory: string,
   contextFiles: LlmContextFile[],
-  initialResponse: {
-    usage: Usage;
-    outputFiles: LlmResponseFile[];
-    reasoning: string;
-  },
+  initialResponse: LlmGenerateFilesResponse,
   attemptDetails: AttemptDetails[],
   skipScreenshots: boolean,
   skipAxeTesting: boolean,
@@ -72,7 +68,7 @@ export async function attemptBuild(
 
   // Clone the original files, because we're going to mutate them between repair
   // attempts and we don't want the different runs to influence each other.
-  const finalOutputFiles = initialResponse.outputFiles.map((file) => ({
+  const finalOutputFiles = initialResponse.files.map((file) => ({
     ...file,
   }));
   let buildResult = await workerConcurrencyQueue.add(
@@ -86,8 +82,11 @@ export async function attemptBuild(
     : DEFAULT_MAX_REPAIR_ATTEMPTS;
 
   attemptDetails.push({
-    outputFiles: initialResponse.outputFiles,
-    usage: initialResponse.usage,
+    outputFiles: initialResponse.files,
+    usage: {
+      ...{ inputTokens: 0, outputTokens: 0, totalTokens: 0 },
+      ...initialResponse.usage,
+    },
     reasoning: initialResponse.reasoning,
     buildResult,
     attempt: 0,

runner/orchestration/build.ts

@@ -3,6 +3,7 @@ import {
   LlmResponse,
   LlmResponseFile,
   RootPromptDefinition,
+  ToolLogEntry,
   Usage,
 } from '../shared-interfaces.js';
 import {
@@ -49,6 +50,7 @@ export async function generateCodeWithAI(
   let usage: Usage;
   let success: boolean;
   let reasoning: string;
+  let toolLogs: ToolLogEntry[];
 
   const contextMessageData = prepareContextFilesMessage(contextFiles);
   const messages: PromptDataMessage[] | undefined = contextMessageData
@@ -72,6 +74,7 @@ export async function generateCodeWithAI(
       totalTokens: response.usage?.totalTokens ?? 0,
     };
     reasoning = response.reasoning;
+    toolLogs = response.toolLogs ?? [];
 
     progress.log(
       promptDef,
@@ -100,6 +103,7 @@ export async function generateCodeWithAI(
     usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
     success = false;
     reasoning = '';
+    toolLogs = [];
     errors.push(error + '');
     progress.log(
       promptDef,
@@ -117,6 +121,7 @@ export async function generateCodeWithAI(
     errors,
     usage,
     reasoning,
+    toolLogs,
   } satisfies LlmResponse;
 }