Store agent trace

Author: jahooma

evals/git-evals2/agent-runner.ts

@@ -7,11 +7,18 @@ import { withTestRepo } from '../subagents/test-repo-utils'
 
 import type { EvalCommit } from './types'
 
+export interface AgentStep {
+  response: string
+  toolCalls: any[]
+  toolResults: any[]
+}
+
 export interface AgentRunResult {
   diff: string
   durationMs: number
   cost: number
   error?: string
+  trace: AgentStep[]
 }
 
 export async function runAgentOnCommit({
@@ -31,6 +38,7 @@ export async function runAgentOnCommit({
   let diff = ''
   let error: string | undefined
   let cost = 0
+  const trace: AgentStep[] = []
 
   try {
     await withTestRepo(
@@ -45,13 +53,44 @@ export async function runAgentOnCommit({
           await loadLocalAgents({ agentsPath }),
         )
 
+        let responseText = ''
+        let toolCalls: any[] = []
+        let toolResults: any[] = []
+        
+        function flushStep() {
+          if (responseText.length > 0 || toolCalls.length > 0 || toolResults.length > 0) {
+            trace.push({ response: responseText, toolCalls, toolResults })
+            responseText = ''
+            toolCalls = []
+            toolResults = []
+          }
+        }
+
         const result = await client.run({
           agent: agentId,
           prompt: commit.spec,
           agentDefinitions: localAgentDefinitions,
           cwd: repoDir,
+          handleEvent: (event) => {
+            if (event.type === 'text') {
+              if (toolResults.length > 0) {
+                flushStep()
+              }
+              responseText += event.text
+            } else if (event.type === 'tool_call') {
+              if (event.toolName === 'set_messages') {
+                return
+              }
+              toolCalls.push(event)
+            } else if (event.type === 'tool_result') {
+              toolResults.push(event)
+            } else if (event.type === 'finish') {
+              flushStep()
+            }
+          },
         })
 
+        flushStep()
         cost = result.sessionState.mainAgentState.creditsUsed / 100
 
         execSync('git add .', { cwd: repoDir, stdio: 'ignore' })
@@ -72,5 +111,6 @@ export async function runAgentOnCommit({
     durationMs,
     cost,
     error,
+    trace,
   }
 }

evals/git-evals2/run-git-evals2.ts

@@ -20,9 +20,7 @@ export async function runGitEvals2(
 ): Promise<GitEvals2Result> {
   const { evalDataPath, agents, outputPath, limit, onProgress } = options
 
-  const evalData: EvalData = JSON.parse(
-    fs.readFileSync(evalDataPath, 'utf-8'),
-  )
+  const evalData: EvalData = JSON.parse(fs.readFileSync(evalDataPath, 'utf-8'))
   const commitsToRun = limit
     ? evalData.evalCommits.slice(0, limit)
     : evalData.evalCommits
@@ -36,6 +34,16 @@ export async function runGitEvals2(
   const startTime = Date.now()
   const results = new Map<string, AgentEvalResults>()
 
+  // Create logs directory with current date and time
+  const date = new Date().toISOString().replace(/:/g, '-').slice(0, 16) // YYYY-MM-DDTHH-MM
+  const outputDir = outputPath
+    ? path.dirname(outputPath)
+    : 'evals/git-evals2/results'
+  const logsDir = path.join(outputDir, 'logs', date)
+  if (!fs.existsSync(logsDir)) {
+    fs.mkdirSync(logsDir, { recursive: true })
+  }
+
   for (const agentId of agents) {
     results.set(agentId, {
       agentId,
@@ -85,6 +93,28 @@ export async function runGitEvals2(
           error: agentResult.error,
         }
 
+        // Save trace to logs directory
+        const safeAgentId = agentId.replace(/[^a-zA-Z0-9-]/g, '_')
+        const safeCommitShort = commit.sha.slice(0, 7)
+        const traceFilename = `${safeAgentId}-${safeCommitShort}.json`
+        const tracePath = path.join(logsDir, traceFilename)
+
+        const traceData = {
+          agentId,
+          commitSha: commit.sha,
+          spec: commit.spec,
+          trace: agentResult.trace,
+          diff: agentResult.diff,
+          judgeResult,
+          cost: agentResult.cost,
+          durationMs: agentResult.durationMs,
+          error: agentResult.error,
+          timestamp: new Date().toISOString(),
+        }
+
+        fs.writeFileSync(tracePath, JSON.stringify(traceData, null, 2))
+        console.log(`Trace saved to ${tracePath}`)
+
         onProgress?.({
           type: 'agent_complete',
           agent: agentId,
@@ -172,6 +202,7 @@ export async function runGitEvals2(
     console.log(`\nResults written to ${outputPath}`)
   }
 
+  console.log(`\nTraces saved to ${logsDir}`)
   console.log('\n=== Summary ===')
   for (const [agentId, data] of results) {
     console.log(`\n${agentId}:`)