Log trace when agent errors

Author: jahooma

evals/git-evals2/judge.ts

@@ -165,17 +165,30 @@ ${agentDiff || '(No changes made)'}
 \`\`\`
 ${error ? `\n## Error Encountered\n${error}` : ''}`
 
+  const agentOutput: string[] = []
   const judgeResult = await client.run({
     agent: 'git-evals2-judge',
     prompt: judgePrompt,
     agentDefinitions: [judgeAgent],
+    handleEvent: (event) => {
+      if (event.type === 'text') {
+        agentOutput.push(event.text)
+      }
+      else if (event.type === 'tool_call') {
+        agentOutput.push(JSON.stringify(event, null, 2))
+      }
+    },
   })
 
   if (judgeResult.output.type !== 'structuredOutput') {
     console.error(
       'Error running judge agent - not structured output',
       JSON.stringify(judgeResult.output, null, 2),
     )
+    console.error(
+      'Judge agent output trace:',
+      agentOutput.join(''),
+    )
     return {
       analysis: 'Error running judge agent - not structured output',
       strengths: [],

evals/git-evals2/trace-analyzer.ts

@@ -247,10 +247,18 @@ Analyze how these agents approached the problem, focusing on their processes and
 
 Focus on the HOW, not the WHAT: We want to understand and improve how agents work, not evaluate their specific code output.`
 
+  const agentOutput: string[] = []
   const analyzerResult = await client.run({
     agent: 'git-evals2-trace-analyzer',
     prompt,
     agentDefinitions: [traceAnalyzerAgent],
+    handleEvent: (event) => {
+      if (event.type === 'text') {
+        agentOutput.push(event.text)
+      } else if (event.type === 'tool_call') {
+        agentOutput.push(JSON.stringify(event, null, 2))
+      }
+    },
   })
 
   const { output } = analyzerResult
@@ -260,6 +268,7 @@ Focus on the HOW, not the WHAT: We want to understand and improve how agents wor
       'Error running trace analyzer - not structured output',
       JSON.stringify(output, null, 2),
     )
+    console.error('Trace analyzer output trace:', agentOutput.join(''))
     return {
       overallAnalysis: 'Error running trace analyzer - not structured output',
       agentFeedback: [],