BuffBench nightly evals

Author: jahooma

.github/workflows/nightly-evals.yml

@@ -47,7 +47,7 @@ jobs:
           echo "CODEBUFF_GITHUB_TOKEN=${{ secrets.CODEBUFF_GITHUB_TOKEN }}" >> $GITHUB_ENV
 
       - name: Run nightly evals
-        run: cd evals && bun run-eval-set --concurrency 10 --email --title "Nightly Eval Run ($(date '+%Y-%m-%d'))"
+        run: cd evals && bun run-buffbench-nightly
 
       - name: Workflow completed
         run: echo "Nightly evals workflow completed successfully"

evals/buffbench/main-nightly.ts

@@ -0,0 +1,93 @@
+import path from 'path'
+
+import { sendBasicEmail } from '@codebuff/internal/loops'
+
+import { runBuffBench } from './run-buffbench'
+import type { AgentEvalResults } from './types'
+
+async function main() {
+  console.log('Starting nightly buffbench evaluation...')
+  console.log('Agents: base, base2')
+  console.log('Eval set: codebuff')
+  console.log()
+
+  const results = await runBuffBench({
+    evalDataPath: path.join(__dirname, 'eval-codebuff.json'),
+    agents: ['base', 'base2'],
+    taskConcurrency: 20,
+  })
+
+  console.log('\nNightly buffbench evaluation completed successfully!')
+
+  // Send email with results
+  const recipientEmail = process.env.EVAL_RESULTS_EMAIL || 'team@codebuff.com'
+  console.log(`\n📧 Sending buffbench results email to ${recipientEmail}...`)
+
+  const { metadata, ...agentResults } = results
+  const emailContent = formatBuffBenchEmailContent(agentResults, metadata)
+
+  try {
+    const emailResult = await sendBasicEmail({
+      email: recipientEmail,
+      data: emailContent,
+      logger: console,
+    })
+
+    if (emailResult.success) {
+      console.log('✅ BuffBench results email sent successfully!')
+    } else {
+      console.log('⚠️ Email sending was skipped (likely missing configuration)')
+    }
+  } catch (emailError) {
+    console.error('❌ Failed to send buffbench results email:', emailError)
+  }
+
+  process.exit(0)
+}
+
+function formatBuffBenchEmailContent(
+  results: Record<string, AgentEvalResults>,
+  metadata: any,
+) {
+  const agents = Object.keys(results)
+  const date = new Date().toLocaleDateString()
+
+  const agentScores = agents
+    .map((agentId) => `${agentId}: ${results[agentId].averageScore.toFixed(1)}`)
+    .join(' | ')
+
+  const subject = `Nightly BuffBench Results - ${date} - ${agentScores}`
+
+  const agentComparison = agents
+    .map(
+      (agentId) =>
+        `${agentId}:
+  - Average Score: ${results[agentId].averageScore.toFixed(2)}/10
+  - Average Cost: ${results[agentId].averageCost.toFixed(4)}
+  - Average Duration: ${(results[agentId].averageDuration / 1000).toFixed(1)}s
+  - Valid Runs: ${results[agentId].runs.length}`,
+    )
+    .join('\n\n')
+
+  const message = `📊 NIGHTLY BUFFBENCH RESULTS
+
+📈 AGENT RESULTS:
+${agentComparison}
+
+📁 Results Location: ${metadata.logsDirectory}
+⏱️  Total Evaluation Time: ${(metadata.totalDuration / 1000 / 60).toFixed(1)} minutes
+• Total Tasks: ${metadata.commitsEvaluated}
+• Agents Tested: ${agents.join(', ')}
+
+Generated on: ${metadata.timestamp}
+Repository: ${metadata.repoUrl}`
+
+  return { subject, message }
+}
+
+if (import.meta.main) {
+  main().catch((error) => {
+    console.error('Error running nightly buffbench:', error)
+    process.exit(1)
+  })
+}

evals/package.json

@@ -21,6 +21,8 @@
     "run-single-eval": "bun run git-evals/run-single-eval.ts --eval-file git-evals/eval-manifold.json --commit-sha ebabf7796a92ce8ece8e2452b0f3f896a513ba0e",
     "run-git-evals": "bun run git-evals/run-git-evals.ts",
     "run-eval-set": "bun run git-evals/run-eval-set.ts",
+    "run-buffbench": "bun run buffbench/main.ts",
+    "run-buffbench-nightly": "bun run buffbench/main-nightly.ts",
     "setup-codebuff-repo": "bun run setup-codebuff-repo.ts"
   },
   "sideEffects": false,