Save partial results to file for generating evals

Author: jahooma

evals/git-evals2/gen-evals.ts

@@ -104,6 +104,35 @@ function getCommitMessage(repoPath: string, commitSha: string): string {
   }).trim()
 }
 
+function printTaskResult(taskResult: {
+  id: string
+  reasoning: string
+  spec: string
+  prompt: string
+  supplementalFiles: string[]
+}) {
+  console.log('\n' + '='.repeat(80))
+  console.log('📋 GENERATED TASK')
+  console.log('='.repeat(80))
+  console.log(`\n🏷️  Task ID: ${taskResult.id}\n`)
+  console.log(`💭 Reasoning:\n${taskResult.reasoning}\n`)
+  console.log(`📝 Spec:\n${taskResult.spec}\n`)
+  console.log(`💬 Prompt:\n${taskResult.prompt}\n`)
+  console.log(`📁 Supplemental Files (${taskResult.supplementalFiles.length}):`)
+  taskResult.supplementalFiles.forEach((file, idx) => {
+    console.log(`   ${idx + 1}. ${file}`)
+  })
+  console.log('='.repeat(80) + '\n')
+}
+
+function savePartialResults(
+  partialPath: string,
+  evalData: EvalDataV2,
+): void {
+  fs.writeFileSync(partialPath, JSON.stringify(evalData, null, 2))
+  console.log(`💾 Saved partial results to ${partialPath}`)
+}
+
 export async function generateEvalFileV2({
   repoUrl,
   commitShas,
@@ -119,7 +148,13 @@ export async function generateEvalFileV2({
     apiKey: process.env[API_KEY_ENV_VAR] || getUserCredentials()?.authToken,
   })
 
+  const finalOutputPath =
+    outputPath || path.join(__dirname, `eval-${actualRepoName}-v2.json`)
+  const partialOutputPath = finalOutputPath.replace(/\.json$/, '.partial.json')
+
   console.log(`Processing ${commitShas.length} commits in parallel...`)
+  console.log(`Partial results will be saved to: ${partialOutputPath}`)
+  console.log(`Final results will be saved to: ${finalOutputPath}\n`)
 
   const BATCH_SIZE = 5
   const evalCommits: EvalCommitV2[] = []
@@ -159,16 +194,9 @@ export async function generateEvalFileV2({
           },
         })
 
-        console.log(`Task ID: ${taskResult.id}`)
-        console.log(`Generated spec: ${taskResult.spec.substring(0, 100)}...`)
-        console.log(
-          `Generated prompt: ${taskResult.prompt.substring(0, 100)}...`,
-        )
-        console.log(
-          `Supplemental files: ${taskResult.supplementalFiles.length} files`,
-        )
+        printTaskResult(taskResult)
 
-        return {
+        const evalCommit: EvalCommitV2 = {
           id: taskResult.id,
           sha: commitSha,
           parentSha,
@@ -177,24 +205,44 @@ export async function generateEvalFileV2({
           supplementalFiles: taskResult.supplementalFiles,
           fileDiffs,
         }
+
+        return evalCommit
       },
     )
   }
 
-  const batchResults = await mapLimit(commitShas, BATCH_SIZE, processCommit)
-  evalCommits.push(...(batchResults.filter(Boolean) as EvalCommitV2[]))
+  const batchResults = await mapLimit(
+    commitShas,
+    BATCH_SIZE,
+    async (commitSha: string) => {
+      const result = await processCommit(commitSha)
+      if (result) {
+        evalCommits.push(result)
+
+        const partialEvalData: EvalDataV2 = {
+          repoUrl,
+          generationDate: new Date().toISOString(),
+          evalCommits: [...evalCommits],
+        }
+        savePartialResults(partialOutputPath, partialEvalData)
+      }
+      return result
+    },
+  )
 
   const evalData: EvalDataV2 = {
     repoUrl,
     generationDate: new Date().toISOString(),
     evalCommits,
   }
 
-  const generatedOutputPath =
-    outputPath || path.join(__dirname, `eval-${actualRepoName}-v2.json`)
+  fs.writeFileSync(finalOutputPath, JSON.stringify(evalData, null, 2))
+  console.log(`\n✅ Eval data written to ${finalOutputPath}`)
 
-  fs.writeFileSync(generatedOutputPath, JSON.stringify(evalData, null, 2))
-  console.log(`Eval data written to ${generatedOutputPath}`)
+  if (fs.existsSync(partialOutputPath)) {
+    fs.unlinkSync(partialOutputPath)
+    console.log(`🗑️  Removed partial file: ${partialOutputPath}`)
+  }
 }
 
 if (require.main === module) {

evals/git-evals2/migrate-evals-to-v2.ts

@@ -98,16 +98,19 @@ async function migrateCommit(
         agentDefinitions,
       })
 
+      console.log(`\n--- Generated Task Result ---`)
       console.log(`Task ID: ${taskResult.id}`)
-      console.log(
-        `Generated spec: ${taskResult.spec.substring(0, 100)}...`,
-      )
-      console.log(
-        `Generated prompt: ${taskResult.prompt.substring(0, 100)}...`,
-      )
-      console.log(
-        `Supplemental files: ${taskResult.supplementalFiles.length} files`,
-      )
+      console.log(`\nReasoning:`)
+      console.log(taskResult.reasoning)
+      console.log(`\nSpec:`)
+      console.log(taskResult.spec)
+      console.log(`\nPrompt:`)
+      console.log(taskResult.prompt)
+      console.log(`\nSupplemental Files (${taskResult.supplementalFiles.length}):`)  
+      taskResult.supplementalFiles.forEach((file, i) => {
+        console.log(`  ${i + 1}. ${file}`)
+      })
+      console.log(`--- End Task Result ---\n`)
 
       return {
         id: taskResult.id,
@@ -150,6 +153,9 @@ export async function migrateEvalFile({
   const migratedCommits: EvalCommitV2[] = []
   const failedCommits: Array<{ sha: string; error: string }> = []
 
+  const finalOutputPath = outputPath || inputPath.replace(/\.json$/, '-v2.json')
+  const partialOutputPath = finalOutputPath.replace(/\.json$/, '.partial.json')
+
   const processCommit = async (
     oldCommit: EvalCommit,
     index: number,
@@ -159,12 +165,28 @@ export async function migrateEvalFile({
     )
 
     try {
-      return await migrateCommit(
+      const result = await migrateCommit(
         oldCommit,
         oldEvalData.repoUrl,
         client,
         localAgentDefinitions,
       )
+
+      if (result) {
+        migratedCommits.push(result)
+
+        const partialData: EvalDataV2 = {
+          repoUrl: oldEvalData.repoUrl,
+          testRepoName: oldEvalData.testRepoName,
+          generationDate: new Date().toISOString(),
+          initCommand: oldEvalData.initCommand,
+          evalCommits: migratedCommits,
+        }
+        fs.writeFileSync(partialOutputPath, JSON.stringify(partialData, null, 2))
+        console.log(`✓ Saved partial results to ${partialOutputPath}`)
+      }
+
+      return result
     } catch (error) {
       const errorMessage =
         error instanceof Error ? error.message : String(error)
@@ -180,7 +202,7 @@ export async function migrateEvalFile({
     }
   }
 
-  const results = await mapLimit(
+  await mapLimit(
     oldEvalData.evalCommits,
     batchSize,
     async (commit: EvalCommit) => {
@@ -189,8 +211,6 @@ export async function migrateEvalFile({
     },
   )
 
-  migratedCommits.push(...(results.filter(Boolean) as EvalCommitV2[]))
-
   console.log(
     `\n✓ Successfully migrated ${migratedCommits.length}/${oldEvalData.evalCommits.length} commits`,
   )
@@ -210,10 +230,13 @@ export async function migrateEvalFile({
     evalCommits: migratedCommits,
   }
 
-  const finalOutputPath = outputPath || inputPath.replace(/\.json$/, '-v2.json')
-
   fs.writeFileSync(finalOutputPath, JSON.stringify(newEvalData, null, 2))
 
+  if (fs.existsSync(partialOutputPath)) {
+    fs.unlinkSync(partialOutputPath)
+    console.log(`\n✓ Removed partial file: ${partialOutputPath}`)
+  }
+
   const oldSize = fs.statSync(inputPath).size
   const newSize = fs.statSync(finalOutputPath).size