Migrate evals to use shallow git clones. Store parentSha per eval task

Author: jahooma

evals/git-evals/eval-codebuff.json

@@ -30,6 +30,43 @@ Please wrap your final specification in <spec></spec> tags.`
 const fingerprintId = 'evals-v2'
 const userInputId = 'evals-v2'
 
+async function getParentSha(
+  repoPath: string,
+  commitSha: string,
+): Promise<string | null> {
+  try {
+    const parentSha = execSync(`git rev-parse ${commitSha}^`, {
+      cwd: repoPath,
+      encoding: 'utf-8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim()
+    return parentSha
+  } catch (error) {
+    try {
+      const parents = execSync(`git log --pretty=%P -n 1 ${commitSha}`, {
+        cwd: repoPath,
+        encoding: 'utf-8',
+        stdio: ['ignore', 'pipe', 'ignore'],
+      }).trim()
+      const parentCount = parents.split(' ').filter(Boolean).length
+      if (parentCount === 0) {
+        console.warn(
+          `Skipping ${commitSha.slice(0, 8)} - initial commit (no parent)`,
+        )
+        return null
+      } else if (parentCount > 1) {
+        console.warn(
+          `Skipping ${commitSha.slice(0, 8)} - merge commit (${parentCount} parents)`,
+        )
+        return null
+      }
+    } catch (e) {
+      console.error(`Error checking parents for ${commitSha.slice(0, 8)}:`, e)
+    }
+    return null
+  }
+}
+
 async function generateFileStateFromCommit(
   repoPath: string,
   commitSha: string,
@@ -191,6 +228,14 @@ export async function generateEvalFile({
       return null
     }
 
+    // Get parent SHA - either provided or computed from commit
+    const parentSha =
+      evalInput.parentSha ?? (await getParentSha(repoPath, evalInput.commitSha))
+
+    if (!parentSha) {
+      return null
+    }
+
     // Get file states - either provided or computed from commit
     const fileStates =
       evalInput.fileStates ??
@@ -205,6 +250,7 @@ export async function generateEvalFile({
 
     return {
       sha: evalInput.commitSha,
+      parentSha,
       spec,
       fileStates,
     }

evals/git-evals/eval-codebuff2.json

@@ -69,8 +69,8 @@ export async function runSingleEval(
   process.on('unhandledRejection', unhandledHandler)
 
   try {
-    // Reset to the commit before the target commit
-    resetRepoToCommit(projectPath, `${evalCommit.sha}^`)
+    // Reset to the parent commit
+    resetRepoToCommit(projectPath, evalCommit.parentSha)
 
     // Initialize state
     let runner: Runner
@@ -221,7 +221,7 @@ Explain your reasoning in detail. Do not ask Codebuff to git commit changes.`,
   const endTime = new Date()
   const durationMs = endTime.getTime() - startTime.getTime()
 
-  const fileStates = getCodebuffFileStates(evalCommit.sha, projectPath)
+  const fileStates = getCodebuffFileStates(evalCommit, projectPath)
 
   if (fs.existsSync(projectPath) && fs.statSync(projectPath).isDirectory()) {
     fs.rmSync(projectPath, { recursive: true, force: true })
@@ -284,13 +284,12 @@ function writeJsonToFile(json: any, path: string) {
 }
 
 function getCodebuffFileStates(
-  evalCommitSha: string,
+  evalCommit: EvalCommit,
   projectPath: string,
 ): string {
-  // Get all changes since the commit before the target commit
   execFileSync('git', ['add', '.'], { cwd: projectPath, stdio: 'ignore' })
 
-  return execFileSync('git', ['diff', `${evalCommitSha}^`], {
+  return execFileSync('git', ['diff', evalCommit.parentSha], {
     cwd: projectPath,
     stdio: ['ignore', 'pipe', 'pipe'],
   }).toString()
@@ -453,6 +452,7 @@ export async function runGitEvals(
               evalCommit.sha,
               true,
               evalData.initCommand,
+              evalCommit.parentSha,
             )
 
             console.log(

evals/git-evals/eval-manifold.json

@@ -170,6 +170,7 @@ async function runSingleEvalTask(options: {
     evalCommit.sha,
     undefined,
     evalData.initCommand,
+    evalCommit.parentSha,
   )
   console.log(`Repository cloned to: ${projectPath}`)