Add logger for evals

Author: jahooma

evals/buffbench/run-buffbench.ts

@@ -12,7 +12,7 @@ import { judgeCommitResult } from './judge'
 import { analyzeAgentTraces, type AgentTraceData } from './trace-analyzer'
 import { extractAgentLessons, saveAgentLessons } from './lessons-extractor'
 import { CodebuffClient } from '../../sdk/src/client'
-
+import { logger } from '../logger'
 import type { AgentEvalResults, EvalDataV2 } from './types'
 import { analyzeAllTasks } from './meta-analyzer'
 
@@ -245,6 +245,7 @@ export async function runBuffBench(options: {
   const client =
     options.client ??
     new CodebuffClient({
+      logger,
       apiKey: process.env[API_KEY_ENV_VAR] || getUserCredentials()?.authToken,
     })
 

evals/logger.ts

@@ -0,0 +1,69 @@
+import { mkdirSync } from 'fs'
+import path, { dirname } from 'path'
+import { pino } from 'pino'
+
+let logPath: string | undefined = undefined
+let pinoLogger: any = undefined
+
+const loggingLevels = ['info', 'debug', 'warn', 'error', 'fatal'] as const
+type LogLevel = (typeof loggingLevels)[number]
+
+function initPinoLoggerWithPath(path: string): void {
+  if (path === logPath) return // nothing to do
+
+  logPath = path
+  mkdirSync(dirname(path), { recursive: true })
+
+  // ──────────────────────────────────────────────────────────────
+  //  pino.destination(..) → SonicBoom stream, no worker thread
+  // ──────────────────────────────────────────────────────────────
+  const fileStream = pino.destination({
+    dest: path, // absolute or relative file path
+    mkdir: true, // create parent dirs if they don’t exist
+    sync: false, // set true if you *must* block on every write
+  })
+
+  pinoLogger = pino(
+    {
+      level: 'debug',
+      formatters: {
+        level: (label) => ({ level: label.toUpperCase() }),
+      },
+      timestamp: () => `,"timestamp":"${new Date().toISOString()}"`,
+    },
+    fileStream, // <-- no worker thread involved
+  )
+}
+
+function log(level: LogLevel, data: any, msg?: string, ...args: any[]): void {
+  if (
+    process.env.CODEBUFF_GITHUB_ACTIONS !== 'true' &&
+    process.env.NEXT_PUBLIC_CB_ENVIRONMENT !== 'test'
+  ) {
+    const projectRoot = path.join(__dirname, '..')
+    const logTarget = path.join(projectRoot, 'debug', 'evals.log')
+
+    initPinoLoggerWithPath(logTarget)
+  }
+
+  if (pinoLogger !== undefined) {
+    pinoLogger[level]({ data }, msg, ...args)
+  }
+}
+
+/**
+ * Wrapper around Pino logger.
+ *
+ * To also send to Posthog, set data.eventId to type AnalyticsEvent
+ *
+ * e.g. logger.info({eventId: AnalyticsEvent.SOME_EVENT, field: value}, 'some message')
+ */
+export const logger: Record<LogLevel, pino.LogFn> = Object.fromEntries(
+  loggingLevels.map((level) => {
+    return [
+      level,
+      (data: any, msg?: string, ...args: any[]) =>
+        log(level, data, msg, ...args),
+    ]
+  }),
+) as Record<LogLevel, pino.LogFn>