feat: support configuring timeout for evals

Right now we hard-coded to 30min. This is way too large for
non-remote environment evaluations. We previously only increased for
such situations.

Author: devversion

runner/configuration/environment-config.ts

@@ -59,6 +59,12 @@ export const environmentConfigSchema = z.object({
   codeRatingPrompt: z.string().optional(),
   /** When enabled, the system prompts for this environment won't be included in the report. */
   classifyPrompts: z.boolean().optional(),
+  /**
+   * Timeout in minutes for a single prompt evaluation.
+   *
+   * E.g. if a single app takes longer than 10min, it will be aborted.
+   */
+  promptTimeoutMin: z.number().optional(),
   /** Executor to be used for this environment. */
   executor: executorSchema
     .optional()

runner/configuration/environment.ts

@@ -36,6 +36,8 @@ export class Environment {
   readonly isBuiltIn: boolean;
   /** Configured executor. */
   readonly executor: Executor;
+  /** Timeout for a single eval prompt in minutes. */
+  readonly promptTimeoutMin: number | undefined;
 
   constructor(
     rootPath: string,
@@ -62,6 +64,7 @@ export class Environment {
     this.classifyPrompts = config.classifyPrompts ?? false;
     this.isBuiltIn = rootPath.includes('node_modules');
     this.executor = config.executor;
+    this.promptTimeoutMin = config.promptTimeoutMin;
   }
 
   /** Prompts that should be executed as a part of the evaluation. */

runner/orchestration/generate.ts

@@ -167,10 +167,8 @@ export async function generateCodeAndAssess(options: AssessmentConfig): Promise<
                   workerConcurrencyQueue,
                   progress,
                 ),
-              // 30min max per app evaluation. We just want to make sure it never gets stuck.
-              // Note that this timeout is expected to never be hit as individual action timeouts
-              // should fire first. E.g. local executor build or test timeouts.
-              30,
+              // A timeout is used to prevent from stuck evaluations.
+              env.promptTimeoutMin ?? 10,
             );
             return results;
           } catch (e: unknown) {