Update screenshot collector (#1373)

# why

- when transitioning to v3, we did not use the latest version of
screenshot collector
- screenshot collector currently fails due to not having page.on and
page.off support for the load, and domcontentloaded events.

# what changed

- added latest version of screenshot collector 

# test plan

- ran evals in cli with additional logging to also verify everything is
working as expected
























<!-- This is an auto-generated description by cubic. -->
---
## Summary by cubic
Updated the evals CLI screenshot collector to the latest version, adding
image-diff filtering and a V3 event bus that emits agent screenshots.
This reduces duplicate screenshots and stabilizes capture on v3 pages
where navigation events are disabled.

- **New Features**
  - Skip similar screenshots using MSE/SSIM thresholds with sharp.
- Event bus integration: agents emit screenshots; collector can ingest
them.
- Non-blocking initial/final captures and safer interval capture with
error handling.

- **Dependencies**
  - Added sharp ^0.34.5 for image processing (evals and core).
  - Patch bump via changeset for @browserbasehq/stagehand-evals.

<sup>Written for commit f4e90f8.
Summary will update automatically on new commits.</sup>

<!-- End of auto-generated description by cubic. -->

---------

Co-authored-by: miguel <miguelg71921@gmail.com>
Co-authored-by: Miguel <36487034+miguelg719@users.noreply.github.com>

Author: tkattkat

.changeset/beige-taxes-punch.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand-evals": patch
+---
+
+Update screenshot collector in agent evals cli

.prettierignore

@@ -12,4 +12,8 @@ packages/core/lib/dom/build/
 packages/core/lib/v3/dom/build/
 packages/evals/dist/
 packages/docs/
-*.min.js
+*.min.js
+.browserbase/
+.browserbase/**
+**/.browserbase/
+**/.browserbase/**

eslint.config.mjs

@@ -14,6 +14,9 @@ export default [
       "packages/core/lib/v3/dom/build/**",
       "**/*.config.js",
       "**/*.config.mjs",
+      ".browserbase/**",
+      "**/.browserbase/**",
+      "**/*.json",
     ],
   },
   pluginJs.configs.recommended,

package.json

@@ -7,7 +7,7 @@
     "build": "turbo run build",
     "build:cli": "turbo run build:cli",
     "lint": "turbo run lint",
-    "format": "turbo run format",
+    "format": "prettier --write .",
     "prettier": "prettier --write .",
     "eslint": "eslint .",
     "test": "turbo run test",

packages/core/lib/utils.ts

@@ -7,6 +7,7 @@ import { ModelProvider } from "./v3/types/public/model";
 import { ZodPathSegments } from "./v3/types/private/internal";
 import type { StagehandZodSchema } from "./v3/zodCompat";
 import { isZod4Schema } from "./v3/zodCompat";
+import sharp from "sharp";
 
 const ID_PATTERN = /^\d+-\d+$/;
 
@@ -837,3 +838,21 @@ export function jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {
       return z.any();
   }
 }
+
+export async function imageResize(
+  img: Buffer,
+  scaleFactor: number,
+): Promise<Buffer> {
+  const metadata = await sharp(img).metadata();
+  // calculate new dimensions
+  const width = Math.round(metadata.width * scaleFactor);
+  const height = Math.round(metadata.height * scaleFactor);
+  return await sharp(img)
+    .resize(width, height, { fit: "inside", kernel: sharp.kernel.lanczos3 })
+    .png({
+      compressionLevel: 9,
+      adaptiveFiltering: true,
+      palette: true,
+    })
+    .toBuffer();
+}

packages/core/lib/v3/handlers/v3AgentHandler.ts

@@ -171,6 +171,17 @@ export class V3AgentHandler {
           }
         }
         state.currentPageUrl = (await this.v3.context.awaitActivePage()).url();
+
+        // Capture screenshot after tool execution and emit event
+        try {
+          await this.captureAndEmitScreenshot();
+        } catch (e) {
+          this.logger({
+            category: "agent",
+            message: `Warning: Failed to capture screenshot: ${getErrorMessage(e)}`,
+            level: 1,
+          });
+        }
       }
 
       if (userCallback) {
@@ -448,4 +459,21 @@ export class V3AgentHandler {
     }
     return stepCountIs(maxSteps)(result);
   }
+
+  /**
+   * Capture a screenshot and emit it via the event bus
+   */
+  private async captureAndEmitScreenshot(): Promise<void> {
+    try {
+      const page = await this.v3.context.awaitActivePage();
+      const screenshot = await page.screenshot({ fullPage: false });
+      this.v3.bus.emit("agent_screensot_taken_event", screenshot);
+    } catch (error) {
+      this.logger({
+        category: "agent",
+        message: `Error capturing screenshot: ${getErrorMessage(error)}`,
+        level: 0,
+      });
+    }
+  }
 }

packages/core/lib/v3/handlers/v3CuaAgentHandler.ts

@@ -23,8 +23,6 @@ export class V3CuaAgentHandler {
   private agentClient: AgentClient;
   private options: AgentHandlerOptions;
   private highlightCursor: boolean;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  private screenshotCollector?: any;
 
   constructor(
     v3: V3,
@@ -545,9 +543,8 @@ export class V3CuaAgentHandler {
     try {
       const page = await this.v3.context.awaitActivePage();
       const base64Image = await page.screenshot({ fullPage: false });
-      if (this.screenshotCollector) {
-        this.screenshotCollector.addScreenshot(base64Image);
-      }
+      // Emit screenshot event via the bus
+      this.v3.bus.emit("agent_screensot_taken_event", base64Image);
       const currentUrl = page.url();
       return await this.agentClient.captureScreenshot({
         base64Image,
@@ -571,20 +568,4 @@ export class V3CuaAgentHandler {
       // Best-effort only
     }
   }
-
-  /**
-   * Set the screenshot collector for this agent handler
-   */
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  setScreenshotCollector(collector: any): void {
-    this.screenshotCollector = collector;
-  }
-
-  /**
-   * Get the screenshot collector
-   */
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  getScreenshotCollector(): any {
-    return this.screenshotCollector;
-  }
 }

packages/core/lib/v3/v3.ts

@@ -1,4 +1,5 @@
 import dotenv from "dotenv";
+import { EventEmitter } from "events";
 import fs from "fs";
 import os from "os";
 import path from "path";
@@ -140,6 +141,12 @@ export class V3 {
   private observeHandler: ObserveHandler | null = null;
   private ctx: V3Context | null = null;
   public llmClient!: LLMClient;
+
+  /**
+   * Event bus for internal communication.
+   * Emits events like 'screenshot' when screenshots are captured during agent execution.
+   */
+  public readonly bus: EventEmitter = new EventEmitter();
   private modelName: AvailableModel;
   private modelClientOptions: ClientOptions;
   private llmProvider: LLMProvider;

packages/core/package.json

@@ -57,6 +57,7 @@
     "pino": "^9.6.0",
     "pino-pretty": "^13.0.0",
     "playwright": "^1.52.0",
+    "sharp": "^0.34.5",
     "ws": "^8.18.0",
     "zod-to-json-schema": "^3.25.0"
   },

packages/evals/index.eval.ts

@@ -386,13 +386,7 @@ const generateFilteredTestcases = (): Testcase[] => {
           // Pass full EvalInput to the task (data-driven params available via input.params)
           let result;
           try {
-            result = await taskFunction({
-              // ...taskInput,
-              v3: v3Input?.v3,
-              v3Agent: v3Input?.agent,
-              logger: v3Input?.logger,
-              v3Input,
-            });
+            result = await taskFunction({ ...v3Input, input });
             // Log result to console
             if (result && result._success) {
               console.log(`✅ ${input.name}: Passed`);