simplified page representation.

Author: bhouston

packages/agent/src/tools/browser/browseMessage.ts

@@ -3,7 +3,9 @@ import { zodToJsonSchema } from 'zod-to-json-schema';
 
 import { Tool } from '../../core/types.js';
 import { errorToString } from '../../utils/errorToString.js';
+import { sleep } from '../../utils/sleep.js';
 
+import { getRenderedDOM } from './getRenderedDOM.js';
 import { browserSessions, type BrowserAction, SelectorType } from './types.js';
 
 // Schema for browser action
@@ -111,7 +113,8 @@ export const browseMessageTool: Tool<Parameters, ReturnType> = {
               `Navigating to ${action.url} with 'domcontentloaded' waitUntil`,
             );
             await page.goto(action.url, { waitUntil: 'domcontentloaded' });
-            const content = await page.content();
+            await sleep(3000);
+            const content = await getRenderedDOM(page);
             logger.verbose(
               'Navigation completed with domcontentloaded strategy',
             );
@@ -128,7 +131,8 @@ export const browseMessageTool: Tool<Parameters, ReturnType> = {
 
             try {
               await page.goto(action.url);
-              const content = await page.content();
+              await sleep(3000);
+              const content = await getRenderedDOM(page);
               logger.verbose('Navigation completed with basic strategy');
               logger.verbose(`Content: ${content}`);
               return { status: 'success', content };

packages/agent/src/tools/browser/browseStart.ts

@@ -5,7 +5,9 @@ import { zodToJsonSchema } from 'zod-to-json-schema';
 
 import { Tool } from '../../core/types.js';
 import { errorToString } from '../../utils/errorToString.js';
+import { sleep } from '../../utils/sleep.js';
 
+import { getRenderedDOM } from './getRenderedDOM.js';
 import { browserSessions } from './types.js';
 
 const parameterSchema = z.object({
@@ -99,7 +101,8 @@ export const browseStartTool: Tool<Parameters, ReturnType> = {
             `Navigating to ${url} with 'domcontentloaded' waitUntil`,
           );
           await page.goto(url, { waitUntil: 'domcontentloaded', timeout });
-          content = await page.content();
+          await sleep(3000);
+          content = await getRenderedDOM(page);
           logger.verbose('Navigation completed with domcontentloaded strategy');
         } catch (error) {
           // If that fails, try with no waitUntil option at all (most basic)
@@ -112,7 +115,8 @@ export const browseStartTool: Tool<Parameters, ReturnType> = {
 
           try {
             await page.goto(url, { timeout });
-            content = await page.content();
+            await sleep(3000);
+            content = await getRenderedDOM(page);
             logger.verbose('Navigation completed with basic strategy');
           } catch (innerError) {
             logger.error(

packages/agent/src/tools/browser/getRenderedDOM.ts

@@ -0,0 +1,45 @@
+import { Page } from 'playwright';
+
+export async function getRenderedDOM(page: Page): Promise<string> {
+  // Get the serialized HTML after JavaScript execution
+  const domContent = await page.evaluate(() => {
+    // Clone the document to avoid modifying the original
+    const clone = document.documentElement; //.cloneNode(true) as HTMLElement;
+
+    // for each element, get computed styles to determine visibility, and remove if not visible
+    const elements = clone.querySelectorAll('*');
+
+    const elementsToRemove: Element[] = [];
+    elements.forEach((element) => {
+      const computedStyle = window.getComputedStyle(element);
+      const isVisible =
+        computedStyle.display !== 'none' &&
+        computedStyle.visibility !== 'hidden' &&
+        computedStyle.opacity !== '0';
+
+      if (!isVisible) {
+        elementsToRemove.push(element);
+      }
+    });
+
+    // Optional: Remove other non-visual elements
+    const nonVisualTags = clone.querySelectorAll(
+      'noscript, iframe, link[rel="stylesheet"], meta, svg, img, symbol, path, style, script',
+    );
+    nonVisualTags.forEach((element) => elementsToRemove.push(element));
+
+    elementsToRemove.forEach((element) => element.remove());
+
+    console.log(
+      'removing ',
+      elementsToRemove.length,
+      ' elements out of a total ',
+      elements.length,
+    );
+
+    return clone.outerHTML;
+  });
+
+  // remove all newlines using a regex, and also collapse multiple spaces into one
+  return domContent.replace(/\n/g, '').replace(/\s+/g, ' ');
+}