more robust browser usage, allow using user's session.

bhouston · bhouston · commit fe1d1b181e40 · 2025-02-27T13:50:16.000-05:00
diff --git a/packages/agent/src/core/types.ts b/packages/agent/src/core/types.ts
@@ -10,6 +10,7 @@ export type ToolContext = {
   logger: Logger;
   workingDirectory: string;
   headless: boolean;
+  userSession?: boolean;
   tokenTracker: TokenTracker;
 };
 
diff --git a/packages/agent/src/tools/browser/browseMessage.ts b/packages/agent/src/tools/browser/browseMessage.ts
@@ -72,6 +72,23 @@ export const browseMessageTool: Tool<Parameters, ReturnType> = {
   returns: zodToJsonSchema(returnSchema),
 
   execute: async ({ instanceId, action }, { logger }): Promise<ReturnType> => {
+    // Validate action format
+    if (!action || typeof action !== 'object') {
+      logger.error('Invalid action format: action must be an object');
+      return {
+        status: 'error',
+        error: 'Invalid action format: action must be an object',
+      };
+    }
+
+    if (!action.actionType) {
+      logger.error('Invalid action format: actionType is required');
+      return {
+        status: 'error',
+        error: 'Invalid action format: actionType is required',
+      };
+    }
+
     logger.verbose(`Executing browser action: ${action.actionType}`);
 
     try {
@@ -87,10 +104,41 @@ export const browseMessageTool: Tool<Parameters, ReturnType> = {
           if (!action.url) {
             throw new Error('URL required for goto action');
           }
-          await page.goto(action.url, { waitUntil: 'networkidle' });
-          const content = await page.content();
-          logger.verbose('Navigation completed successfully');
-          return { status: 'success', content };
+
+          try {
+            // Try with 'domcontentloaded' first which is more reliable than 'networkidle'
+            logger.verbose(
+              `Navigating to ${action.url} with 'domcontentloaded' waitUntil`,
+            );
+            await page.goto(action.url, { waitUntil: 'domcontentloaded' });
+            const content = await page.content();
+            logger.verbose(
+              'Navigation completed with domcontentloaded strategy',
+            );
+            logger.verbose(`Content: ${content}`);
+            return { status: 'success', content };
+          } catch (navError) {
+            // If that fails, try with no waitUntil option
+            logger.warn(
+              `Failed with domcontentloaded strategy: ${errorToString(navError)}`,
+            );
+            logger.verbose(
+              `Retrying navigation to ${action.url} with no waitUntil option`,
+            );
+
+            try {
+              await page.goto(action.url);
+              const content = await page.content();
+              logger.verbose('Navigation completed with basic strategy');
+              logger.verbose(`Content: ${content}`);
+              return { status: 'success', content };
+            } catch (innerError) {
+              logger.error(
+                `Failed with basic navigation strategy: ${errorToString(innerError)}`,
+              );
+              throw innerError; // Re-throw to be caught by outer catch block
+            }
+          }
         }
 
         case 'click': {
diff --git a/packages/agent/src/tools/browser/browseStart.ts b/packages/agent/src/tools/browser/browseStart.ts
@@ -40,23 +40,34 @@ export const browseStartTool: Tool<Parameters, ReturnType> = {
 
   execute: async (
     { url, timeout = 30000 },
-    { logger, headless = true },
+    { logger, headless = true, userSession = false },
   ): Promise<ReturnType> => {
     logger.verbose(`Starting browser session${url ? ` at ${url}` : ''}`);
+    logger.verbose(`User session mode: ${userSession ? 'enabled' : 'disabled'}`);
 
     try {
       const instanceId = uuidv4();
 
       // Launch browser
-      const browser = await chromium.launch({
+      const launchOptions = {
         headless,
-      });
+      };
+
+      // Use system Chrome installation if userSession is true
+      if (userSession) {
+        logger.verbose('Using system Chrome installation');
+        // For Chrome, we use the channel option to specify Chrome
+        launchOptions['channel'] = 'chrome';
+      }
+
+      const browser = await chromium.launch(launchOptions);
 
       // Create new context with default settings
       const context = await browser.newContext({
         viewport: null,
         userAgent:
           'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        serviceWorkers: 'block', // Block service workers which can cause continuous network activity
       });
 
       // Create new page
@@ -80,11 +91,38 @@ export const browseStartTool: Tool<Parameters, ReturnType> = {
       // Navigate to URL if provided
       let content = '';
       if (url) {
-        await page.goto(url, { waitUntil: 'networkidle' });
-        content = await page.content();
+        try {
+          // Try with 'domcontentloaded' first which is more reliable than 'networkidle'
+          logger.verbose(
+            `Navigating to ${url} with 'domcontentloaded' waitUntil`,
+          );
+          await page.goto(url, { waitUntil: 'domcontentloaded', timeout });
+          content = await page.content();
+          logger.verbose('Navigation completed with domcontentloaded strategy');
+        } catch (error) {
+          // If that fails, try with no waitUntil option at all (most basic)
+          logger.warn(
+            `Failed with domcontentloaded strategy: ${errorToString(error)}`,
+          );
+          logger.verbose(
+            `Retrying navigation to ${url} with no waitUntil option`,
+          );
+
+          try {
+            await page.goto(url, { timeout });
+            content = await page.content();
+            logger.verbose('Navigation completed with basic strategy');
+          } catch (innerError) {
+            logger.error(
+              `Failed with basic navigation strategy: ${errorToString(innerError)}`,
+            );
+            throw innerError; // Re-throw to be caught by outer catch block
+          }
+        }
       }
 
       logger.verbose('Browser session started successfully');
+      logger.verbose(`Content: ${content}`);
 
       return {
         instanceId,
diff --git a/packages/cli/README.md b/packages/cli/README.md
@@ -54,6 +54,8 @@ mycoder --promptFile=your-prompt.txt
 - `-f, --file`: Read prompt from a specified file
 - `--log`: Set log level (info, verbose, warn, error)
 - `--tokenUsage`: Output token usage at info log level
+- `--headless`: Use browser in headless mode with no UI showing (default: true)
+- `--userSession`: Use user's existing browser session instead of sandboxed session (default: false)
 - `-h, --help`: Show help
 - `-V, --version`: Show version
 
diff --git a/packages/cli/src/commands/$default.ts b/packages/cli/src/commands/$default.ts
@@ -128,6 +128,7 @@ export const command: CommandModule<SharedOptions, DefaultArgs> = {
       const result = await toolAgent(prompt, tools, undefined, {
         logger,
         headless: argv.headless ?? true,
+        userSession: argv.userSession ?? false,
         workingDirectory: '.',
         tokenTracker,
       });
diff --git a/packages/cli/src/options.ts b/packages/cli/src/options.ts
@@ -4,6 +4,7 @@ export type SharedOptions = {
   readonly file?: string;
   readonly tokenUsage?: boolean;
   readonly headless?: boolean;
+  readonly userSession?: boolean;
 };
 
 export const sharedOptions = {
@@ -35,4 +36,9 @@ export const sharedOptions = {
     description: 'Use browser in headless mode with no UI showing',
     default: true,
   } as const,
+  userSession: {
+    type: 'boolean',
+    description: 'Use user\'s existing browser session instead of sandboxed session',
+    default: false,
+  } as const,
 };
