show grid overlay

Author: SentienceDEV

examples/show_grid_examples.py

@@ -0,0 +1,117 @@
+"""
+Example: Grid Overlay Visualization
+
+Demonstrates how to use the grid overlay feature to visualize detected grids
+on a webpage, including highlighting specific grids and identifying the dominant group.
+"""
+
+import os
+import time
+
+from sentience import SentienceBrowser, snapshot
+from sentience.models import SnapshotOptions
+
+
+def main():
+    # Get API key from environment variable (optional - uses free tier if not set)
+    api_key = os.environ.get("SENTIENCE_API_KEY")
+
+    try:
+        with SentienceBrowser(api_key=api_key, headless=False) as browser:
+            # Navigate to a page with grid layouts (e.g., product listings, article feeds)
+            browser.page.goto("https://example.com/products", wait_until="domcontentloaded")
+            time.sleep(2)  # Wait for page to fully load
+
+            print("=" * 60)
+            print("Example 1: Show all detected grids")
+            print("=" * 60)
+            # Show all grids (all in purple)
+            snap = snapshot(browser, SnapshotOptions(show_grid=True))
+            print(f"✅ Found {len(snap.elements)} elements")
+            print("   Purple borders appear around all detected grids for 5 seconds")
+            time.sleep(6)  # Wait to see the overlay
+
+            print("\n" + "=" * 60)
+            print("Example 2: Highlight a specific grid in red")
+            print("=" * 60)
+            # Get grid information first
+            grids = snap.get_grid_bounds()
+            if grids:
+                print(f"✅ Found {len(grids)} grids:")
+                for grid in grids:
+                    print(f"   Grid {grid.grid_id}: {grid.item_count} items, "
+                          f"{grid.row_count}x{grid.col_count} rows/cols, "
+                          f"label: {grid.label or 'none'}")
+                
+                # Highlight the first grid in red
+                if len(grids) > 0:
+                    target_grid_id = grids[0].grid_id
+                    print(f"\n   Highlighting Grid {target_grid_id} in red...")
+                    snap = snapshot(browser, SnapshotOptions(
+                        show_grid=True,
+                        grid_id=target_grid_id  # This grid will be highlighted in red
+                    ))
+                    time.sleep(6)  # Wait to see the overlay
+            else:
+                print("   ⚠️  No grids detected on this page")
+
+            print("\n" + "=" * 60)
+            print("Example 3: Highlight the dominant group")
+            print("=" * 60)
+            # Find and highlight the dominant grid
+            grids = snap.get_grid_bounds()
+            dominant_grid = next((g for g in grids if g.is_dominant), None)
+            
+            if dominant_grid:
+                print(f"✅ Dominant group detected: Grid {dominant_grid.grid_id}")
+                print(f"   Label: {dominant_grid.label or 'none'}")
+                print(f"   Items: {dominant_grid.item_count}")
+                print(f"   Size: {dominant_grid.row_count}x{dominant_grid.col_count}")
+                print(f"\n   Highlighting dominant grid in red...")
+                snap = snapshot(browser, SnapshotOptions(
+                    show_grid=True,
+                    grid_id=dominant_grid.grid_id  # Highlight dominant grid in red
+                ))
+                time.sleep(6)  # Wait to see the overlay
+            else:
+                print("   ⚠️  No dominant group detected")
+
+            print("\n" + "=" * 60)
+            print("Example 4: Combine element overlay and grid overlay")
+            print("=" * 60)
+            # Show both element borders and grid borders simultaneously
+            snap = snapshot(browser, SnapshotOptions(
+                show_overlay=True,  # Show element borders (green/blue/red)
+                show_grid=True      # Show grid borders (purple/orange/red)
+            ))
+            print("✅ Both overlays are now visible:")
+            print("   - Element borders: Green (regular), Blue (primary), Red (target)")
+            print("   - Grid borders: Purple (regular), Orange (dominant), Red (target)")
+            time.sleep(6)  # Wait to see the overlay
+
+            print("\n" + "=" * 60)
+            print("Example 5: Grid information analysis")
+            print("=" * 60)
+            # Analyze grid structure
+            grids = snap.get_grid_bounds()
+            print(f"✅ Grid Analysis:")
+            for grid in grids:
+                dominant_indicator = "⭐ DOMINANT" if grid.is_dominant else ""
+                print(f"\n   Grid {grid.grid_id} {dominant_indicator}:")
+                print(f"      Label: {grid.label or 'none'}")
+                print(f"      Items: {grid.item_count}")
+                print(f"      Size: {grid.row_count} rows × {grid.col_count} cols")
+                print(f"      BBox: ({grid.bbox.x:.0f}, {grid.bbox.y:.0f}) "
+                      f"{grid.bbox.width:.0f}×{grid.bbox.height:.0f}")
+                print(f"      Confidence: {grid.confidence}")
+
+            print("\n✅ All examples completed!")
+
+    except Exception as e:
+        print(f"❌ Error: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()

sentience/models.py

@@ -118,6 +118,7 @@ class GridInfo(BaseModel):
     label: str | None = (
         None  # Optional inferred label (e.g., "product_grid", "search_results", "navigation")
     )
+    is_dominant: bool = False  # Whether this grid is the dominant group (main content area)
 
 
 class Snapshot(BaseModel):
@@ -190,10 +191,16 @@ def get_grid_bounds(self, grid_id: int | None = None) -> list[GridInfo]:
 
         grid_infos = []
 
+        # First pass: compute all grid infos and count dominant group elements
+        grid_dominant_counts = {}
         for gid, elements_in_grid in sorted(grid_elements.items()):
             if not elements_in_grid:
                 continue
 
+            # Count dominant group elements in this grid
+            dominant_count = sum(1 for elem in elements_in_grid if elem.in_dominant_group is True)
+            grid_dominant_counts[gid] = (dominant_count, len(elements_in_grid))
+
             # Compute bounding box
             min_x = min(elem.bbox.x for elem in elements_in_grid)
             min_y = min(elem.bbox.y for elem in elements_in_grid)
@@ -226,9 +233,42 @@ def get_grid_bounds(self, grid_id: int | None = None) -> list[GridInfo]:
                     item_count=len(elements_in_grid),
                     confidence=1.0,
                     label=label,
+                    is_dominant=False,  # Will be set below
                 )
             )
 
+        # Second pass: identify dominant grid
+        # The grid with the highest count (or highest percentage >= 50%) of dominant group elements
+        if grid_dominant_counts:
+            # Find grid with highest absolute count
+            max_dominant_count = max(count for count, _ in grid_dominant_counts.values())
+            if max_dominant_count > 0:
+                # Find grid(s) with highest count
+                dominant_grids = [
+                    gid
+                    for gid, (count, total) in grid_dominant_counts.items()
+                    if count == max_dominant_count
+                ]
+                # If multiple grids tie, prefer the one with highest percentage
+                if len(dominant_grids) > 1:
+                    dominant_grids.sort(
+                        key=lambda gid: (
+                            grid_dominant_counts[gid][0] / grid_dominant_counts[gid][1]
+                            if grid_dominant_counts[gid][1] > 0
+                            else 0
+                        ),
+                        reverse=True,
+                    )
+                # Mark the dominant grid
+                dominant_gid = dominant_grids[0]
+                # Only mark as dominant if it has >= 50% dominant group elements or >= 3 elements
+                dominant_count, total_count = grid_dominant_counts[dominant_gid]
+                if dominant_count >= 3 or (total_count > 0 and dominant_count / total_count >= 0.5):
+                    for grid_info in grid_infos:
+                        if grid_info.grid_id == dominant_gid:
+                            grid_info.is_dominant = True
+                            break
+
         return grid_infos
 
     @staticmethod
@@ -456,6 +496,10 @@ class SnapshotOptions(BaseModel):
     trace_path: str | None = None  # Path to save trace (default: "trace_{timestamp}.json")
     goal: str | None = None  # Optional goal/task description for the snapshot
     show_overlay: bool = False  # Show visual overlay highlighting elements in browser
+    show_grid: bool = False  # Show visual overlay highlighting detected grids
+    grid_id: int | None = (
+        None  # Optional grid ID to show specific grid (only used if show_grid=True)
+    )
 
     # API credentials (for browser-use integration without SentienceBrowser)
     sentience_api_key: str | None = None  # Sentience API key for Pro/Enterprise features

sentience/snapshot.py

@@ -250,6 +250,9 @@ def _snapshot_via_extension(
     if options.save_trace:
         _save_trace_to_file(result.get("raw_elements", []), options.trace_path)
 
+    # Validate and parse with Pydantic
+    snapshot_obj = Snapshot(**result)
+
     # Show visual overlay if requested
     if options.show_overlay:
         raw_elements = result.get("raw_elements", [])
@@ -265,8 +268,29 @@ def _snapshot_via_extension(
                 raw_elements,
             )
 
-    # Validate and parse with Pydantic
-    snapshot_obj = Snapshot(**result)
+    # Show grid overlay if requested
+    if options.show_grid:
+        # Get all grids (don't filter by grid_id here - we want to show all but highlight the target)
+        grids = snapshot_obj.get_grid_bounds(grid_id=None)
+        if grids:
+            # Convert GridInfo to dict for JavaScript
+            grid_dicts = [grid.model_dump() for grid in grids]
+            # Pass grid_id as targetGridId to highlight it in red
+            target_grid_id = options.grid_id if options.grid_id is not None else None
+            browser.page.evaluate(
+                """
+                (grids, targetGridId) => {
+                    if (window.sentience && window.sentience.showGrid) {
+                        window.sentience.showGrid(grids, targetGridId);
+                    } else {
+                        console.warn('[SDK] showGrid not available in extension');
+                    }
+                }
+                """,
+                grid_dicts,
+                target_grid_id,
+            )
+
     return snapshot_obj
 
 
@@ -308,6 +332,9 @@ def _snapshot_via_api(
         # Merge API result with local data (screenshot, etc.)
         snapshot_data = _merge_api_result_with_local(api_result, raw_result)
 
+        # Create snapshot object
+        snapshot_obj = Snapshot(**snapshot_data)
+
         # Show visual overlay if requested (use API-ranked elements)
         if options.show_overlay:
             elements = api_result.get("elements", [])
@@ -323,7 +350,29 @@ def _snapshot_via_api(
                     elements,
                 )
 
-        return Snapshot(**snapshot_data)
+        # Show grid overlay if requested
+        if options.show_grid:
+            # Get all grids (don't filter by grid_id here - we want to show all but highlight the target)
+            grids = snapshot_obj.get_grid_bounds(grid_id=None)
+            if grids:
+                grid_dicts = [grid.model_dump() for grid in grids]
+                # Pass grid_id as targetGridId to highlight it in red
+                target_grid_id = options.grid_id if options.grid_id is not None else None
+                browser.page.evaluate(
+                    """
+                    (grids, targetGridId) => {
+                        if (window.sentience && window.sentience.showGrid) {
+                            window.sentience.showGrid(grids, targetGridId);
+                        } else {
+                            console.warn('[SDK] showGrid not available in extension');
+                        }
+                    }
+                    """,
+                    grid_dicts,
+                    target_grid_id,
+                )
+
+        return snapshot_obj
     except requests.exceptions.RequestException as e:
         raise RuntimeError(f"API request failed: {e}") from e
 
@@ -440,6 +489,18 @@ async def _snapshot_via_extension_async(
     if options.save_trace:
         _save_trace_to_file(result.get("raw_elements", []), options.trace_path)
 
+    # Extract screenshot_format from data URL if not provided by extension
+    if result.get("screenshot") and not result.get("screenshot_format"):
+        screenshot_data_url = result.get("screenshot", "")
+        if screenshot_data_url.startswith("data:image/"):
+            # Extract format from "data:image/jpeg;base64,..." or "data:image/png;base64,..."
+            format_match = screenshot_data_url.split(";")[0].split("/")[-1]
+            if format_match in ["jpeg", "jpg", "png"]:
+                result["screenshot_format"] = "jpeg" if format_match in ["jpeg", "jpg"] else "png"
+
+    # Validate and parse with Pydantic
+    snapshot_obj = Snapshot(**result)
+
     # Show visual overlay if requested
     if options.show_overlay:
         raw_elements = result.get("raw_elements", [])
@@ -455,17 +516,28 @@ async def _snapshot_via_extension_async(
                 raw_elements,
             )
 
-    # Extract screenshot_format from data URL if not provided by extension
-    if result.get("screenshot") and not result.get("screenshot_format"):
-        screenshot_data_url = result.get("screenshot", "")
-        if screenshot_data_url.startswith("data:image/"):
-            # Extract format from "data:image/jpeg;base64,..." or "data:image/png;base64,..."
-            format_match = screenshot_data_url.split(";")[0].split("/")[-1]
-            if format_match in ["jpeg", "jpg", "png"]:
-                result["screenshot_format"] = "jpeg" if format_match in ["jpeg", "jpg"] else "png"
+    # Show grid overlay if requested
+    if options.show_grid:
+        # Get all grids (don't filter by grid_id here - we want to show all but highlight the target)
+        grids = snapshot_obj.get_grid_bounds(grid_id=None)
+        if grids:
+            grid_dicts = [grid.model_dump() for grid in grids]
+            # Pass grid_id as targetGridId to highlight it in red
+            target_grid_id = options.grid_id if options.grid_id is not None else None
+            await browser.page.evaluate(
+                """
+                (grids, targetGridId) => {
+                    if (window.sentience && window.sentience.showGrid) {
+                        window.sentience.showGrid(grids, targetGridId);
+                    } else {
+                        console.warn('[SDK] showGrid not available in extension');
+                    }
+                }
+                """,
+                grid_dicts,
+                target_grid_id,
+            )
 
-    # Validate and parse with Pydantic
-    snapshot_obj = Snapshot(**result)
     return snapshot_obj
 
 
@@ -584,6 +656,9 @@ async def _snapshot_via_api_async(
             "error": api_result.get("error"),
         }
 
+        # Create snapshot object
+        snapshot_obj = Snapshot(**snapshot_data)
+
         # Show visual overlay if requested
         if options.show_overlay:
             elements = api_result.get("elements", [])
@@ -599,7 +674,29 @@ async def _snapshot_via_api_async(
                     elements,
                 )
 
-        return Snapshot(**snapshot_data)
+        # Show grid overlay if requested
+        if options.show_grid:
+            # Get all grids (don't filter by grid_id here - we want to show all but highlight the target)
+            grids = snapshot_obj.get_grid_bounds(grid_id=None)
+            if grids:
+                grid_dicts = [grid.model_dump() for grid in grids]
+                # Pass grid_id as targetGridId to highlight it in red
+                target_grid_id = options.grid_id if options.grid_id is not None else None
+                await browser.page.evaluate(
+                    """
+                    (grids, targetGridId) => {
+                        if (window.sentience && window.sentience.showGrid) {
+                            window.sentience.showGrid(grids, targetGridId);
+                        } else {
+                            console.warn('[SDK] showGrid not available in extension');
+                        }
+                    }
+                    """,
+                    grid_dicts,
+                    target_grid_id,
+                )
+
+        return snapshot_obj
     except ImportError:
         # Fallback to requests if httpx not available (shouldn't happen in async context)
         raise RuntimeError(