CDP Mode: Patch 68

examples/cdp_mode/ReadMe.md

@@ -445,7 +445,8 @@ sb.cdp.get_text(selector)
 sb.cdp.get_title()
 sb.cdp.get_current_url()
 sb.cdp.get_origin()
-sb.cdp.get_page_source()
+sb.cdp.get_html(include_shadow_dom=True)
+sb.cdp.get_page_source(include_shadow_dom=True)
 sb.cdp.get_user_agent()
 sb.cdp.get_cookie_string()
 sb.cdp.get_locale_code()

examples/cdp_mode/raw_cdp_shadow.py

@@ -0,0 +1,14 @@
+"""An example of displaying Shadow DOM inside HTML"""
+from seleniumbase import sb_cdp
+
+url = "https://seleniumbase.io/apps/turnstile"
+sb = sb_cdp.Chrome(url)
+element = sb.find_element("div.cf-turnstile div")
+html_with_shadow_dom = element.get_html()
+print(html_with_shadow_dom)
+text_to_find = "Widget containing a Cloudflare security challenge"
+sb.assert_true(text_to_find in html_with_shadow_dom)
+sb.solve_captcha()
+sb.assert_element("img#captcha-success", timeout=3)
+sb.set_messenger_theme(location="top_left")
+sb.post_message("SeleniumBase wasn't detected", duration=3)

examples/cdp_mode/raw_handle_alerts.py

@@ -1,17 +1,16 @@
-"""To handle alerts in CDP Mode, reconnect and use WebDriver."""
+"""An example of handling alerts in CDP Mode."""
 from seleniumbase import SB
 
 with SB(uc=True, test=True) as sb:
     url = "https://the-internet.herokuapp.com/javascript_alerts"
     sb.activate_cdp_mode(url)
-    sb.reconnect()
     sb.cdp.gui_click_element('button[onclick="jsAlert()"]')
     sb.sleep(1)
-    sb.accept_alert()
+    sb.uc_gui_press_key("\n")  # Accept Alert
     sb.sleep(1)
     sb.cdp.gui_click_element('button[onclick="jsConfirm()"]')
     sb.sleep(1)
-    sb.dismiss_alert()
+    sb.uc_gui_press_key("ESC")  # Dismiss Alert
     sb.sleep(1)
     sb.cdp.gui_click_element('button[onclick="jsPrompt()"]')
     sb.sleep(1)

help_docs/method_summary.md

@@ -40,6 +40,7 @@ self.refresh()
 # Duplicates: self.refresh_page(), self.reload_page(), self.reload()
 self.get_current_url()
 self.get_origin()
+self.get_html()
 self.get_page_source()
 self.get_title()
 # Duplicates: self.get_page_title()

mkdocs_build/requirements.txt

@@ -14,7 +14,7 @@ pathspec==0.12.1
 Babel==2.17.0
 paginate==0.5.7
 mkdocs==1.6.1
-mkdocs-material==9.6.22
+mkdocs-material==9.6.23
 mkdocs-exclude-search==0.6.6
 mkdocs-simple-hooks==0.1.5
 mkdocs-material-extensions==1.3.1

requirements.txt

@@ -14,7 +14,7 @@ filelock~=3.16.1;python_version<"3.9"
 filelock~=3.19.1;python_version>="3.9" and python_version<"3.10"
 filelock>=3.20.0;python_version>="3.10"
 fasteners>=0.20
-mycdp>=1.2.1
+mycdp>=1.3.0
 pynose>=1.5.5
 platformdirs~=4.3.6;python_version<"3.9"
 platformdirs~=4.4.0;python_version>="3.9" and python_version<"3.10"
@@ -45,7 +45,8 @@ sniffio==1.3.1
 h11==0.16.0
 outcome==1.3.0.post0
 trio==0.27.0;python_version<"3.9"
-trio>=0.31.0,<1;python_version>="3.9"
+trio>=0.31.0,<1;python_version>="3.9" and python_version<"3.10"
+trio>=0.32.0,<1;python_version>="3.10"
 trio-websocket~=0.12.2
 wsproto==1.2.0
 websocket-client~=1.8.0;python_version<"3.9"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.44.3"
+__version__ = "4.44.4"

seleniumbase/core/browser_launcher.py

@@ -295,14 +295,14 @@ def extend_driver(
         driver.set_wire_proxy = DM.set_wire_proxy
     completed_loads = []
     for ext_dir in sb_config._ext_dirs:
-        with suppress(Exception):
-            if ext_dir not in completed_loads:
-                completed_loads.append(ext_dir)
-                if not use_uc and os.path.exists(os.path.abspath(ext_dir)):
-                    driver.webextension.install(os.path.abspath(ext_dir))
+        if ext_dir not in completed_loads:
+            completed_loads.append(ext_dir)
+            if not use_uc and os.path.exists(os.path.realpath(ext_dir)):
+                with suppress(Exception):
+                    driver.webextension.install(os.path.realpath(ext_dir))
     if proxy_auth:
-        with suppress(Exception):
-            if not use_uc and os.path.exists(proxy_helper.PROXY_DIR_PATH):
+        if not use_uc and os.path.exists(proxy_helper.PROXY_DIR_PATH):
+            with suppress(Exception):
                 driver.webextension.install(proxy_helper.PROXY_DIR_PATH)
         # Proxy needs a moment to load in Manifest V3
         if use_uc:
@@ -838,6 +838,7 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs):
     cdp.get_element_position = CDPM.get_element_position
     cdp.get_gui_element_rect = CDPM.get_gui_element_rect
     cdp.get_gui_element_center = CDPM.get_gui_element_center
+    cdp.get_html = CDPM.get_html
     cdp.get_page_source = CDPM.get_page_source
     cdp.get_user_agent = CDPM.get_user_agent
     cdp.get_cookie_string = CDPM.get_cookie_string
@@ -934,6 +935,7 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs):
     cdp.core = core_items
     cdp.loop = cdp.get_event_loop()
     driver.cdp = cdp
+    driver.solve_captcha = CDPM.solve_captcha
     driver._is_using_cdp = True
 
 
@@ -2477,7 +2479,7 @@ def _set_chrome_options(
         # Can be a comma-separated list of .ZIP or .CRX files
         extension_zip_list = extension_zip.split(",")
         for extension_zip_item in extension_zip_list:
-            abs_path = os.path.abspath(extension_zip_item)
+            abs_path = os.path.realpath(extension_zip_item)
             if os.path.exists(abs_path):
                 try:
                     abs_path_dir = os.path.join(
@@ -2494,11 +2496,11 @@ def _set_chrome_options(
     if extension_dir:
         # load-extension input can be a comma-separated list
         abs_path = (
-            ",".join(os.path.abspath(p) for p in extension_dir.split(","))
+            ",".join(os.path.realpath(p) for p in extension_dir.split(","))
         )
         chrome_options = add_chrome_ext_dir(chrome_options, abs_path)
         for p in extension_dir.split(","):
-            sb_config._ext_dirs.append(os.path.abspath(p))
+            sb_config._ext_dirs.append(os.path.realpath(p))
     if (
         page_load_strategy
         and page_load_strategy.lower() in ["eager", "none"]
@@ -2742,10 +2744,12 @@ def _set_chrome_options(
             included_disabled_features.append(item)
     d_f_string = ",".join(included_disabled_features)
     chrome_options.add_argument("--disable-features=%s" % d_f_string)
+    chrome_options.add_argument("--enable-unsafe-extension-debugging")
     if proxy_auth:
         chrome_options.add_argument("--test-type")
     if proxy_auth or sb_config._ext_dirs:
         if not is_using_uc(undetectable, browser_name):
+            chrome_options.add_argument("--remote-debugging-pipe")
             chrome_options.enable_webextensions = True
             chrome_options.enable_bidi = True
     if (
@@ -4577,12 +4581,12 @@ def get_local_driver(
             # Can be a comma-separated list of .ZIP or .CRX files
             extension_zip_list = extension_zip.split(",")
             for extension_zip_item in extension_zip_list:
-                abs_path = os.path.abspath(extension_zip_item)
+                abs_path = os.path.realpath(extension_zip_item)
                 edge_options.add_extension(abs_path)
         if extension_dir:
             # load-extension input can be a comma-separated list
             abs_path = (
-                ",".join(os.path.abspath(p) for p in extension_dir.split(","))
+                ",".join(os.path.realpath(p) for p in extension_dir.split(","))
             )
             edge_options = add_chrome_ext_dir(edge_options, abs_path)
         edge_options.add_argument("--disable-infobars")

seleniumbase/core/sb_cdp.py

@@ -1185,7 +1185,14 @@ def get_origin(self):
             self.page.evaluate("window.location.origin")
         )
 
-    def get_page_source(self):
+    def get_html(self, include_shadow_dom=True):
+        return self.get_page_source(
+            include_shadow_dom=include_shadow_dom
+        )
+
+    def get_page_source(self, include_shadow_dom=True):
+        if include_shadow_dom:
+            return self.find_element("html").get_html()
         try:
             source = self.loop.run_until_complete(
                 self.page.evaluate("document.documentElement.outerHTML")

seleniumbase/fixtures/base_case.py

@@ -1293,8 +1293,11 @@ def get_origin(self):
         self.__check_scope()
         return self.execute_script("return window.location.origin;")
 
-    def get_page_source(self):
-        if self.__is_cdp_swap_needed():
+    def get_html(self, *args, **kwargs):
+        return self.get_page_source(*args, **kwargs)
+
+    def get_page_source(self, *args, **kwargs):
+        if self.__is_cdp_swap_needed(*args, **kwargs):
             return self.cdp.get_page_source()
         self.wait_for_ready_state_complete()
         if self.__needs_minimum_wait:

seleniumbase/undetected/cdp_driver/config.py

@@ -130,6 +130,7 @@ def __init__(
             "--no-default-browser-check",
             "--homepage=about:blank",
             "--no-pings",
+            "--enable-unsafe-extension-debugging",
             "--wm-window-animations-disabled",
             "--animation-duration-scale=0",
             "--enable-privacy-sandbox-ads-apis",

seleniumbase/undetected/cdp_driver/element.py

@@ -806,7 +806,10 @@ async def set_text_async(self, value):
 
     async def get_html_async(self):
         return await self._tab.send(
-            cdp.dom.get_outer_html(backend_node_id=self.backend_node_id)
+            cdp.dom.get_outer_html(
+                backend_node_id=self.backend_node_id,
+                include_shadow_dom=True,
+            )
         )
 
     @property

seleniumbase/undetected/cdp_driver/tab.py

@@ -899,7 +899,10 @@ async def get_content(self):
         """Gets the current page source content (html)"""
         doc: cdp.dom.Node = await self.send(cdp.dom.get_document(-1, True))
         return await self.send(
-            cdp.dom.get_outer_html(backend_node_id=doc.backend_node_id)
+            cdp.dom.get_outer_html(
+                backend_node_id=doc.backend_node_id,
+                include_shadow_dom=True,
+            )
         )
 
     async def maximize(self):

setup.py

@@ -163,7 +163,7 @@
         'filelock~=3.19.1;python_version>="3.9" and python_version<"3.10"',
         'filelock>=3.20.0;python_version>="3.10"',
         'fasteners>=0.20',
-        "mycdp>=1.2.1",
+        "mycdp>=1.3.0",
         "pynose>=1.5.5",
         'platformdirs~=4.3.6;python_version<"3.9"',
         'platformdirs~=4.4.0;python_version>="3.9" and python_version<"3.10"',
@@ -194,7 +194,8 @@
         'h11==0.16.0',
         'outcome==1.3.0.post0',
         'trio==0.27.0;python_version<"3.9"',
-        'trio>=0.31.0,<1;python_version>="3.9"',
+        'trio>=0.31.0,<1;python_version>="3.9" and python_version<"3.10"',
+        'trio>=0.32.0,<1;python_version>="3.10"',
         'trio-websocket~=0.12.2',
         'wsproto==1.2.0',
         'websocket-client~=1.8.0;python_version<"3.9"',