diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index cedc13dd069..5d35dda1d2e 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -445,7 +445,8 @@ sb.cdp.get_text(selector) sb.cdp.get_title() sb.cdp.get_current_url() sb.cdp.get_origin() -sb.cdp.get_page_source() +sb.cdp.get_html(include_shadow_dom=True) +sb.cdp.get_page_source(include_shadow_dom=True) sb.cdp.get_user_agent() sb.cdp.get_cookie_string() sb.cdp.get_locale_code() diff --git a/examples/cdp_mode/raw_cdp_shadow.py b/examples/cdp_mode/raw_cdp_shadow.py new file mode 100644 index 00000000000..60a581c19ff --- /dev/null +++ b/examples/cdp_mode/raw_cdp_shadow.py @@ -0,0 +1,14 @@ +"""An example of displaying Shadow DOM inside HTML""" +from seleniumbase import sb_cdp + +url = "https://seleniumbase.io/apps/turnstile" +sb = sb_cdp.Chrome(url) +element = sb.find_element("div.cf-turnstile div") +html_with_shadow_dom = element.get_html() +print(html_with_shadow_dom) +text_to_find = "Widget containing a Cloudflare security challenge" +sb.assert_true(text_to_find in html_with_shadow_dom) +sb.solve_captcha() +sb.assert_element("img#captcha-success", timeout=3) +sb.set_messenger_theme(location="top_left") +sb.post_message("SeleniumBase wasn't detected", duration=3) diff --git a/examples/cdp_mode/raw_handle_alerts.py b/examples/cdp_mode/raw_handle_alerts.py index 55cd582e30c..098dcc57298 100644 --- a/examples/cdp_mode/raw_handle_alerts.py +++ b/examples/cdp_mode/raw_handle_alerts.py @@ -1,17 +1,16 @@ -"""To handle alerts in CDP Mode, reconnect and use WebDriver.""" +"""An example of handling alerts in CDP Mode.""" from seleniumbase import SB with SB(uc=True, test=True) as sb: url = "https://the-internet.herokuapp.com/javascript_alerts" sb.activate_cdp_mode(url) - sb.reconnect() sb.cdp.gui_click_element('button[onclick="jsAlert()"]') sb.sleep(1) - sb.accept_alert() + sb.uc_gui_press_key("\n") # Accept Alert sb.sleep(1) sb.cdp.gui_click_element('button[onclick="jsConfirm()"]') sb.sleep(1) - sb.dismiss_alert() + sb.uc_gui_press_key("ESC") # Dismiss Alert sb.sleep(1) sb.cdp.gui_click_element('button[onclick="jsPrompt()"]') sb.sleep(1) diff --git a/help_docs/method_summary.md b/help_docs/method_summary.md index 477c8e938b2..1e397998b9e 100644 --- a/help_docs/method_summary.md +++ b/help_docs/method_summary.md @@ -40,6 +40,7 @@ self.refresh() # Duplicates: self.refresh_page(), self.reload_page(), self.reload() self.get_current_url() self.get_origin() +self.get_html() self.get_page_source() self.get_title() # Duplicates: self.get_page_title() diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index 03b41036faa..01ba471b559 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -14,7 +14,7 @@ pathspec==0.12.1 Babel==2.17.0 paginate==0.5.7 mkdocs==1.6.1 -mkdocs-material==9.6.22 +mkdocs-material==9.6.23 mkdocs-exclude-search==0.6.6 mkdocs-simple-hooks==0.1.5 mkdocs-material-extensions==1.3.1 diff --git a/requirements.txt b/requirements.txt index 80b4a0d2cdc..9eab7011c98 100755 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ filelock~=3.16.1;python_version<"3.9" filelock~=3.19.1;python_version>="3.9" and python_version<"3.10" filelock>=3.20.0;python_version>="3.10" fasteners>=0.20 -mycdp>=1.2.1 +mycdp>=1.3.0 pynose>=1.5.5 platformdirs~=4.3.6;python_version<"3.9" platformdirs~=4.4.0;python_version>="3.9" and python_version<"3.10" @@ -45,7 +45,8 @@ sniffio==1.3.1 h11==0.16.0 outcome==1.3.0.post0 trio==0.27.0;python_version<"3.9" -trio>=0.31.0,<1;python_version>="3.9" +trio>=0.31.0,<1;python_version>="3.9" and python_version<"3.10" +trio>=0.32.0,<1;python_version>="3.10" trio-websocket~=0.12.2 wsproto==1.2.0 websocket-client~=1.8.0;python_version<"3.9" diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 9e31ff27d4f..c16c8036661 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.44.3" +__version__ = "4.44.4" diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 93380aaa1d5..ebdb023912d 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -295,14 +295,14 @@ def extend_driver( driver.set_wire_proxy = DM.set_wire_proxy completed_loads = [] for ext_dir in sb_config._ext_dirs: - with suppress(Exception): - if ext_dir not in completed_loads: - completed_loads.append(ext_dir) - if not use_uc and os.path.exists(os.path.abspath(ext_dir)): - driver.webextension.install(os.path.abspath(ext_dir)) + if ext_dir not in completed_loads: + completed_loads.append(ext_dir) + if not use_uc and os.path.exists(os.path.realpath(ext_dir)): + with suppress(Exception): + driver.webextension.install(os.path.realpath(ext_dir)) if proxy_auth: - with suppress(Exception): - if not use_uc and os.path.exists(proxy_helper.PROXY_DIR_PATH): + if not use_uc and os.path.exists(proxy_helper.PROXY_DIR_PATH): + with suppress(Exception): driver.webextension.install(proxy_helper.PROXY_DIR_PATH) # Proxy needs a moment to load in Manifest V3 if use_uc: @@ -838,6 +838,7 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs): cdp.get_element_position = CDPM.get_element_position cdp.get_gui_element_rect = CDPM.get_gui_element_rect cdp.get_gui_element_center = CDPM.get_gui_element_center + cdp.get_html = CDPM.get_html cdp.get_page_source = CDPM.get_page_source cdp.get_user_agent = CDPM.get_user_agent cdp.get_cookie_string = CDPM.get_cookie_string @@ -934,6 +935,7 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs): cdp.core = core_items cdp.loop = cdp.get_event_loop() driver.cdp = cdp + driver.solve_captcha = CDPM.solve_captcha driver._is_using_cdp = True @@ -2477,7 +2479,7 @@ def _set_chrome_options( # Can be a comma-separated list of .ZIP or .CRX files extension_zip_list = extension_zip.split(",") for extension_zip_item in extension_zip_list: - abs_path = os.path.abspath(extension_zip_item) + abs_path = os.path.realpath(extension_zip_item) if os.path.exists(abs_path): try: abs_path_dir = os.path.join( @@ -2494,11 +2496,11 @@ def _set_chrome_options( if extension_dir: # load-extension input can be a comma-separated list abs_path = ( - ",".join(os.path.abspath(p) for p in extension_dir.split(",")) + ",".join(os.path.realpath(p) for p in extension_dir.split(",")) ) chrome_options = add_chrome_ext_dir(chrome_options, abs_path) for p in extension_dir.split(","): - sb_config._ext_dirs.append(os.path.abspath(p)) + sb_config._ext_dirs.append(os.path.realpath(p)) if ( page_load_strategy and page_load_strategy.lower() in ["eager", "none"] @@ -2742,10 +2744,12 @@ def _set_chrome_options( included_disabled_features.append(item) d_f_string = ",".join(included_disabled_features) chrome_options.add_argument("--disable-features=%s" % d_f_string) + chrome_options.add_argument("--enable-unsafe-extension-debugging") if proxy_auth: chrome_options.add_argument("--test-type") if proxy_auth or sb_config._ext_dirs: if not is_using_uc(undetectable, browser_name): + chrome_options.add_argument("--remote-debugging-pipe") chrome_options.enable_webextensions = True chrome_options.enable_bidi = True if ( @@ -4577,12 +4581,12 @@ def get_local_driver( # Can be a comma-separated list of .ZIP or .CRX files extension_zip_list = extension_zip.split(",") for extension_zip_item in extension_zip_list: - abs_path = os.path.abspath(extension_zip_item) + abs_path = os.path.realpath(extension_zip_item) edge_options.add_extension(abs_path) if extension_dir: # load-extension input can be a comma-separated list abs_path = ( - ",".join(os.path.abspath(p) for p in extension_dir.split(",")) + ",".join(os.path.realpath(p) for p in extension_dir.split(",")) ) edge_options = add_chrome_ext_dir(edge_options, abs_path) edge_options.add_argument("--disable-infobars") diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index d7f6b3e779a..d565e438958 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -1185,7 +1185,14 @@ def get_origin(self): self.page.evaluate("window.location.origin") ) - def get_page_source(self): + def get_html(self, include_shadow_dom=True): + return self.get_page_source( + include_shadow_dom=include_shadow_dom + ) + + def get_page_source(self, include_shadow_dom=True): + if include_shadow_dom: + return self.find_element("html").get_html() try: source = self.loop.run_until_complete( self.page.evaluate("document.documentElement.outerHTML") diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 2827c1649ea..d107dc0acbe 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -1293,8 +1293,11 @@ def get_origin(self): self.__check_scope() return self.execute_script("return window.location.origin;") - def get_page_source(self): - if self.__is_cdp_swap_needed(): + def get_html(self, *args, **kwargs): + return self.get_page_source(*args, **kwargs) + + def get_page_source(self, *args, **kwargs): + if self.__is_cdp_swap_needed(*args, **kwargs): return self.cdp.get_page_source() self.wait_for_ready_state_complete() if self.__needs_minimum_wait: diff --git a/seleniumbase/undetected/cdp_driver/config.py b/seleniumbase/undetected/cdp_driver/config.py index aa9960fdb1a..0051b49a1d1 100644 --- a/seleniumbase/undetected/cdp_driver/config.py +++ b/seleniumbase/undetected/cdp_driver/config.py @@ -130,6 +130,7 @@ def __init__( "--no-default-browser-check", "--homepage=about:blank", "--no-pings", + "--enable-unsafe-extension-debugging", "--wm-window-animations-disabled", "--animation-duration-scale=0", "--enable-privacy-sandbox-ads-apis", diff --git a/seleniumbase/undetected/cdp_driver/element.py b/seleniumbase/undetected/cdp_driver/element.py index d3829b18d75..94a6ab6513b 100644 --- a/seleniumbase/undetected/cdp_driver/element.py +++ b/seleniumbase/undetected/cdp_driver/element.py @@ -806,7 +806,10 @@ async def set_text_async(self, value): async def get_html_async(self): return await self._tab.send( - cdp.dom.get_outer_html(backend_node_id=self.backend_node_id) + cdp.dom.get_outer_html( + backend_node_id=self.backend_node_id, + include_shadow_dom=True, + ) ) @property diff --git a/seleniumbase/undetected/cdp_driver/tab.py b/seleniumbase/undetected/cdp_driver/tab.py index b0b9dc0fa3c..235fbb7fe12 100644 --- a/seleniumbase/undetected/cdp_driver/tab.py +++ b/seleniumbase/undetected/cdp_driver/tab.py @@ -899,7 +899,10 @@ async def get_content(self): """Gets the current page source content (html)""" doc: cdp.dom.Node = await self.send(cdp.dom.get_document(-1, True)) return await self.send( - cdp.dom.get_outer_html(backend_node_id=doc.backend_node_id) + cdp.dom.get_outer_html( + backend_node_id=doc.backend_node_id, + include_shadow_dom=True, + ) ) async def maximize(self): diff --git a/setup.py b/setup.py index 2cbdb91a9f9..8919255f79e 100755 --- a/setup.py +++ b/setup.py @@ -163,7 +163,7 @@ 'filelock~=3.19.1;python_version>="3.9" and python_version<"3.10"', 'filelock>=3.20.0;python_version>="3.10"', 'fasteners>=0.20', - "mycdp>=1.2.1", + "mycdp>=1.3.0", "pynose>=1.5.5", 'platformdirs~=4.3.6;python_version<"3.9"', 'platformdirs~=4.4.0;python_version>="3.9" and python_version<"3.10"', @@ -194,7 +194,8 @@ 'h11==0.16.0', 'outcome==1.3.0.post0', 'trio==0.27.0;python_version<"3.9"', - 'trio>=0.31.0,<1;python_version>="3.9"', + 'trio>=0.31.0,<1;python_version>="3.9" and python_version<"3.10"', + 'trio>=0.32.0,<1;python_version>="3.10"', 'trio-websocket~=0.12.2', 'wsproto==1.2.0', 'websocket-client~=1.8.0;python_version<"3.9"',