diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index bde2c106be2..c4836064c1c 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -202,9 +202,13 @@ with SB(uc=True, test=True, locale="en") as sb: sb.type('input[id="search-term"]', location) sb.sleep(1.2) sb.click('li[data-js="suggestion"]') - sb.sleep(1.2) + sb.sleep(0.6) + sb.click_if_visible('button[aria-label="Close"]') + sb.sleep(0.6) sb.click("button.be-button-shop") - sb.sleep(6) + sb.sleep(1) + sb.click_if_visible('[label="Find Hotels"]') + sb.sleep(5) card_info = 'div[data-booking-status="BOOKABLE"] [class*="HotelCard_info"]' hotels = sb.select_all(card_info) print("Hyatt Hotels in %s:" % location) diff --git a/examples/cdp_mode/playwright/raw_gas_info_sync.py b/examples/cdp_mode/playwright/raw_gas_info_sync.py new file mode 100644 index 00000000000..9b1d74c566b --- /dev/null +++ b/examples/cdp_mode/playwright/raw_gas_info_sync.py @@ -0,0 +1,31 @@ +from playwright.sync_api import sync_playwright +from seleniumbase import sb_cdp + +sb = sb_cdp.Chrome() +endpoint_url = sb.get_endpoint_url() + +with sync_playwright() as p: + browser = p.chromium.connect_over_cdp(endpoint_url) + context = browser.contexts[0] + page = context.pages[0] + url = ( + "https://www.gassaferegister.co.uk/gas-safety" + "/gas-safety-certificates-records/building-regulations-certificate" + "/order-replacement-building-regulations-certificate/" + ) + page.goto(url) + sb.sleep(0.5) + sb.solve_captcha() + page.wait_for_selector("#SearchTerm") + allow_cookies = 'button:contains("Allow all cookies")' + sb.click_if_visible(allow_cookies, timeout=2) + sb.sleep(1) + page.fill("#SearchTerm", "Hydrogen") + page.click("button.search-button") + sb.sleep(3) + items = page.locator("div.search-result") + for i in range(items.count()): + item_text = items.nth(i).inner_text() + print(item_text.replace("\n\n", "\n") + "\n") + sb.scroll_to_bottom() + sb.sleep(1) diff --git a/examples/cdp_mode/playwright/raw_reddit_sync.py b/examples/cdp_mode/playwright/raw_reddit_sync.py new file mode 100644 index 00000000000..10c0c67996c --- /dev/null +++ b/examples/cdp_mode/playwright/raw_reddit_sync.py @@ -0,0 +1,25 @@ +from playwright.sync_api import sync_playwright +from seleniumbase import sb_cdp + +sb = sb_cdp.Chrome(use_chromium=True) +endpoint_url = sb.get_endpoint_url() + +with sync_playwright() as p: + browser = p.chromium.connect_over_cdp(endpoint_url) + context = browser.contexts[0] + page = context.pages[0] + search = "reddit+scraper" + url = f"https://www.reddit.com/r/webscraping/search/?q={search}" + page.goto(url) + sb.solve_captcha() # Might not be needed + sb.sleep(1) + post_title = '[data-testid="post-title"]' + page.wait_for_selector(post_title) + for i in range(8): + sb.scroll_down(25) + sb.sleep(0.2) + print('*** Reddit Posts for "%s":' % search) + items = page.locator(post_title) + for i in range(items.count()): + item_text = items.nth(i).inner_text() + print("* " + item_text) diff --git a/examples/cdp_mode/raw_easyjet.py b/examples/cdp_mode/raw_easyjet.py index 768cac9cabc..ea3e61e38a0 100644 --- a/examples/cdp_mode/raw_easyjet.py +++ b/examples/cdp_mode/raw_easyjet.py @@ -3,31 +3,29 @@ with SB(uc=True, test=True, locale="en", ad_block=True) as sb: url = "https://www.easyjet.com/en/" sb.activate_cdp_mode(url) - sb.sleep(2) - sb.click_if_visible("button#ensCloseBanner") - sb.sleep(1.2) + sb.sleep(1.5) + sb.click_if_visible("button#ensRejectAds", timeout=2) + sb.sleep(1) sb.click('input[name="from"]') - sb.sleep(1.2) + sb.sleep(1) sb.type('input[name="from"]', "London Gatwick") - sb.sleep(0.6) - sb.click_if_visible("button#ensCloseBanner") - sb.sleep(0.6) + sb.sleep(1) sb.click('span[data-testid="airport-name"]') - sb.sleep(1.2) + sb.sleep(1) sb.type('input[name="to"]', "Paris") - sb.sleep(1.2) + sb.sleep(1) sb.click('span[data-testid="airport-name"]') - sb.sleep(1.2) + sb.sleep(1) sb.click('input[name="when"]') - sb.sleep(1.2) + sb.sleep(1) sb.click('[data-testid="month"]:last-of-type [aria-disabled="false"]') - sb.sleep(1.2) + sb.sleep(1) sb.click('[data-testid="month"]:last-of-type [aria-disabled="false"]') - sb.sleep(1.2) + sb.sleep(1) sb.click('button[data-testid="submit"]') - sb.sleep(4.2) + sb.sleep(4) sb.connect() - sb.sleep(1.2) + sb.sleep(1) for window in sb.driver.window_handles: sb.switch_to_window(window) if "/buy/flights" in sb.get_current_url(): diff --git a/examples/cdp_mode/raw_gas_records.py b/examples/cdp_mode/raw_gas_records.py index 73e2ee66d2a..c484baa5c2c 100644 --- a/examples/cdp_mode/raw_gas_records.py +++ b/examples/cdp_mode/raw_gas_records.py @@ -8,21 +8,18 @@ "/order-replacement-building-regulations-certificate/" ) sb.activate_cdp_mode(url) - sb.sleep(0.6) + sb.sleep(0.5) sb.solve_captcha() - sb.sleep(1) sb.wait_for_element("#SearchTerm", timeout=5) sb.sleep(2) allow_cookies = 'button:contains("Allow all cookies")' sb.click_if_visible(allow_cookies, timeout=2) - sb.sleep(1.2) + sb.sleep(1) sb.press_keys("#SearchTerm", "Hydrogen") - sb.sleep(0.5) sb.click("button.search-button") sb.sleep(3) results = sb.find_elements("div.search-result") for result in results: - print(result.text.replace(" " * 12, " ").strip()) - print() + print(result.text.replace(" " * 12, " ").strip() + "\n") sb.scroll_to_bottom() sb.sleep(1) diff --git a/examples/cdp_mode/raw_hyatt.py b/examples/cdp_mode/raw_hyatt.py index 9876c283c94..54d6095e312 100644 --- a/examples/cdp_mode/raw_hyatt.py +++ b/examples/cdp_mode/raw_hyatt.py @@ -12,9 +12,13 @@ sb.type('input[id="search-term"]', location) sb.sleep(1.2) sb.click('li[data-js="suggestion"]') - sb.sleep(1.2) + sb.sleep(0.6) + sb.click_if_visible('button[aria-label="Close"]') + sb.sleep(0.6) sb.click("button.be-button-shop") - sb.sleep(6) + sb.sleep(1) + sb.click_if_visible('[label="Find Hotels"]') + sb.sleep(5) card_info = 'div[data-booking-status="BOOKABLE"] [class*="HotelCard_info"]' hotels = sb.select_all(card_info) print("Hyatt Hotels in %s:" % location) diff --git a/examples/cdp_mode/raw_priceline.py b/examples/cdp_mode/raw_priceline.py index d9757fa778d..fb76ff41dae 100644 --- a/examples/cdp_mode/raw_priceline.py +++ b/examples/cdp_mode/raw_priceline.py @@ -1,6 +1,6 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale="en") as sb: +with SB(uc=True, test=True, locale="en", guest=True, ad_block=True) as sb: url = "https://www.priceline.com" sb.activate_cdp_mode(url) sb.sleep(1.8) @@ -9,19 +9,15 @@ location = "Portland, OR" selection = "Oregon, United States" # (Dropdown option) sb.press_keys('input[name="endLocation"]', location) - sb.sleep(1.5) + sb.sleep(0.5) sb.click_if_visible('input[name="endLocation"]') - sb.sleep(0.6) - sb.click(selection) - sb.sleep(1.5) - sb.click('button[aria-label="Dismiss calendar"]') sb.sleep(0.5) + sb.click(selection) + sb.scroll_down(25) + sb.click_if_visible('button[aria-label="Dismiss calendar"]') + sb.click_if_visible("div.sidebar-iframe-close") + sb.click_if_visible('div[aria-label="Close Modal"]') sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]') - sb.sleep(0.5) - if sb.is_element_visible('[aria-label="Close Modal"]'): - sb.click('[aria-label="Close Modal"]') - sb.sleep(0.5) - sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]') sb.sleep(4.8) if len(sb.cdp.get_tabs()) > 1: sb.cdp.close_active_tab() diff --git a/examples/cdp_mode/raw_ralphlauren.py b/examples/cdp_mode/raw_ralphlauren.py index 7139759a8df..5ba1bdd4b1e 100644 --- a/examples/cdp_mode/raw_ralphlauren.py +++ b/examples/cdp_mode/raw_ralphlauren.py @@ -2,9 +2,8 @@ with SB(uc=True, test=True, locale="en") as sb: url = "https://www.ralphlauren.com.au/" - sb.activate_cdp_mode() - sb.open(url) - sb.sleep(1.2) + sb.activate_cdp_mode(url) + sb.sleep(1.6) if not sb.is_element_present('[title="Locate Stores"]'): sb.evaluate("window.location.reload();") sb.sleep(1.2) @@ -15,8 +14,8 @@ sb.click('a:contains("%s")' % search) sb.sleep(3.8) for i in range(6): - sb.scroll_down(34) - sb.sleep(0.25) + sb.scroll_down(25) + sb.sleep(0.2) print('*** Ralph Lauren Search for "%s":' % search) unique_item_text = [] items = sb.find_elements('div.product-data') diff --git a/examples/cdp_mode/raw_zoro.py b/examples/cdp_mode/raw_zoro.py index 035c2412622..0f0beecc311 100644 --- a/examples/cdp_mode/raw_zoro.py +++ b/examples/cdp_mode/raw_zoro.py @@ -16,14 +16,15 @@ sb.press_keys(search_box, search) sb.sleep(0.6) sb.click('button[data-za="searchButton"]') - sb.sleep(3.8) + sb.sleep(3.2) + sb.wait_for_element('[data-za="product-cards-list"]', timeout=5) print('*** Zoro Search for "%s":' % search) print(' (Results must contain "%s".)' % required_text) unique_item_text = [] - items = sb.find_elements('div[data-za="search-product-card"]') + items = sb.find_elements('[data-za="search-product-card"]') for item in items: if required_text in item.text: - description = item.querySelector('div[data-za="product-title"]') + description = item.querySelector('[data-za="product-title"]') if description and description.text not in unique_item_text: unique_item_text.append(description.text) print("* " + description.text) diff --git a/examples/presenter/uc_presentation_4.py b/examples/presenter/uc_presentation_4.py index bb093c2ae3f..444720c0eea 100644 --- a/examples/presenter/uc_presentation_4.py +++ b/examples/presenter/uc_presentation_4.py @@ -768,7 +768,9 @@ def test_presentation_4(self): ) self.begin_presentation(filename="uc_presentation.html") - with SB(uc=True, test=True, locale="en") as sb: + with SB( + uc=True, test=True, locale="en", guest=True, ad_block=True + ) as sb: url = "https://www.priceline.com" sb.activate_cdp_mode(url) sb.sleep(1.8) @@ -777,19 +779,15 @@ def test_presentation_4(self): location = "Portland, Oregon, US" selection = "Oregon, United States" # (Dropdown option) sb.press_keys('input[name="endLocation"]', location) - sb.sleep(1.5) + sb.sleep(0.5) sb.click_if_visible('input[name="endLocation"]') - sb.sleep(0.6) - sb.click(selection) - sb.sleep(1.5) - sb.click('button[aria-label="Dismiss calendar"]') sb.sleep(0.5) + sb.click(selection) + sb.scroll_down(25) + sb.click_if_visible('button[aria-label="Dismiss calendar"]') + sb.click_if_visible("div.sidebar-iframe-close") + sb.click_if_visible('div[aria-label="Close Modal"]') sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]') - sb.sleep(0.5) - if sb.is_element_visible('[aria-label="Close Modal"]'): - sb.click('[aria-label="Close Modal"]') - sb.sleep(0.5) - sb.click('button[data-testid="HOTELS_SUBMIT_BUTTON"]') sb.sleep(4.8) if len(sb.cdp.get_tabs()) > 1: sb.cdp.close_active_tab() diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index ac062a21652..bdfaf02bc56 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.45.11" +__version__ = "4.45.12" diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index 0c93bf4aa89..e9e2fe3ece7 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -795,15 +795,13 @@ def click_if_visible(self, selector, timeout=0): if self.is_element_visible(selector): with suppress(Exception): self.click(selector, timeout=1) + elif timeout == 0: + return else: with suppress(Exception): - element = self.find_element(selector, timeout=timeout) - self.sleep(0.1) - element.scroll_into_view() - self.sleep(0.1) - element.click() - self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.find_element(selector, timeout=timeout) + if self.is_element_visible(selector): + self.click(selector, timeout=1) def click_visible_elements(self, selector, limit=0): """Finds all matching page elements and clicks visible ones in order. @@ -2068,14 +2066,35 @@ def __cdp_click_incapsula_hcaptcha(self): time.sleep(0.05) x_offset = 30 y_offset = 36 + was_clicked = False gui_lock = FileLock(constants.MultiBrowser.PYAUTOGUILOCK) with gui_lock: # Prevent issues with multiple processes self.bring_active_window_to_front() - time.sleep(0.05) + time.sleep(0.056) + if "--debug" in sys.argv: + displayed_selector = "`%s`" % selector + if '"' not in selector: + displayed_selector = '"%s"' % selector + elif "'" not in selector: + displayed_selector = "'%s'" % selector + print( + " click_with_offset(%s, %s, %s)" + % (displayed_selector, x_offset, y_offset) + ) with suppress(Exception): element.click_with_offset(x_offset, y_offset) - time.sleep(0.2) - return True + was_clicked = True + time.sleep(0.056) + if was_clicked: + # Wait a moment for the click to succeed + time.sleep(0.25) + self.__slow_mode_pause_if_set() + self.loop.run_until_complete(self.page.wait()) + if "--debug" in sys.argv: + print(" hCaptcha was clicked!") + return True + if "--debug" in sys.argv: + print(" hCaptcha was NOT clicked!") return False def solve_captcha(self): diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 5a17810165f..e60b3ae665e 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -92,6 +92,7 @@ def test_anything(self): logging.getLogger("requests").setLevel(logging.ERROR) logging.getLogger("urllib3").setLevel(logging.ERROR) +logging.getLogger("websocket").setLevel(logging.CRITICAL) urllib3.disable_warnings() LOGGER.setLevel(logging.WARNING) is_linux = shared_utils.is_linux() diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py index b39a62d848d..98d77107f1d 100644 --- a/seleniumbase/undetected/cdp_driver/browser.py +++ b/seleniumbase/undetected/cdp_driver/browser.py @@ -182,7 +182,6 @@ def stopped(self): if self._process and self._process.returncode is None: return False return True - # return (self._process and self._process.returncode) or False async def wait(self, time: Union[float, int] = 1) -> Browser: """Wait for