diff --git a/ROADMAP.md b/ROADMAP.md index b3e9469d..27b89c21 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,5 +1,6 @@ # Roadmap +- [ ] Extract local download check to chromium implementation class - [x] Fix up browser deps error (eliminate in-package analysis) - [x] Switch to process group and kill that to catch all chromium children - [x] Add helpers for running javascript diff --git a/src/choreographer/protocol/devtools_async_helpers.py b/src/choreographer/protocol/devtools_async_helpers.py index a40e6b04..dafbb3ba 100644 --- a/src/choreographer/protocol/devtools_async_helpers.py +++ b/src/choreographer/protocol/devtools_async_helpers.py @@ -5,11 +5,52 @@ import asyncio from typing import TYPE_CHECKING +import logistro + if TYPE_CHECKING: from choreographer import Browser, Tab + from choreographer.protocol.devtools_async import Session from . import BrowserResponse +_logger = logistro.getLogger(__name__) + +# Abit about the mechanics of chrome: +# Whether or not a Page.loadEventFired event fires is a bit +# racey. Optimistically, it's buffered and fired after subscription +# even if the event happened in the past. +# Doesn't seem to always work out that way, so we also use +# javascript to create a "loaded" event, but for the case +# where we need to timeout- loading a page that never resolves, +# the browser might actually load an about:blank instead and then +# fire the event, misleading the user, so we check the url. + + +async def _check_document_ready(session: Session, url: str) -> BrowserResponse: + return await session.send_command( + "Runtime.evaluate", + params={ + "expression": """ + new Promise((resolve) => { + if ( + (document.readyState === 'complete') && + (window.location==`""" # CONCATENATE! + f"{url!s}" + """`) + ){ + resolve("Was complete"); + } else { + window.addEventListener( + 'load', () => resolve("Event loaded") + ); + } + }) + """, + "awaitPromise": True, + "returnByValue": True, + }, + ) + async def create_and_wait( browser: Browser, @@ -29,24 +70,61 @@ async def create_and_wait( The created Tab """ + _logger.debug("Creating tab") tab = await browser.create_tab(url) + _logger.debug("Creating session") temp_session = await tab.create_session() try: + _logger.debug("Subscribing to loadEven and enabling events.") load_future = temp_session.subscribe_once("Page.loadEventFired") await temp_session.send_command("Page.enable") await temp_session.send_command("Runtime.enable") if url: try: - await asyncio.wait_for(load_future, timeout=timeout) - except (asyncio.TimeoutError, asyncio.CancelledError, TimeoutError): + # JavaScript evaluation to check if document is loaded + js_ready_future = asyncio.create_task( + _check_document_ready(temp_session, url), + ) + _logger.debug(f"Starting wait: timeout={timeout}") + # Race between the two methods: first one to complete wins + done, pending = await asyncio.wait( + [ + load_future, + js_ready_future, + ], + return_when=asyncio.FIRST_COMPLETED, + timeout=timeout, + ) + _logger.debug(f"Finish wait, is done? {bool(done)}") + + for task in pending: + _logger.debug(f"Cancelling: {task}") + task.cancel() + + if not done: + _logger.debug("Timeout waiting for js or event") + raise asyncio.TimeoutError( # noqa: TRY301 + "Page load timeout", + ) + else: + _logger.debug(f"Task which finished: {done}") + + except ( + asyncio.TimeoutError, + asyncio.CancelledError, + TimeoutError, + ) as e: # Stop the page load when timeout occurs + _logger.debug("Need to stop page loading, error.", exc_info=e) await temp_session.send_command("Page.stopLoading") raise finally: + _logger.debug("Closing session") await tab.close_session(temp_session.session_id) + _logger.debug("Returning tab.") return tab @@ -71,9 +149,10 @@ async def navigate_and_wait( temp_session = await tab.create_session() try: + # Subscribe BEFORE enabling domains to avoid race condition + load_future = temp_session.subscribe_once("Page.loadEventFired") await temp_session.send_command("Page.enable") await temp_session.send_command("Runtime.enable") - load_future = temp_session.subscribe_once("Page.loadEventFired") try: async def _freezers() -> None: diff --git a/src/choreographer/resources/last_known_good_chrome.json b/src/choreographer/resources/last_known_good_chrome.json index c99834a6..f3274275 100644 --- a/src/choreographer/resources/last_known_good_chrome.json +++ b/src/choreographer/resources/last_known_good_chrome.json @@ -1 +1 @@ -{"version": "144.0.7527.0", "revision": "1544685", "downloads": {"chrome": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/linux64/chrome-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-arm64/chrome-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-x64/chrome-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win32/chrome-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win64/chrome-win64.zip"}], "chromedriver": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/linux64/chromedriver-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-arm64/chromedriver-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-x64/chromedriver-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win32/chromedriver-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win64/chromedriver-win64.zip"}], "chrome-headless-shell": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/linux64/chrome-headless-shell-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-arm64/chrome-headless-shell-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-x64/chrome-headless-shell-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win32/chrome-headless-shell-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win64/chrome-headless-shell-win64.zip"}]}} +{"version": "142.0.7444.175", "revision": "1522585", "downloads": {"chrome": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/linux64/chrome-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-arm64/chrome-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-x64/chrome-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win32/chrome-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win64/chrome-win64.zip"}], "chromedriver": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/linux64/chromedriver-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-arm64/chromedriver-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-x64/chromedriver-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win32/chromedriver-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win64/chromedriver-win64.zip"}], "chrome-headless-shell": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/linux64/chrome-headless-shell-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-arm64/chrome-headless-shell-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-x64/chrome-headless-shell-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win32/chrome-headless-shell-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win64/chrome-headless-shell-win64.zip"}]}} diff --git a/tests/test_devtools_async_helpers.py b/tests/test_devtools_async_helpers.py index 309518ec..86a83132 100644 --- a/tests/test_devtools_async_helpers.py +++ b/tests/test_devtools_async_helpers.py @@ -28,7 +28,7 @@ async def test_create_and_wait(browser): initial_tab_count = len(browser.tabs) # Create a simple HTML page as a data URL - data_url = "chrome://version" + data_url = "https://www.example.com" # Test 1: Create tab with data URL - should succeed tab1 = await create_and_wait(browser, url=data_url, timeout=5.0)