From e7a1bfdd0804733811832c69bcbcf0996e908b32 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 25 Nov 2025 15:46:47 -0500 Subject: [PATCH 01/12] Extend timeout --- tests/test_devtools_async_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_devtools_async_helpers.py b/tests/test_devtools_async_helpers.py index 309518ec..a84593a3 100644 --- a/tests/test_devtools_async_helpers.py +++ b/tests/test_devtools_async_helpers.py @@ -31,7 +31,7 @@ async def test_create_and_wait(browser): data_url = "chrome://version" # Test 1: Create tab with data URL - should succeed - tab1 = await create_and_wait(browser, url=data_url, timeout=5.0) + tab1 = await create_and_wait(browser, url=data_url, timeout=10.0) assert tab1 is not None # Verify the page loaded correctly using execute_js_and_wait From 9c9110e8697fd9c47e1ff1c71b7f31036b9cd216 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 25 Nov 2025 15:51:35 -0500 Subject: [PATCH 02/12] Reduce timeout again --- tests/test_devtools_async_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_devtools_async_helpers.py b/tests/test_devtools_async_helpers.py index a84593a3..309518ec 100644 --- a/tests/test_devtools_async_helpers.py +++ b/tests/test_devtools_async_helpers.py @@ -31,7 +31,7 @@ async def test_create_and_wait(browser): data_url = "chrome://version" # Test 1: Create tab with data URL - should succeed - tab1 = await create_and_wait(browser, url=data_url, timeout=10.0) + tab1 = await create_and_wait(browser, url=data_url, timeout=5.0) assert tab1 is not None # Verify the page loaded correctly using execute_js_and_wait From 2bd8d4e8650d4769c87c197ed939486566decb7b Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 25 Nov 2025 15:56:45 -0500 Subject: [PATCH 03/12] Move off canary chrome --- src/choreographer/resources/last_known_good_chrome.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/choreographer/resources/last_known_good_chrome.json b/src/choreographer/resources/last_known_good_chrome.json index c99834a6..f3274275 100644 --- a/src/choreographer/resources/last_known_good_chrome.json +++ b/src/choreographer/resources/last_known_good_chrome.json @@ -1 +1 @@ -{"version": "144.0.7527.0", "revision": "1544685", "downloads": {"chrome": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/linux64/chrome-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-arm64/chrome-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-x64/chrome-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win32/chrome-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win64/chrome-win64.zip"}], "chromedriver": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/linux64/chromedriver-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-arm64/chromedriver-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-x64/chromedriver-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win32/chromedriver-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win64/chromedriver-win64.zip"}], "chrome-headless-shell": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/linux64/chrome-headless-shell-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-arm64/chrome-headless-shell-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/mac-x64/chrome-headless-shell-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win32/chrome-headless-shell-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/144.0.7527.0/win64/chrome-headless-shell-win64.zip"}]}} +{"version": "142.0.7444.175", "revision": "1522585", "downloads": {"chrome": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/linux64/chrome-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-arm64/chrome-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-x64/chrome-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win32/chrome-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win64/chrome-win64.zip"}], "chromedriver": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/linux64/chromedriver-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-arm64/chromedriver-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-x64/chromedriver-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win32/chromedriver-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win64/chromedriver-win64.zip"}], "chrome-headless-shell": [{"platform": "linux64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/linux64/chrome-headless-shell-linux64.zip"}, {"platform": "mac-arm64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-arm64/chrome-headless-shell-mac-arm64.zip"}, {"platform": "mac-x64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/mac-x64/chrome-headless-shell-mac-x64.zip"}, {"platform": "win32", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win32/chrome-headless-shell-win32.zip"}, {"platform": "win64", "url": "https://storage.googleapis.com/chrome-for-testing-public/142.0.7444.175/win64/chrome-headless-shell-win64.zip"}]}} From 5b66b752362bf27ae9139aa8639f03f960d5c429 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 25 Nov 2025 16:20:30 -0500 Subject: [PATCH 04/12] Reorder subscriptions --- src/choreographer/protocol/devtools_async_helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/choreographer/protocol/devtools_async_helpers.py b/src/choreographer/protocol/devtools_async_helpers.py index a40e6b04..3059d3ee 100644 --- a/src/choreographer/protocol/devtools_async_helpers.py +++ b/src/choreographer/protocol/devtools_async_helpers.py @@ -71,9 +71,10 @@ async def navigate_and_wait( temp_session = await tab.create_session() try: + # Subscribe BEFORE enabling domains to avoid race condition + load_future = temp_session.subscribe_once("Page.loadEventFired") await temp_session.send_command("Page.enable") await temp_session.send_command("Runtime.enable") - load_future = temp_session.subscribe_once("Page.loadEventFired") try: async def _freezers() -> None: From a1428578704e61e025e46bfc9b7dbb954da1d949 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 25 Nov 2025 16:25:21 -0500 Subject: [PATCH 05/12] Change test url --- tests/test_devtools_async_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_devtools_async_helpers.py b/tests/test_devtools_async_helpers.py index 309518ec..44ab11cd 100644 --- a/tests/test_devtools_async_helpers.py +++ b/tests/test_devtools_async_helpers.py @@ -28,7 +28,7 @@ async def test_create_and_wait(browser): initial_tab_count = len(browser.tabs) # Create a simple HTML page as a data URL - data_url = "chrome://version" + data_url = "https://example.com" # Test 1: Create tab with data URL - should succeed tab1 = await create_and_wait(browser, url=data_url, timeout=5.0) From c98988f900da19aca6524ca0454a781b4c71b669 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 25 Nov 2025 16:41:09 -0500 Subject: [PATCH 06/12] Add in double check --- .../protocol/devtools_async_helpers.py | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/choreographer/protocol/devtools_async_helpers.py b/src/choreographer/protocol/devtools_async_helpers.py index 3059d3ee..b907b70a 100644 --- a/src/choreographer/protocol/devtools_async_helpers.py +++ b/src/choreographer/protocol/devtools_async_helpers.py @@ -39,7 +39,49 @@ async def create_and_wait( if url: try: - await asyncio.wait_for(load_future, timeout=timeout) + # JavaScript evaluation to check if document is loaded + async def check_document_ready() -> None: + await temp_session.send_command( + "Runtime.evaluate", + params={ + "expression": """ + new Promise((resolve) => { + if (document.readyState === 'complete') { + resolve(true); + } else { + window.addEventListener( + 'load', () => resolve(true) + ); + } + }) + """, + "awaitPromise": True, + "returnByValue": True, + }, + ) + + js_ready_future = asyncio.create_task(check_document_ready()) + + # Race between the two methods: first one to complete wins + done, pending = await asyncio.wait( + [ + load_future, + js_ready_future, + ], + return_when=asyncio.FIRST_COMPLETED, + timeout=timeout, + ) + + # Cancel pending tasks + for task in pending: + task.cancel() + + # Check if timeout occurred + if not done: + raise asyncio.TimeoutError( # noqa: TRY301 + "Page load timeout", + ) + except (asyncio.TimeoutError, asyncio.CancelledError, TimeoutError): # Stop the page load when timeout occurs await temp_session.send_command("Page.stopLoading") From 21f61abc3cf72ebb37764231b6e223f470def3b6 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Tue, 25 Nov 2025 16:45:40 -0500 Subject: [PATCH 07/12] Move back to old URL --- tests/test_devtools_async_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_devtools_async_helpers.py b/tests/test_devtools_async_helpers.py index 44ab11cd..309518ec 100644 --- a/tests/test_devtools_async_helpers.py +++ b/tests/test_devtools_async_helpers.py @@ -28,7 +28,7 @@ async def test_create_and_wait(browser): initial_tab_count = len(browser.tabs) # Create a simple HTML page as a data URL - data_url = "https://example.com" + data_url = "chrome://version" # Test 1: Create tab with data URL - should succeed tab1 = await create_and_wait(browser, url=data_url, timeout=5.0) From 8a52f597df884d1ea3295827b7814597a8fd326b Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 26 Nov 2025 12:36:15 -0500 Subject: [PATCH 08/12] Update ROADMAP.md --- ROADMAP.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ROADMAP.md b/ROADMAP.md index b3e9469d..27b89c21 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,5 +1,6 @@ # Roadmap +- [ ] Extract local download check to chromium implementation class - [x] Fix up browser deps error (eliminate in-package analysis) - [x] Switch to process group and kill that to catch all chromium children - [x] Add helpers for running javascript From a561550f3b3b20bcacbdd4ceca2e2416dfe61002 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 26 Nov 2025 14:40:24 -0500 Subject: [PATCH 09/12] Finalize js helper to create tag to catch load --- .../protocol/devtools_async_helpers.py | 63 ++++++++++++------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/src/choreographer/protocol/devtools_async_helpers.py b/src/choreographer/protocol/devtools_async_helpers.py index b907b70a..92454297 100644 --- a/src/choreographer/protocol/devtools_async_helpers.py +++ b/src/choreographer/protocol/devtools_async_helpers.py @@ -7,9 +7,46 @@ if TYPE_CHECKING: from choreographer import Browser, Tab + from choreographer.protocol.devtools_async import Session from . import BrowserResponse +# Abit about the mechanics of chrome: +# Whether or not a Page.loadEventFired event fires is a bit +# racey. Optimistically, it's buffered and fired after subscription +# even if the event happened in the past. +# Doesn't seem to always work out that way, so we also use +# javascript to create a "loaded" event, but for the case +# where we need to timeout- loading a page that never resolves, +# the browser might actually load an about:blank instead and then +# fire the event, misleading the user, so we check the url. + + +async def _check_document_ready(session: Session, url: str) -> BrowserResponse: + return await session.send_command( + "Runtime.evaluate", + params={ + "expression": """ + new Promise((resolve) => { + if ( + (document.readyState === 'complete') && + (window.location==`""" # CONCATENATE! + f"{url!s}" + """`) + ){ + resolve("Was complete"); + } else { + window.addEventListener( + 'load', () => resolve("Event loaded") + ); + } + }) + """, + "awaitPromise": True, + "returnByValue": True, + }, + ) + async def create_and_wait( browser: Browser, @@ -40,27 +77,9 @@ async def create_and_wait( if url: try: # JavaScript evaluation to check if document is loaded - async def check_document_ready() -> None: - await temp_session.send_command( - "Runtime.evaluate", - params={ - "expression": """ - new Promise((resolve) => { - if (document.readyState === 'complete') { - resolve(true); - } else { - window.addEventListener( - 'load', () => resolve(true) - ); - } - }) - """, - "awaitPromise": True, - "returnByValue": True, - }, - ) - - js_ready_future = asyncio.create_task(check_document_ready()) + js_ready_future = asyncio.create_task( + _check_document_ready(temp_session, url), + ) # Race between the two methods: first one to complete wins done, pending = await asyncio.wait( @@ -72,11 +91,9 @@ async def check_document_ready() -> None: timeout=timeout, ) - # Cancel pending tasks for task in pending: task.cancel() - # Check if timeout occurred if not done: raise asyncio.TimeoutError( # noqa: TRY301 "Page load timeout", From 593532d3582f481653612da3284457c63ee85aa0 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 26 Nov 2025 15:01:07 -0500 Subject: [PATCH 10/12] Add logging to create_and_wait --- src/choreographer/protocol/devtools_async_helpers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/choreographer/protocol/devtools_async_helpers.py b/src/choreographer/protocol/devtools_async_helpers.py index 92454297..7ec5fc01 100644 --- a/src/choreographer/protocol/devtools_async_helpers.py +++ b/src/choreographer/protocol/devtools_async_helpers.py @@ -5,12 +5,16 @@ import asyncio from typing import TYPE_CHECKING +import logistro + if TYPE_CHECKING: from choreographer import Browser, Tab from choreographer.protocol.devtools_async import Session from . import BrowserResponse +_logger = logistro.getLogger(__name__) + # Abit about the mechanics of chrome: # Whether or not a Page.loadEventFired event fires is a bit # racey. Optimistically, it's buffered and fired after subscription @@ -66,10 +70,13 @@ async def create_and_wait( The created Tab """ + _logger.debug("Creating tab") tab = await browser.create_tab(url) + _logger.debug("Creating session") temp_session = await tab.create_session() try: + _logger.debug("Subscribing to loadEven and enabling events.") load_future = temp_session.subscribe_once("Page.loadEventFired") await temp_session.send_command("Page.enable") await temp_session.send_command("Runtime.enable") @@ -98,14 +105,19 @@ async def create_and_wait( raise asyncio.TimeoutError( # noqa: TRY301 "Page load timeout", ) + else: + _logger.debug(done) except (asyncio.TimeoutError, asyncio.CancelledError, TimeoutError): # Stop the page load when timeout occurs + _logger.debug("Need to stop page loading, error.") await temp_session.send_command("Page.stopLoading") raise finally: + _logger.debug("Closing session") await tab.close_session(temp_session.session_id) + _logger.debug("Returning tab.") return tab From b0c69605995a31bb40adc459ce6c0c6fe41a129b Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 26 Nov 2025 15:08:54 -0500 Subject: [PATCH 11/12] Change testing URL --- tests/test_devtools_async_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_devtools_async_helpers.py b/tests/test_devtools_async_helpers.py index 309518ec..86a83132 100644 --- a/tests/test_devtools_async_helpers.py +++ b/tests/test_devtools_async_helpers.py @@ -28,7 +28,7 @@ async def test_create_and_wait(browser): initial_tab_count = len(browser.tabs) # Create a simple HTML page as a data URL - data_url = "chrome://version" + data_url = "https://www.example.com" # Test 1: Create tab with data URL - should succeed tab1 = await create_and_wait(browser, url=data_url, timeout=5.0) From 11ab28d67aeb5a1b749fe094e47771dd2f12bed5 Mon Sep 17 00:00:00 2001 From: Andrew Pikul Date: Wed, 26 Nov 2025 15:40:41 -0500 Subject: [PATCH 12/12] Add more debugging --- .../protocol/devtools_async_helpers.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/choreographer/protocol/devtools_async_helpers.py b/src/choreographer/protocol/devtools_async_helpers.py index 7ec5fc01..dafbb3ba 100644 --- a/src/choreographer/protocol/devtools_async_helpers.py +++ b/src/choreographer/protocol/devtools_async_helpers.py @@ -87,7 +87,7 @@ async def create_and_wait( js_ready_future = asyncio.create_task( _check_document_ready(temp_session, url), ) - + _logger.debug(f"Starting wait: timeout={timeout}") # Race between the two methods: first one to complete wins done, pending = await asyncio.wait( [ @@ -97,20 +97,27 @@ async def create_and_wait( return_when=asyncio.FIRST_COMPLETED, timeout=timeout, ) + _logger.debug(f"Finish wait, is done? {bool(done)}") for task in pending: + _logger.debug(f"Cancelling: {task}") task.cancel() if not done: + _logger.debug("Timeout waiting for js or event") raise asyncio.TimeoutError( # noqa: TRY301 "Page load timeout", ) else: - _logger.debug(done) + _logger.debug(f"Task which finished: {done}") - except (asyncio.TimeoutError, asyncio.CancelledError, TimeoutError): + except ( + asyncio.TimeoutError, + asyncio.CancelledError, + TimeoutError, + ) as e: # Stop the page load when timeout occurs - _logger.debug("Need to stop page loading, error.") + _logger.debug("Need to stop page loading, error.", exc_info=e) await temp_session.send_command("Page.stopLoading") raise finally: