From 193b183c2e14e42753ac9d05b5a3a8c5359b4fcc Mon Sep 17 00:00:00 2001 From: David Cockbill Date: Tue, 9 Sep 2025 13:51:01 +0100 Subject: [PATCH 1/4] Fetch Domain access #81 - Add getFetch method to ChromeDevToolsSession (cherry picked from commit dd38e73327a6f7fb0833677bdf0ec2968af3c5b7) --- .../chrome/devtools/client/ChromeDevToolsSession.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java index 039c59c..4bbc5b8 100644 --- a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java +++ b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java @@ -31,6 +31,7 @@ import com.hubspot.chrome.devtools.client.core.domsnapshot.DOMSnapshot; import com.hubspot.chrome.devtools.client.core.domstorage.DOMStorage; import com.hubspot.chrome.devtools.client.core.emulation.Emulation; +import com.hubspot.chrome.devtools.client.core.fetch.Fetch; import com.hubspot.chrome.devtools.client.core.headlessexperimental.HeadlessExperimental; import com.hubspot.chrome.devtools.client.core.heapprofiler.HeapProfiler; import com.hubspot.chrome.devtools.client.core.indexeddb.IndexedDB; @@ -690,6 +691,10 @@ public Emulation getEmulation() { return new Emulation(this, objectMapper); } + public Fetch getFetch() { + return new Fetch(this, objectMapper); + } + public HeadlessExperimental getHeadlessExperimental() { return new HeadlessExperimental(this, objectMapper); } From 691a65c3328aa30e35bcd1a43643a51703f1a470 Mon Sep 17 00:00:00 2001 From: David Cockbill Date: Thu, 11 Sep 2025 14:05:38 +0100 Subject: [PATCH 2/4] Add support for Browser Contexts #80 - Use `ChromeDevToolsClient.createBrowserContext()`` to create a Browser Context. - Attach to the target with `ChromeDevToolsBrowserContext.attach()` - There are two creation methods; one using host and port, the other directly using WebSocket address. (cherry picked from commit b35223edf00276b25da85eec753fd3011d481cdd) --- .../chrome/devtools/base/ChromeRequest.java | 9 ++ .../devtools/base/ChromeVersionInfoIF.java | 26 ++++ .../client/ChromeDevToolsBrowserContext.java | 114 ++++++++++++++++++ .../devtools/client/ChromeDevToolsClient.java | 35 ++++++ .../client/ChromeDevToolsSession.java | 2 +- .../examples/BrowserContextExample.java | 54 +++++++++ README.md | 21 +++- release/README.md | 2 +- 8 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeVersionInfoIF.java create mode 100644 ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsBrowserContext.java create mode 100644 ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/examples/BrowserContextExample.java diff --git a/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeRequest.java b/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeRequest.java index 60446cf..e279a48 100644 --- a/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeRequest.java +++ b/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeRequest.java @@ -24,6 +24,7 @@ public class ChromeRequest { private final Integer id; private String method; private Map params; + private String sessionId; public ChromeRequest(String method) { this.id = requestNumber.getAndIncrement(); @@ -35,6 +36,10 @@ public Integer getId() { return id; } + public String getSessionId() { + return sessionId; + } + public String getMethod() { return method; } @@ -55,4 +60,8 @@ public ChromeRequest putParams(String key, Object value) { } return this; } + + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } } diff --git a/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeVersionInfoIF.java b/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeVersionInfoIF.java new file mode 100644 index 0000000..5418d9a --- /dev/null +++ b/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeVersionInfoIF.java @@ -0,0 +1,26 @@ +package com.hubspot.chrome.devtools.base; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.immutables.value.Value; + +@Value.Immutable +@ChromeStyle +public interface ChromeVersionInfoIF { + @JsonProperty("Browser") + String getBrowser(); + + @JsonProperty("Protocol-Version") + String getProtocolVersion(); + + @JsonProperty("User-Agent") + String getUserAgent(); + + @JsonProperty("V8-Version") + String getV8Version(); + + @JsonProperty("WebKit-Version") + String getWebKitVersion(); + + @JsonProperty("webSocketDebuggerUrl") + String getWebSocketDebuggerUrl(); +} diff --git a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsBrowserContext.java b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsBrowserContext.java new file mode 100644 index 0000000..d10fc22 --- /dev/null +++ b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsBrowserContext.java @@ -0,0 +1,114 @@ +package com.hubspot.chrome.devtools.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.hubspot.chrome.devtools.base.ChromeRequest; +import com.hubspot.chrome.devtools.client.core.browser.BrowserContextID; +import com.hubspot.chrome.devtools.client.core.target.SessionID; +import com.hubspot.chrome.devtools.client.core.target.Target; +import com.hubspot.chrome.devtools.client.core.target.TargetID; +import java.net.URI; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ChromeDevToolsBrowserContext extends ChromeDevToolsSession { + + private static final Logger LOG = LoggerFactory.getLogger( + ChromeDevToolsBrowserContext.class + ); + private static final String BLANK_TAB = "about:blank"; + private BrowserContextID browserContextId; + private SessionID sessionId; + + ChromeDevToolsBrowserContext( + final URI uri, + final ObjectMapper objectMapper, + final ExecutorService executorService, + final long actionTimeoutMillis + ) { + super(uri, objectMapper, executorService, actionTimeoutMillis); + this.browserContextId = null; + this.sessionId = null; + } + + public void attach() { + if (sessionId == null) { + final Target target = getTarget(); + browserContextId = target.createBrowserContext(); + final TargetID targetId = target.createTarget( + BLANK_TAB, + null, // left + null, // top + null, // width + null, // height + null, // windowState + browserContextId + ); + + sessionId = target.attachToTarget(targetId, true); + } else { + LOG.warn("Already attached. sessionId={}", sessionId); + } + } + + public BrowserContextID getBrowserContextId() { + return browserContextId; + } + + @Override + public void close() { + if (browserContextId != null) { + sessionId = null; + getTarget().disposeBrowserContext(browserContextId); + browserContextId = null; + } else { + LOG.debug("Not attached"); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!super.equals(o)) { + return false; + } + if (getClass() != o.getClass()) { + return false; + } + final ChromeDevToolsBrowserContext other = (ChromeDevToolsBrowserContext) o; + return ( + Objects.equals( + browserContextId != null ? browserContextId.getValue() : null, + other.browserContextId != null ? other.browserContextId.getValue() : null + ) && + Objects.equals( + sessionId != null ? sessionId.getValue() : null, + other.sessionId != null ? other.sessionId.getValue() : null + ) + ); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), + browserContextId != null ? browserContextId.getValue() : null, + sessionId != null ? sessionId.getValue() : null + ); + } + + @Override + void sendChromeRequest(ChromeRequest request) { + addBrowserContextSessionIdIfRequired(request); + super.sendChromeRequest(request); + } + + private void addBrowserContextSessionIdIfRequired(ChromeRequest request) { + if (sessionId != null) { + request.setSessionId(sessionId.getValue()); + } + } +} diff --git a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsClient.java b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsClient.java index 893c613..e754c75 100644 --- a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsClient.java +++ b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsClient.java @@ -8,6 +8,7 @@ import com.github.rholder.retry.StopStrategies; import com.github.rholder.retry.WaitStrategies; import com.hubspot.chrome.devtools.base.ChromeSessionInfo; +import com.hubspot.chrome.devtools.base.ChromeVersionInfo; import com.hubspot.chrome.devtools.client.core.target.TargetID; import com.hubspot.chrome.devtools.client.exceptions.ChromeDevToolsException; import com.hubspot.horizon.HttpClient; @@ -82,6 +83,22 @@ public ChromeDevToolsSession connect(String host, int port) throws URISyntaxExce ); } + public ChromeDevToolsBrowserContext createBrowserContext(String host, int port) + throws URISyntaxException { + final String wsUri = getWebSocketDebuggerUrl(host, port); + return createBrowserContext(wsUri); + } + + public ChromeDevToolsBrowserContext createBrowserContext(final String wsUri) + throws URISyntaxException { + return new ChromeDevToolsBrowserContext( + new URI(wsUri), + objectMapper, + executorService, + actionTimeoutMillis + ); + } + @Override public void close() { try { @@ -92,6 +109,24 @@ public void close() { } } + private String getWebSocketDebuggerUrl(String host, int port) { + final String url = String.format("http://%s:%d/json/version", host, port); + final HttpRequest httpRequest = HttpRequest + .newBuilder() + .setUrl(url) + .setMethod(Method.GET) + .build(); + + final HttpResponse response = httpClient.execute(httpRequest); + + if (response.isError()) { + throw new ChromeDevToolsException("Unable to find available chrome version info."); + } + + final ChromeVersionInfo versionInfo = response.getAs(new TypeReference<>() {}); + return versionInfo.getWebSocketDebuggerUrl(); + } + private TargetID getFirstAvailableTargetId(String host, int port) { if (defaultStartNewTarget) { return startNewTarget(host, port); diff --git a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java index 4bbc5b8..7b42b6d 100644 --- a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java +++ b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java @@ -179,7 +179,7 @@ public CompletableFuture sendAsync( ); } - private void sendChromeRequest(ChromeRequest request) { + void sendChromeRequest(ChromeRequest request) { try { String json = objectMapper.writeValueAsString(request); LOG.trace("Sending request: {}", json); diff --git a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/examples/BrowserContextExample.java b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/examples/BrowserContextExample.java new file mode 100644 index 0000000..99a3e73 --- /dev/null +++ b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/examples/BrowserContextExample.java @@ -0,0 +1,54 @@ +package com.hubspot.chrome.devtools.client.examples; + +import com.hubspot.chrome.devtools.client.ChromeDevToolsBrowserContext; +import com.hubspot.chrome.devtools.client.ChromeDevToolsClient; +import com.hubspot.chrome.devtools.client.core.EventType; +import com.hubspot.chrome.devtools.client.core.browser.BrowserContextID; +import com.hubspot.chrome.devtools.client.core.network.Cookie; +import java.net.URISyntaxException; +import java.util.List; +import java.util.concurrent.Semaphore; + +// Run chrome with args --headless --disable-gpu --remote-debugging-port=9292 +public class BrowserContextExample { + + private static final String URL = "https://www.example.com/"; + + public static void main(String[] args) throws URISyntaxException { + // Create the client + ChromeDevToolsClient client = ChromeDevToolsClient.defaultClient(); + + // Get a browser context + try ( + ChromeDevToolsBrowserContext context = client.createBrowserContext( + "127.0.0.1", + 9292 + ) + ) { + context.attach(); + + final Semaphore sem = new Semaphore(0); + context.addEventConsumer(EventType.PAGE_LOAD_EVENT_FIRED, event -> sem.release()); + context.getPage().enable(); + + context.navigate(URL); + final boolean loaded = sem.tryAcquire(1); + if (loaded) { + System.out.println("Loaded: " + URL); + + final BrowserContextID browserContextId = context.getBrowserContextId(); + List cookies = context.getStorage().getCookies(browserContextId); + for (var cookie : cookies) { + System.out.println("cookie: " + cookie.getName() + "=" + cookie.getValue()); + } + } else { + System.out.println("Failed to load: " + URL); + } + } + + // Close the client when we are done with it to cleanly shut down executors + client.close(); + + System.exit(0); + } +} diff --git a/README.md b/README.md index f11680c..b2eb4f5 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,26 @@ client.close(); The `ChromeDevToolsSession` class provides all methods for interacting with Chrome. There are synchronous and asynchronous versions available for each method. -You can also check out our [examples](TODO) for more. +You can also check out our [examples](ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/examples) for more. + +## Usage with Browser Contexts + +Browser Contexts can be used to run multiple independent browser sessions; thus providing session isolation. +The context is used in the same manner as the `ChromeDevToolsSession` described previously, however sessions are isolated in that cookie, local storage and caches are not shared. + +```java +// Create the client +ChromeDevToolsClient client = ChromeDevToolsClient.defaultClient(); + +// Connect to Chrome Dev Tools Running on port 9292 on your local machine and create a browser context +try (ChromeDevToolsBrowserContext context = client.createBrowserContext("127.0.0.1", 9292)) { + context.attach(); + context.navigate("https://www.hubspot.com/"); +} + +// Close the client when your finished +client.close(); +``` ## Configuring ChromeDevToolsClient diff --git a/release/README.md b/release/README.md index fdb22f9..acf5ba6 100644 --- a/release/README.md +++ b/release/README.md @@ -10,7 +10,7 @@ https://source.chromium.org/chromium/chromium/src/+/refs/tags/91.0.4472.114:thir Copy the content of this file to CodeGeneration/src/main/resources/browser_protocol.pdl -Run the pdl_to_json.py scipt to update the corresponding json file: +Run the pdl_to_json.py script to update the corresponding json file: ``` python pdl_to_json.py --pdl_file ../CodeGeneration/src/main/resources/browser_protocol.pdl --json_file ../CodeGeneration/src/main/resources/browser_protocol.json From cb46897517999c195b47415aaf903e5969b36a12 Mon Sep 17 00:00:00 2001 From: David Cockbill Date: Fri, 12 Sep 2025 09:55:21 +0100 Subject: [PATCH 3/4] Screaming Frog Release - Change groupId: uk.co.screamingfrog - Add Distribution Management to parent POM - Change artifactId (422: Unprocessable Entity) (cherry picked from commit cd0208217296868105e74386038d2484a6d36bdf) --- ChromeDevToolsBase/pom.xml | 11 ++++++----- ChromeDevToolsClient/pom.xml | 19 ++++++++++--------- CodeGeneration/pom.xml | 19 +++++++++++-------- pom.xml | 32 ++++++++++++++++++++++++-------- 4 files changed, 51 insertions(+), 30 deletions(-) diff --git a/ChromeDevToolsBase/pom.xml b/ChromeDevToolsBase/pom.xml index bd254d2..24128a0 100644 --- a/ChromeDevToolsBase/pom.xml +++ b/ChromeDevToolsBase/pom.xml @@ -1,12 +1,13 @@ - + + 4.0.0 - com.hubspot.chrome - ChromeDevTools-parent - 138.0.7204.157-SNAPSHOT + uk.co.screamingfrog + chromedevtools-parent + 138.0.7204.157-sf3 - ChromeDevToolsBase + chromedevtools-base true diff --git a/ChromeDevToolsClient/pom.xml b/ChromeDevToolsClient/pom.xml index 398046b..ef92432 100644 --- a/ChromeDevToolsClient/pom.xml +++ b/ChromeDevToolsClient/pom.xml @@ -1,13 +1,14 @@ - + + 4.0.0 - com.hubspot.chrome - ChromeDevTools-parent - 138.0.7204.157-SNAPSHOT + uk.co.screamingfrog + chromedevtools-parent + 138.0.7204.157-sf3 - ChromeDevToolsClient + chromedevtools-client true @@ -35,8 +36,8 @@ guava - com.hubspot.chrome - ChromeDevToolsBase + uk.co.screamingfrog + chromedevtools-base com.hubspot @@ -84,8 +85,8 @@ - com.hubspot.chrome - CodeGeneration + uk.co.screamingfrog + code-generation ${project.version} diff --git a/CodeGeneration/pom.xml b/CodeGeneration/pom.xml index 9c66431..f7d1de9 100644 --- a/CodeGeneration/pom.xml +++ b/CodeGeneration/pom.xml @@ -1,13 +1,14 @@ - + + 4.0.0 - com.hubspot.chrome - ChromeDevTools-parent - 138.0.7204.157-SNAPSHOT + uk.co.screamingfrog + chromedevtools-parent + 138.0.7204.157-sf3 - CodeGeneration + code-generation @@ -15,7 +16,9 @@ com.fasterxml.jackson.datatype jackson-datatype-guava ${dep.jackson.version} - + + + @@ -46,8 +49,8 @@ error_prone_annotations - com.hubspot.chrome - ChromeDevToolsBase + uk.co.screamingfrog + chromedevtools-base com.squareup diff --git a/pom.xml b/pom.xml index d2407b6..1e00451 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,5 @@ - + + 4.0.0 @@ -6,9 +7,9 @@ basepom 63.4 - com.hubspot.chrome - ChromeDevTools-parent - 138.0.7204.157-SNAPSHOT + uk.co.screamingfrog + chromedevtools-parent + 138.0.7204.157-sf3 pom @@ -26,13 +27,13 @@ - com.hubspot.chrome - ChromeDevToolsBase + uk.co.screamingfrog + chromedevtools-base ${project.version} - com.hubspot.chrome - CodeGeneration + uk.co.screamingfrog + code-generation ${project.version} @@ -85,4 +86,19 @@ + + + + sf-cdt-client + https://maven.pkg.github.com/screamingfrog/ChromeDevToolsClient + + + + + + sf-cdt-client + GitHub Packages + https://maven.pkg.github.com/screamingfrog/ChromeDevToolsClient + + From 9b7fb62273a7a2b868332c6b08dd5f26474ed52b Mon Sep 17 00:00:00 2001 From: David Cockbill Date: Mon, 15 Sep 2025 11:25:15 +0100 Subject: [PATCH 4/4] Response parsing fails for empty results #83 - Handle response results with no return parameters - Catch NoSuchElementException and perform normal results extraction. --- .../chrome/devtools/client/ChromeDevToolsSession.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java index 7b42b6d..209bb20 100644 --- a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java +++ b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java @@ -73,6 +73,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -208,17 +209,17 @@ private T parseChromeResponse(ChromeResponse response, TypeReference valu // // Here the user must do `callingMethod().getProtocolVersion()` or `someMethod().getJsVersion()`. Iterator elements = response.getResult().elements(); - JsonNode first = elements.next(); try { // We do our best to predict which kind of result to consume the response as, but there's // a small chance that a multi-result response has optional, absent members, and we try and // fail to parse it as a single-result response, which is why we catch the inner JsonMappingException. + JsonNode first = elements.next(); if (elements.hasNext()) { return objectMapper.readValue(response.getResult().toString(), valueType); } else { return objectMapper.readValue(objectMapper.treeAsTokens(first), valueType); } - } catch (JsonMappingException e) { + } catch (JsonMappingException | NoSuchElementException e) { try { return objectMapper.readValue(response.getResult().toString(), valueType); } catch (IOException e1) {