diff --git a/ChromeDevToolsBase/pom.xml b/ChromeDevToolsBase/pom.xml index bd254d2..24128a0 100644 --- a/ChromeDevToolsBase/pom.xml +++ b/ChromeDevToolsBase/pom.xml @@ -1,12 +1,13 @@ - + + 4.0.0 - com.hubspot.chrome - ChromeDevTools-parent - 138.0.7204.157-SNAPSHOT + uk.co.screamingfrog + chromedevtools-parent + 138.0.7204.157-sf3 - ChromeDevToolsBase + chromedevtools-base true diff --git a/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeRequest.java b/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeRequest.java index 60446cf..e279a48 100644 --- a/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeRequest.java +++ b/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeRequest.java @@ -24,6 +24,7 @@ public class ChromeRequest { private final Integer id; private String method; private Map params; + private String sessionId; public ChromeRequest(String method) { this.id = requestNumber.getAndIncrement(); @@ -35,6 +36,10 @@ public Integer getId() { return id; } + public String getSessionId() { + return sessionId; + } + public String getMethod() { return method; } @@ -55,4 +60,8 @@ public ChromeRequest putParams(String key, Object value) { } return this; } + + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } } diff --git a/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeVersionInfoIF.java b/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeVersionInfoIF.java new file mode 100644 index 0000000..5418d9a --- /dev/null +++ b/ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeVersionInfoIF.java @@ -0,0 +1,26 @@ +package com.hubspot.chrome.devtools.base; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.immutables.value.Value; + +@Value.Immutable +@ChromeStyle +public interface ChromeVersionInfoIF { + @JsonProperty("Browser") + String getBrowser(); + + @JsonProperty("Protocol-Version") + String getProtocolVersion(); + + @JsonProperty("User-Agent") + String getUserAgent(); + + @JsonProperty("V8-Version") + String getV8Version(); + + @JsonProperty("WebKit-Version") + String getWebKitVersion(); + + @JsonProperty("webSocketDebuggerUrl") + String getWebSocketDebuggerUrl(); +} diff --git a/ChromeDevToolsClient/pom.xml b/ChromeDevToolsClient/pom.xml index 398046b..ef92432 100644 --- a/ChromeDevToolsClient/pom.xml +++ b/ChromeDevToolsClient/pom.xml @@ -1,13 +1,14 @@ - + + 4.0.0 - com.hubspot.chrome - ChromeDevTools-parent - 138.0.7204.157-SNAPSHOT + uk.co.screamingfrog + chromedevtools-parent + 138.0.7204.157-sf3 - ChromeDevToolsClient + chromedevtools-client true @@ -35,8 +36,8 @@ guava - com.hubspot.chrome - ChromeDevToolsBase + uk.co.screamingfrog + chromedevtools-base com.hubspot @@ -84,8 +85,8 @@ - com.hubspot.chrome - CodeGeneration + uk.co.screamingfrog + code-generation ${project.version} diff --git a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsBrowserContext.java b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsBrowserContext.java new file mode 100644 index 0000000..d10fc22 --- /dev/null +++ b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsBrowserContext.java @@ -0,0 +1,114 @@ +package com.hubspot.chrome.devtools.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.hubspot.chrome.devtools.base.ChromeRequest; +import com.hubspot.chrome.devtools.client.core.browser.BrowserContextID; +import com.hubspot.chrome.devtools.client.core.target.SessionID; +import com.hubspot.chrome.devtools.client.core.target.Target; +import com.hubspot.chrome.devtools.client.core.target.TargetID; +import java.net.URI; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ChromeDevToolsBrowserContext extends ChromeDevToolsSession { + + private static final Logger LOG = LoggerFactory.getLogger( + ChromeDevToolsBrowserContext.class + ); + private static final String BLANK_TAB = "about:blank"; + private BrowserContextID browserContextId; + private SessionID sessionId; + + ChromeDevToolsBrowserContext( + final URI uri, + final ObjectMapper objectMapper, + final ExecutorService executorService, + final long actionTimeoutMillis + ) { + super(uri, objectMapper, executorService, actionTimeoutMillis); + this.browserContextId = null; + this.sessionId = null; + } + + public void attach() { + if (sessionId == null) { + final Target target = getTarget(); + browserContextId = target.createBrowserContext(); + final TargetID targetId = target.createTarget( + BLANK_TAB, + null, // left + null, // top + null, // width + null, // height + null, // windowState + browserContextId + ); + + sessionId = target.attachToTarget(targetId, true); + } else { + LOG.warn("Already attached. sessionId={}", sessionId); + } + } + + public BrowserContextID getBrowserContextId() { + return browserContextId; + } + + @Override + public void close() { + if (browserContextId != null) { + sessionId = null; + getTarget().disposeBrowserContext(browserContextId); + browserContextId = null; + } else { + LOG.debug("Not attached"); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!super.equals(o)) { + return false; + } + if (getClass() != o.getClass()) { + return false; + } + final ChromeDevToolsBrowserContext other = (ChromeDevToolsBrowserContext) o; + return ( + Objects.equals( + browserContextId != null ? browserContextId.getValue() : null, + other.browserContextId != null ? other.browserContextId.getValue() : null + ) && + Objects.equals( + sessionId != null ? sessionId.getValue() : null, + other.sessionId != null ? other.sessionId.getValue() : null + ) + ); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), + browserContextId != null ? browserContextId.getValue() : null, + sessionId != null ? sessionId.getValue() : null + ); + } + + @Override + void sendChromeRequest(ChromeRequest request) { + addBrowserContextSessionIdIfRequired(request); + super.sendChromeRequest(request); + } + + private void addBrowserContextSessionIdIfRequired(ChromeRequest request) { + if (sessionId != null) { + request.setSessionId(sessionId.getValue()); + } + } +} diff --git a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsClient.java b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsClient.java index 893c613..e754c75 100644 --- a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsClient.java +++ b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsClient.java @@ -8,6 +8,7 @@ import com.github.rholder.retry.StopStrategies; import com.github.rholder.retry.WaitStrategies; import com.hubspot.chrome.devtools.base.ChromeSessionInfo; +import com.hubspot.chrome.devtools.base.ChromeVersionInfo; import com.hubspot.chrome.devtools.client.core.target.TargetID; import com.hubspot.chrome.devtools.client.exceptions.ChromeDevToolsException; import com.hubspot.horizon.HttpClient; @@ -82,6 +83,22 @@ public ChromeDevToolsSession connect(String host, int port) throws URISyntaxExce ); } + public ChromeDevToolsBrowserContext createBrowserContext(String host, int port) + throws URISyntaxException { + final String wsUri = getWebSocketDebuggerUrl(host, port); + return createBrowserContext(wsUri); + } + + public ChromeDevToolsBrowserContext createBrowserContext(final String wsUri) + throws URISyntaxException { + return new ChromeDevToolsBrowserContext( + new URI(wsUri), + objectMapper, + executorService, + actionTimeoutMillis + ); + } + @Override public void close() { try { @@ -92,6 +109,24 @@ public void close() { } } + private String getWebSocketDebuggerUrl(String host, int port) { + final String url = String.format("http://%s:%d/json/version", host, port); + final HttpRequest httpRequest = HttpRequest + .newBuilder() + .setUrl(url) + .setMethod(Method.GET) + .build(); + + final HttpResponse response = httpClient.execute(httpRequest); + + if (response.isError()) { + throw new ChromeDevToolsException("Unable to find available chrome version info."); + } + + final ChromeVersionInfo versionInfo = response.getAs(new TypeReference<>() {}); + return versionInfo.getWebSocketDebuggerUrl(); + } + private TargetID getFirstAvailableTargetId(String host, int port) { if (defaultStartNewTarget) { return startNewTarget(host, port); diff --git a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java index 039c59c..209bb20 100644 --- a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java +++ b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java @@ -31,6 +31,7 @@ import com.hubspot.chrome.devtools.client.core.domsnapshot.DOMSnapshot; import com.hubspot.chrome.devtools.client.core.domstorage.DOMStorage; import com.hubspot.chrome.devtools.client.core.emulation.Emulation; +import com.hubspot.chrome.devtools.client.core.fetch.Fetch; import com.hubspot.chrome.devtools.client.core.headlessexperimental.HeadlessExperimental; import com.hubspot.chrome.devtools.client.core.heapprofiler.HeapProfiler; import com.hubspot.chrome.devtools.client.core.indexeddb.IndexedDB; @@ -72,6 +73,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -178,7 +180,7 @@ public CompletableFuture sendAsync( ); } - private void sendChromeRequest(ChromeRequest request) { + void sendChromeRequest(ChromeRequest request) { try { String json = objectMapper.writeValueAsString(request); LOG.trace("Sending request: {}", json); @@ -207,17 +209,17 @@ private T parseChromeResponse(ChromeResponse response, TypeReference valu // // Here the user must do `callingMethod().getProtocolVersion()` or `someMethod().getJsVersion()`. Iterator elements = response.getResult().elements(); - JsonNode first = elements.next(); try { // We do our best to predict which kind of result to consume the response as, but there's // a small chance that a multi-result response has optional, absent members, and we try and // fail to parse it as a single-result response, which is why we catch the inner JsonMappingException. + JsonNode first = elements.next(); if (elements.hasNext()) { return objectMapper.readValue(response.getResult().toString(), valueType); } else { return objectMapper.readValue(objectMapper.treeAsTokens(first), valueType); } - } catch (JsonMappingException e) { + } catch (JsonMappingException | NoSuchElementException e) { try { return objectMapper.readValue(response.getResult().toString(), valueType); } catch (IOException e1) { @@ -690,6 +692,10 @@ public Emulation getEmulation() { return new Emulation(this, objectMapper); } + public Fetch getFetch() { + return new Fetch(this, objectMapper); + } + public HeadlessExperimental getHeadlessExperimental() { return new HeadlessExperimental(this, objectMapper); } diff --git a/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/examples/BrowserContextExample.java b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/examples/BrowserContextExample.java new file mode 100644 index 0000000..99a3e73 --- /dev/null +++ b/ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/examples/BrowserContextExample.java @@ -0,0 +1,54 @@ +package com.hubspot.chrome.devtools.client.examples; + +import com.hubspot.chrome.devtools.client.ChromeDevToolsBrowserContext; +import com.hubspot.chrome.devtools.client.ChromeDevToolsClient; +import com.hubspot.chrome.devtools.client.core.EventType; +import com.hubspot.chrome.devtools.client.core.browser.BrowserContextID; +import com.hubspot.chrome.devtools.client.core.network.Cookie; +import java.net.URISyntaxException; +import java.util.List; +import java.util.concurrent.Semaphore; + +// Run chrome with args --headless --disable-gpu --remote-debugging-port=9292 +public class BrowserContextExample { + + private static final String URL = "https://www.example.com/"; + + public static void main(String[] args) throws URISyntaxException { + // Create the client + ChromeDevToolsClient client = ChromeDevToolsClient.defaultClient(); + + // Get a browser context + try ( + ChromeDevToolsBrowserContext context = client.createBrowserContext( + "127.0.0.1", + 9292 + ) + ) { + context.attach(); + + final Semaphore sem = new Semaphore(0); + context.addEventConsumer(EventType.PAGE_LOAD_EVENT_FIRED, event -> sem.release()); + context.getPage().enable(); + + context.navigate(URL); + final boolean loaded = sem.tryAcquire(1); + if (loaded) { + System.out.println("Loaded: " + URL); + + final BrowserContextID browserContextId = context.getBrowserContextId(); + List cookies = context.getStorage().getCookies(browserContextId); + for (var cookie : cookies) { + System.out.println("cookie: " + cookie.getName() + "=" + cookie.getValue()); + } + } else { + System.out.println("Failed to load: " + URL); + } + } + + // Close the client when we are done with it to cleanly shut down executors + client.close(); + + System.exit(0); + } +} diff --git a/CodeGeneration/pom.xml b/CodeGeneration/pom.xml index 9c66431..f7d1de9 100644 --- a/CodeGeneration/pom.xml +++ b/CodeGeneration/pom.xml @@ -1,13 +1,14 @@ - + + 4.0.0 - com.hubspot.chrome - ChromeDevTools-parent - 138.0.7204.157-SNAPSHOT + uk.co.screamingfrog + chromedevtools-parent + 138.0.7204.157-sf3 - CodeGeneration + code-generation @@ -15,7 +16,9 @@ com.fasterxml.jackson.datatype jackson-datatype-guava ${dep.jackson.version} - + + + @@ -46,8 +49,8 @@ error_prone_annotations - com.hubspot.chrome - ChromeDevToolsBase + uk.co.screamingfrog + chromedevtools-base com.squareup diff --git a/README.md b/README.md index f11680c..b2eb4f5 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,26 @@ client.close(); The `ChromeDevToolsSession` class provides all methods for interacting with Chrome. There are synchronous and asynchronous versions available for each method. -You can also check out our [examples](TODO) for more. +You can also check out our [examples](ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/examples) for more. + +## Usage with Browser Contexts + +Browser Contexts can be used to run multiple independent browser sessions; thus providing session isolation. +The context is used in the same manner as the `ChromeDevToolsSession` described previously, however sessions are isolated in that cookie, local storage and caches are not shared. + +```java +// Create the client +ChromeDevToolsClient client = ChromeDevToolsClient.defaultClient(); + +// Connect to Chrome Dev Tools Running on port 9292 on your local machine and create a browser context +try (ChromeDevToolsBrowserContext context = client.createBrowserContext("127.0.0.1", 9292)) { + context.attach(); + context.navigate("https://www.hubspot.com/"); +} + +// Close the client when your finished +client.close(); +``` ## Configuring ChromeDevToolsClient diff --git a/pom.xml b/pom.xml index d2407b6..1e00451 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,5 @@ - + + 4.0.0 @@ -6,9 +7,9 @@ basepom 63.4 - com.hubspot.chrome - ChromeDevTools-parent - 138.0.7204.157-SNAPSHOT + uk.co.screamingfrog + chromedevtools-parent + 138.0.7204.157-sf3 pom @@ -26,13 +27,13 @@ - com.hubspot.chrome - ChromeDevToolsBase + uk.co.screamingfrog + chromedevtools-base ${project.version} - com.hubspot.chrome - CodeGeneration + uk.co.screamingfrog + code-generation ${project.version} @@ -85,4 +86,19 @@ + + + + sf-cdt-client + https://maven.pkg.github.com/screamingfrog/ChromeDevToolsClient + + + + + + sf-cdt-client + GitHub Packages + https://maven.pkg.github.com/screamingfrog/ChromeDevToolsClient + + diff --git a/release/README.md b/release/README.md index fdb22f9..acf5ba6 100644 --- a/release/README.md +++ b/release/README.md @@ -10,7 +10,7 @@ https://source.chromium.org/chromium/chromium/src/+/refs/tags/91.0.4472.114:thir Copy the content of this file to CodeGeneration/src/main/resources/browser_protocol.pdl -Run the pdl_to_json.py scipt to update the corresponding json file: +Run the pdl_to_json.py script to update the corresponding json file: ``` python pdl_to_json.py --pdl_file ../CodeGeneration/src/main/resources/browser_protocol.pdl --json_file ../CodeGeneration/src/main/resources/browser_protocol.json