Handle NoSuchElementException exception when processing responses

ChromeDevToolsBase/pom.xml

@@ -1,12 +1,13 @@
-<?xml version="1.0" ?><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
-    <groupId>com.hubspot.chrome</groupId>
-    <artifactId>ChromeDevTools-parent</artifactId>
-    <version>138.0.7204.157-SNAPSHOT</version>
+    <groupId>uk.co.screamingfrog</groupId>
+    <artifactId>chromedevtools-parent</artifactId>
+    <version>138.0.7204.157-sf3</version>
   </parent>
 
-  <artifactId>ChromeDevToolsBase</artifactId>
+  <artifactId>chromedevtools-base</artifactId>
 
   <properties>
     <maven.javadoc.skip>true</maven.javadoc.skip>

ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeRequest.java

@@ -24,6 +24,7 @@ public class ChromeRequest {
   private final Integer id;
   private String method;
   private Map<String, Object> params;
+  private String sessionId;
 
   public ChromeRequest(String method) {
     this.id = requestNumber.getAndIncrement();
@@ -35,6 +36,10 @@ public Integer getId() {
     return id;
   }
 
+  public String getSessionId() {
+    return sessionId;
+  }
+
   public String getMethod() {
     return method;
   }
@@ -55,4 +60,8 @@ public ChromeRequest putParams(String key, Object value) {
     }
     return this;
   }
+
+  public void setSessionId(String sessionId) {
+    this.sessionId = sessionId;
+  }
 }

ChromeDevToolsBase/src/main/java/com/hubspot/chrome/devtools/base/ChromeVersionInfoIF.java

@@ -0,0 +1,26 @@
+package com.hubspot.chrome.devtools.base;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.immutables.value.Value;
+
+@Value.Immutable
+@ChromeStyle
+public interface ChromeVersionInfoIF {
+  @JsonProperty("Browser")
+  String getBrowser();
+
+  @JsonProperty("Protocol-Version")
+  String getProtocolVersion();
+
+  @JsonProperty("User-Agent")
+  String getUserAgent();
+
+  @JsonProperty("V8-Version")
+  String getV8Version();
+
+  @JsonProperty("WebKit-Version")
+  String getWebKitVersion();
+
+  @JsonProperty("webSocketDebuggerUrl")
+  String getWebSocketDebuggerUrl();
+}

ChromeDevToolsClient/pom.xml

@@ -1,13 +1,14 @@
-<?xml version="1.0" ?><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
-    <groupId>com.hubspot.chrome</groupId>
-    <artifactId>ChromeDevTools-parent</artifactId>
-    <version>138.0.7204.157-SNAPSHOT</version>
+    <groupId>uk.co.screamingfrog</groupId>
+    <artifactId>chromedevtools-parent</artifactId>
+    <version>138.0.7204.157-sf3</version>
   </parent>
 
-  <artifactId>ChromeDevToolsClient</artifactId>
+  <artifactId>chromedevtools-client</artifactId>
 
   <properties>
     <basepom.check.skip-findbugs>true</basepom.check.skip-findbugs>
@@ -35,8 +36,8 @@
       <artifactId>guava</artifactId>
     </dependency>
     <dependency>
-      <groupId>com.hubspot.chrome</groupId>
-      <artifactId>ChromeDevToolsBase</artifactId>
+      <groupId>uk.co.screamingfrog</groupId>
+      <artifactId>chromedevtools-base</artifactId>
     </dependency>
     <dependency>
       <groupId>com.hubspot</groupId>
@@ -84,8 +85,8 @@
         <dependencies>
           <!-- Ensures the Generator is compiled first so it can actually run -->
           <dependency>
-            <groupId>com.hubspot.chrome</groupId>
-            <artifactId>CodeGeneration</artifactId>
+            <groupId>uk.co.screamingfrog</groupId>
+            <artifactId>code-generation</artifactId>
             <version>${project.version}</version>
           </dependency>
         </dependencies>

ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsBrowserContext.java

@@ -0,0 +1,114 @@
+package com.hubspot.chrome.devtools.client;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.hubspot.chrome.devtools.base.ChromeRequest;
+import com.hubspot.chrome.devtools.client.core.browser.BrowserContextID;
+import com.hubspot.chrome.devtools.client.core.target.SessionID;
+import com.hubspot.chrome.devtools.client.core.target.Target;
+import com.hubspot.chrome.devtools.client.core.target.TargetID;
+import java.net.URI;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ChromeDevToolsBrowserContext extends ChromeDevToolsSession {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+    ChromeDevToolsBrowserContext.class
+  );
+  private static final String BLANK_TAB = "about:blank";
+  private BrowserContextID browserContextId;
+  private SessionID sessionId;
+
+  ChromeDevToolsBrowserContext(
+    final URI uri,
+    final ObjectMapper objectMapper,
+    final ExecutorService executorService,
+    final long actionTimeoutMillis
+  ) {
+    super(uri, objectMapper, executorService, actionTimeoutMillis);
+    this.browserContextId = null;
+    this.sessionId = null;
+  }
+
+  public void attach() {
+    if (sessionId == null) {
+      final Target target = getTarget();
+      browserContextId = target.createBrowserContext();
+      final TargetID targetId = target.createTarget(
+        BLANK_TAB,
+        null, // left
+        null, // top
+        null, // width
+        null, // height
+        null, // windowState
+        browserContextId
+      );
+
+      sessionId = target.attachToTarget(targetId, true);
+    } else {
+      LOG.warn("Already attached. sessionId={}", sessionId);
+    }
+  }
+
+  public BrowserContextID getBrowserContextId() {
+    return browserContextId;
+  }
+
+  @Override
+  public void close() {
+    if (browserContextId != null) {
+      sessionId = null;
+      getTarget().disposeBrowserContext(browserContextId);
+      browserContextId = null;
+    } else {
+      LOG.debug("Not attached");
+    }
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!super.equals(o)) {
+      return false;
+    }
+    if (getClass() != o.getClass()) {
+      return false;
+    }
+    final ChromeDevToolsBrowserContext other = (ChromeDevToolsBrowserContext) o;
+    return (
+      Objects.equals(
+        browserContextId != null ? browserContextId.getValue() : null,
+        other.browserContextId != null ? other.browserContextId.getValue() : null
+      ) &&
+      Objects.equals(
+        sessionId != null ? sessionId.getValue() : null,
+        other.sessionId != null ? other.sessionId.getValue() : null
+      )
+    );
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(
+      super.hashCode(),
+      browserContextId != null ? browserContextId.getValue() : null,
+      sessionId != null ? sessionId.getValue() : null
+    );
+  }
+
+  @Override
+  void sendChromeRequest(ChromeRequest request) {
+    addBrowserContextSessionIdIfRequired(request);
+    super.sendChromeRequest(request);
+  }
+
+  private void addBrowserContextSessionIdIfRequired(ChromeRequest request) {
+    if (sessionId != null) {
+      request.setSessionId(sessionId.getValue());
+    }
+  }
+}

ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsClient.java

@@ -8,6 +8,7 @@
 import com.github.rholder.retry.StopStrategies;
 import com.github.rholder.retry.WaitStrategies;
 import com.hubspot.chrome.devtools.base.ChromeSessionInfo;
+import com.hubspot.chrome.devtools.base.ChromeVersionInfo;
 import com.hubspot.chrome.devtools.client.core.target.TargetID;
 import com.hubspot.chrome.devtools.client.exceptions.ChromeDevToolsException;
 import com.hubspot.horizon.HttpClient;
@@ -82,6 +83,22 @@ public ChromeDevToolsSession connect(String host, int port) throws URISyntaxExce
     );
   }
 
+  public ChromeDevToolsBrowserContext createBrowserContext(String host, int port)
+    throws URISyntaxException {
+    final String wsUri = getWebSocketDebuggerUrl(host, port);
+    return createBrowserContext(wsUri);
+  }
+
+  public ChromeDevToolsBrowserContext createBrowserContext(final String wsUri)
+    throws URISyntaxException {
+    return new ChromeDevToolsBrowserContext(
+      new URI(wsUri),
+      objectMapper,
+      executorService,
+      actionTimeoutMillis
+    );
+  }
+
   @Override
   public void close() {
     try {
@@ -92,6 +109,24 @@ public void close() {
     }
   }
 
+  private String getWebSocketDebuggerUrl(String host, int port) {
+    final String url = String.format("http://%s:%d/json/version", host, port);
+    final HttpRequest httpRequest = HttpRequest
+      .newBuilder()
+      .setUrl(url)
+      .setMethod(Method.GET)
+      .build();
+
+    final HttpResponse response = httpClient.execute(httpRequest);
+
+    if (response.isError()) {
+      throw new ChromeDevToolsException("Unable to find available chrome version info.");
+    }
+
+    final ChromeVersionInfo versionInfo = response.getAs(new TypeReference<>() {});
+    return versionInfo.getWebSocketDebuggerUrl();
+  }
+
   private TargetID getFirstAvailableTargetId(String host, int port) {
     if (defaultStartNewTarget) {
       return startNewTarget(host, port);

ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/ChromeDevToolsSession.java

@@ -31,6 +31,7 @@
 import com.hubspot.chrome.devtools.client.core.domsnapshot.DOMSnapshot;
 import com.hubspot.chrome.devtools.client.core.domstorage.DOMStorage;
 import com.hubspot.chrome.devtools.client.core.emulation.Emulation;
+import com.hubspot.chrome.devtools.client.core.fetch.Fetch;
 import com.hubspot.chrome.devtools.client.core.headlessexperimental.HeadlessExperimental;
 import com.hubspot.chrome.devtools.client.core.heapprofiler.HeapProfiler;
 import com.hubspot.chrome.devtools.client.core.indexeddb.IndexedDB;
@@ -72,6 +73,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.Objects;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
@@ -178,7 +180,7 @@ public <T> CompletableFuture<T> sendAsync(
     );
   }
 
-  private void sendChromeRequest(ChromeRequest request) {
+  void sendChromeRequest(ChromeRequest request) {
     try {
       String json = objectMapper.writeValueAsString(request);
       LOG.trace("Sending request: {}", json);
@@ -207,17 +209,17 @@ private <T> T parseChromeResponse(ChromeResponse response, TypeReference<T> valu
     //
     //       Here the user must do `callingMethod().getProtocolVersion()` or `someMethod().getJsVersion()`.
     Iterator<JsonNode> elements = response.getResult().elements();
-    JsonNode first = elements.next();
     try {
       // We do our best to predict which kind of result to consume the response as, but there's
       // a small chance that a multi-result response has optional, absent members, and we try and
       // fail to parse it as a single-result response, which is why we catch the inner JsonMappingException.
+      JsonNode first = elements.next();
       if (elements.hasNext()) {
         return objectMapper.readValue(response.getResult().toString(), valueType);
       } else {
         return objectMapper.readValue(objectMapper.treeAsTokens(first), valueType);
       }
-    } catch (JsonMappingException e) {
+    } catch (JsonMappingException | NoSuchElementException e) {
       try {
         return objectMapper.readValue(response.getResult().toString(), valueType);
       } catch (IOException e1) {
@@ -690,6 +692,10 @@ public Emulation getEmulation() {
     return new Emulation(this, objectMapper);
   }
 
+  public Fetch getFetch() {
+    return new Fetch(this, objectMapper);
+  }
+
   public HeadlessExperimental getHeadlessExperimental() {
     return new HeadlessExperimental(this, objectMapper);
   }

ChromeDevToolsClient/src/main/java/com/hubspot/chrome/devtools/client/examples/BrowserContextExample.java

@@ -0,0 +1,54 @@
+package com.hubspot.chrome.devtools.client.examples;
+
+import com.hubspot.chrome.devtools.client.ChromeDevToolsBrowserContext;
+import com.hubspot.chrome.devtools.client.ChromeDevToolsClient;
+import com.hubspot.chrome.devtools.client.core.EventType;
+import com.hubspot.chrome.devtools.client.core.browser.BrowserContextID;
+import com.hubspot.chrome.devtools.client.core.network.Cookie;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+
+// Run chrome with args --headless --disable-gpu --remote-debugging-port=9292
+public class BrowserContextExample {
+
+  private static final String URL = "https://www.example.com/";
+
+  public static void main(String[] args) throws URISyntaxException {
+    // Create the client
+    ChromeDevToolsClient client = ChromeDevToolsClient.defaultClient();
+
+    // Get a browser context
+    try (
+      ChromeDevToolsBrowserContext context = client.createBrowserContext(
+        "127.0.0.1",
+        9292
+      )
+    ) {
+      context.attach();
+
+      final Semaphore sem = new Semaphore(0);
+      context.addEventConsumer(EventType.PAGE_LOAD_EVENT_FIRED, event -> sem.release());
+      context.getPage().enable();
+
+      context.navigate(URL);
+      final boolean loaded = sem.tryAcquire(1);
+      if (loaded) {
+        System.out.println("Loaded: " + URL);
+
+        final BrowserContextID browserContextId = context.getBrowserContextId();
+        List<Cookie> cookies = context.getStorage().getCookies(browserContextId);
+        for (var cookie : cookies) {
+          System.out.println("cookie: " + cookie.getName() + "=" + cookie.getValue());
+        }
+      } else {
+        System.out.println("Failed to load: " + URL);
+      }
+    }
+
+    // Close the client when we are done with it to cleanly shut down executors
+    client.close();
+
+    System.exit(0);
+  }
+}