GEODE-10556: Fix race condition in PulseSecurityWithSSLTest

geode-assembly/build.gradle

@@ -579,6 +579,11 @@ tasks.named('distributedTest') {
   ]
 }
 
+// Increase heap size for integrationTest to prevent OOM with Pulse SSL tests
+tasks.named('integrationTest') {
+  maxHeapSize = '2g'
+}
+
 
 acceptanceTest.dependsOn(rootProject.getTasksByName("publishToMavenLocal", true))
 installDist.dependsOn ':extensions:geode-modules-assembly:dist'

geode-assembly/geode-assembly-test/src/main/java/org/apache/geode/test/junit/rules/GeodeHttpClientRule.java

@@ -25,6 +25,7 @@
 import org.apache.hc.client5.http.classic.HttpClient;
 import org.apache.hc.client5.http.classic.methods.HttpGet;
 import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.client5.http.config.RequestConfig;
 import org.apache.hc.client5.http.cookie.BasicCookieStore;
 import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
 import org.apache.hc.client5.http.impl.DefaultRedirectStrategy;
@@ -41,6 +42,7 @@
 import org.apache.hc.core5.http.ProtocolException;
 import org.apache.hc.core5.http.message.BasicNameValuePair;
 import org.apache.hc.core5.net.URIBuilder;
+import org.apache.hc.core5.util.TimeValue;
 import org.junit.rules.ExternalResource;
 
 import org.apache.geode.internal.net.SocketCreatorFactory;
@@ -98,9 +100,56 @@ public ClassicHttpResponse loginToPulse(String username, String password) throws
     HttpClientContext freshLoginContext = HttpClientContext.create();
     // Explicitly set an empty cookie store to ensure no cookies from previous requests
     freshLoginContext.setCookieStore(new BasicCookieStore());
-    ClassicHttpResponse response = (ClassicHttpResponse) httpClientNoRedirect.execute(host,
-        buildHttpPost("/pulse/login", "username", username, "password", password),
-        freshLoginContext);
+
+    long startTime = System.currentTimeMillis();
+    System.err.println("[HTTP] ===== HTTP POST REQUEST START =====");
+    System.err
+        .println("[HTTP] Timestamp: " + java.time.Instant.now() + " (epoch: " + startTime + ")");
+    System.err.println("[HTTP] Target: " + host + "/pulse/login");
+    System.err.println("[HTTP] Username: " + username);
+    System.err.println("[HTTP] About to call httpClientNoRedirect.execute()...");
+
+    ClassicHttpResponse response;
+    try {
+      System.err.println("[HTTP] Executing HTTP request NOW at " + java.time.Instant.now());
+      response = (ClassicHttpResponse) httpClientNoRedirect.execute(host,
+          buildHttpPost("/pulse/login", "username", username, "password", password),
+          freshLoginContext);
+      long endTime = System.currentTimeMillis();
+      System.err.println("[HTTP] ===== HTTP POST REQUEST COMPLETED =====");
+      System.err.println("[HTTP] Duration: " + (endTime - startTime) + "ms");
+      System.err.println("[HTTP] Response code: " + response.getCode());
+      System.err.println("[HTTP] Response Location header: " +
+          (response.getFirstHeader("Location") != null
+              ? response.getFirstHeader("Location").getValue() : "null"));
+      System.err.println("[HTTP] Completed at: " + java.time.Instant.now());
+    } catch (org.apache.hc.client5.http.impl.classic.RequestFailedException e) {
+      long endTime = System.currentTimeMillis();
+      System.err.println("[HTTP] RequestFailedException after " + (endTime - startTime) +
+          "ms: " + e.getMessage() + " (request aborted/cancelled)");
+      throw e;
+    } catch (org.apache.hc.core5.http.ConnectionRequestTimeoutException e) {
+      long endTime = System.currentTimeMillis();
+      System.err.println("[HTTP] ConnectionRequestTimeoutException after " + (endTime - startTime) +
+          "ms: " + e.getMessage() + " (pool exhausted waiting for connection)");
+      throw e;
+    } catch (java.net.SocketTimeoutException e) {
+      long endTime = System.currentTimeMillis();
+      System.err.println("[HTTP] SocketTimeoutException after " + (endTime - startTime) +
+          "ms: " + e.getMessage() + " (connect or read timeout)");
+      throw e;
+    } catch (org.apache.hc.core5.http.ConnectionClosedException e) {
+      long endTime = System.currentTimeMillis();
+      System.err.println("[HTTP] ConnectionClosedException after " + (endTime - startTime) +
+          "ms: " + e.getMessage() + " (connection closed by server)");
+      throw e;
+    } catch (Exception e) {
+      long endTime = System.currentTimeMillis();
+      System.err
+          .println("[HTTP] " + e.getClass().getSimpleName() + " after " + (endTime - startTime) +
+              "ms: " + e.getMessage());
+      throw e;
+    }
     // If login is successful (302 redirect to clusterDetail), update shared context with new
     // session
     if (response.getCode() == 302 && response.getFirstHeader("Location") != null
@@ -160,6 +209,10 @@ private void connect() {
       return;
     }
 
+    System.err
+        .println("[HTTP-INIT] Starting HTTP client initialization at " + java.time.Instant.now());
+    System.err.println("[HTTP-INIT] SSL enabled: " + useSSL);
+
     // Jakarta EE migration: Create lenient redirect strategy for URIs with placeholders
     // Apache HttpComponents 5 is stricter about URI validation than version 4
     // This allows Spring Security OAuth2 redirect URIs with property placeholders
@@ -184,37 +237,91 @@ public boolean isRedirected(
       }
     };
 
-    host = new HttpHost(useSSL ? "https" : "http", hostName, portSupplier.get());
+    int port = portSupplier.get();
+    host = new HttpHost(useSSL ? "https" : "http", hostName, port);
+    System.err.println(
+        "[HTTP-INIT] Target host: " + (useSSL ? "https" : "http") + "://" + hostName + ":" + port);
+
+    // Configure timeouts to prevent tests from hanging indefinitely
+    // when Pulse JMX authentication backend is not ready.
+    // Using shorter timeouts (5s) allows await() retry logic to poll more frequently
+    // rather than waiting 30s per failed attempt.
+    RequestConfig requestConfig = RequestConfig.custom()
+        .setConnectionRequestTimeout(5000, java.util.concurrent.TimeUnit.MILLISECONDS)
+        .setConnectTimeout(5000, java.util.concurrent.TimeUnit.MILLISECONDS)
+        .setResponseTimeout(5000, java.util.concurrent.TimeUnit.MILLISECONDS)
+        .build();
+    System.err.println(
+        "[HTTP-INIT] Request config: ConnectionRequestTimeout=5s, ConnectTimeout=5s, ResponseTimeout=5s");
+
     if (useSSL) {
+      System.err.println("[HTTP-INIT] Configuring SSL connection manager...");
       HttpClientBuilder clientBuilder = HttpClients.custom();
       SSLContext ctx = SocketCreatorFactory
           .getSocketCreatorForComponent(SecurableCommunicationChannel.WEB).getSslContext();
       // HttpClient 5.x: SSL configuration via connection manager
+      // Increase pool size and enable eviction to prevent connection exhaustion during retries
       HttpClientConnectionManager connectionManager =
           PoolingHttpClientConnectionManagerBuilder.create()
+              .setMaxConnPerRoute(10) // Increase from default 2 to handle multiple retries
+              .setMaxConnTotal(20) // Increase from default 20
               .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
                   .setSslContext(ctx)
                   .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                   .build())
               .build();
+      System.err.println(
+          "[HTTP-INIT] SSL connection manager created: maxConnPerRoute=10, maxConnTotal=20");
       clientBuilder.setConnectionManager(connectionManager);
+      // Enable connection eviction to release stale connections
+      System.err
+          .println("[HTTP-INIT] Enabling connection eviction: expired immediately, idle after 10s");
+      clientBuilder.evictExpiredConnections();
+      clientBuilder.evictIdleConnections(TimeValue.ofSeconds(10));
       // Jakarta EE migration: Create client with lenient redirect strategy
       clientBuilder.setRedirectStrategy(lenientRedirectStrategy);
+      clientBuilder.setDefaultRequestConfig(requestConfig);
+      System.err.println("[HTTP-INIT] Building main HTTP client with SSL...");
       httpClient = clientBuilder.build();
+      System.err.println("[HTTP-INIT] Main HTTP client created successfully");
       // Jakarta EE migration: Create client without redirect handling for login verification
+      System.err.println("[HTTP-INIT] Building no-redirect HTTP client...");
       httpClientNoRedirect = HttpClients.custom()
           .setConnectionManager(connectionManager)
+          .setDefaultRequestConfig(requestConfig)
           .disableRedirectHandling()
           .build();
+      System.err.println("[HTTP-INIT] No-redirect HTTP client created successfully");
+      System.err.println(
+          "[HTTP-INIT] HTTP client initialization completed at " + java.time.Instant.now());
     } else {
+      System.err.println("[HTTP-INIT] Configuring non-SSL connection manager...");
+      // Configure connection pool for non-SSL client
+      HttpClientConnectionManager connectionManager =
+          PoolingHttpClientConnectionManagerBuilder.create()
+              .setMaxConnPerRoute(10)
+              .setMaxConnTotal(20)
+              .build();
+      System.err.println(
+          "[HTTP-INIT] Non-SSL connection manager created: maxConnPerRoute=10, maxConnTotal=20");
       // Jakarta EE migration: Create client with lenient redirect strategy
+      System.err.println("[HTTP-INIT] Building non-SSL HTTP clients...");
       httpClient = HttpClients.custom()
+          .setConnectionManager(connectionManager)
           .setRedirectStrategy(lenientRedirectStrategy)
+          .setDefaultRequestConfig(requestConfig)
+          .evictExpiredConnections()
+          .evictIdleConnections(TimeValue.ofSeconds(10))
           .build();
       // Jakarta EE migration: Create client without redirect handling for login verification
       httpClientNoRedirect = HttpClients.custom()
+          .setConnectionManager(connectionManager)
+          .setDefaultRequestConfig(requestConfig)
           .disableRedirectHandling()
           .build();
+      System.err.println("[HTTP-INIT] Non-SSL HTTP clients created successfully");
+      System.err.println(
+          "[HTTP-INIT] HTTP client initialization completed at " + java.time.Instant.now());
     }
   }