diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderTragicEventTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderTragicEventTest.java index 01d84378474..03fcd8a7a7a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderTragicEventTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderTragicEventTest.java @@ -56,6 +56,7 @@ public static void setupCluster() throws Exception { configureCluster(2) .addConfig( "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .withShutdownTimeoutIsError(false) .configure(); } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index a632928d361..f6c8f07501d 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -160,6 +160,7 @@ public class MiniSolrCloudCluster { private final JettyConfig jettyConfig; private final String solrXml; private final boolean trackJettyMetrics; + private final boolean shutdownTimeoutIsError; private final AtomicInteger nodeIds = new AtomicInteger(); private final Map solrClientByCollection = new ConcurrentHashMap<>(); @@ -242,7 +243,8 @@ public MiniSolrCloudCluster( zkTestServer, securityJson, false, - formatZkServer); + formatZkServer, + true); } /** @@ -257,6 +259,7 @@ public MiniSolrCloudCluster( * @param zkTestServer ZkTestServer to use. If null, one will be created * @param securityJson A string representation of security.json file (optional). * @param trackJettyMetrics supply jetties with metrics registry + * @param shutdownTimeoutIsError whether timeout during shutdown is an error (default true) * @throws Exception if there was an error starting the cluster */ MiniSolrCloudCluster( @@ -267,7 +270,8 @@ public MiniSolrCloudCluster( ZkTestServer zkTestServer, Optional securityJson, boolean trackJettyMetrics, - boolean formatZkServer) + boolean formatZkServer, + boolean shutdownTimeoutIsError) throws Exception { Objects.requireNonNull(securityJson); @@ -275,6 +279,7 @@ public MiniSolrCloudCluster( this.jettyConfig = Objects.requireNonNull(jettyConfig); this.solrXml = solrXml == null ? DEFAULT_CLOUD_SOLR_XML : solrXml; this.trackJettyMetrics = trackJettyMetrics; + this.shutdownTimeoutIsError = shutdownTimeoutIsError; log.info("Starting cluster of {} servers in {}", numServers, baseDir); @@ -670,11 +675,26 @@ public void shutdown() throws Exception { for (final JettySolrRunner jetty : jettys) { shutdowns.add(() -> stopJettySolrRunner(jetty)); } - jettys.clear(); + final ExecutorService executorCloser = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrNamedThreadFactory("jetty-closer")); - Collection> futures = executorCloser.invokeAll(shutdowns); + + // Use a 60 second timeout to prevent indefinite hangs during shutdown, especially when cores + // are in a bad state (e.g., after tragic events). This is 2x Jetty's internal timeout. + List> futures; + try { + futures = executorCloser.invokeAll(shutdowns, 60, TimeUnit.SECONDS); + } catch (InterruptedException e) { + log.warn("Interrupted while shutting down jettys", e); + Thread.currentThread().interrupt(); + executorCloser.shutdownNow(); + throw e; + } + ExecutorUtil.shutdownAndAwaitTermination(executorCloser); + + jettys.clear(); + Exception shutdownError = checkForExceptions("Error shutting down MiniSolrCloudCluster", futures); if (shutdownError != null) { @@ -773,9 +793,16 @@ private Exception checkForExceptions(String message, Collection