Fix SAML test initialisation flakiness

x-pack/plugin/security/qa/saml-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlRestTestCase.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.test.junit.RunnableTestRuleAdapter;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
@@ -52,10 +53,7 @@ public class SamlRestTestCase extends ESRestTestCase {
     private static Path caPath;
 
     @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(new RunnableTestRuleAdapter(SamlRestTestCase::initWebserver))
-        .around(cluster)
-        // during the startup, the metadata remains unavailable to prevent caching. After cluster init, make metadata available.
-        .around(new RunnableTestRuleAdapter(() -> makeMetadataAvailable(1, 2, 3)));
+    public static TestRule ruleChain = RuleChain.outerRule(new RunnableTestRuleAdapter(SamlRestTestCase::initWebserver)).around(cluster);
 
     private static void initWebserver() {
         try {
@@ -159,7 +157,7 @@ protected static InetSocketAddress getIdpHttpsAddress() {
     }
 
     private static void configureMetadataResource(int realmNumber) throws CertificateException, IOException, URISyntaxException {
-        metadataAvailable.putIfAbsent(realmNumber, false);
+        metadataAvailable.put(realmNumber, false);
 
         var signingCert = getDataResource(SAML_SIGNING_CRT);
         var metadataBody = new SamlIdpMetadataBuilder().entityId(getIdpEntityId(realmNumber)).sign(signingCert).asString();
@@ -194,6 +192,22 @@ public static void loadCertificateAuthority() throws Exception {
         caPath = PathUtils.get(resource.toURI());
     }
 
+    /**
+     * Make metadata available by default before each test, but make this behaviour controllable by subclasses.
+     */
+    @Before
+    public void initMetadata() {
+        if (isMetadataAvailable()) {
+            makeMetadataAvailable(1, 2, 3);
+        } else {
+            makeAllMetadataUnavailable();
+        }
+    }
+
+    protected boolean isMetadataAvailable() {
+        return true;
+    }
+
     @Override
     protected String getTestRestCluster() {
         return cluster.getHttpAddresses();

x-pack/plugin/security/qa/saml-rest-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/saml/SamlServiceProviderMetadataIT.java

@@ -25,9 +25,16 @@
 
 public class SamlServiceProviderMetadataIT extends SamlRestTestCase {
 
+    /**
+     * Within this class we the metadata not to be enabled at the start of each test
+     */
+    @Override
+    protected boolean isMetadataAvailable() {
+        return false;
+    }
+
     public void testAuthenticationWhenMetadataIsUnreliable() throws Exception {
-        // Start with no metadata available
-        makeAllMetadataUnavailable();
+        // initially, metadata in unavailable for all realms
 
         final String username = randomAlphaOfLengthBetween(4, 12);
         for (int realmNumber : shuffledList(List.of(1, 2, 3))) {