Randomize initial state timeout (#92817)

We should run the test suite with `discovery.initial_state_timeout: 0s`
sometimes, to verify the behaviour if a master node is not elected
immediately at startup. We don't do this today, and indeed we forbid
this setting in some circumstances (since #23728, i.e. 55a3fd1). This
restriction doesn't apply any more, so this commit removes it and
randomly removes the startup wait.

server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java

@@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
 import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.plugins.Plugin;
@@ -50,6 +51,11 @@ import static org.hamcrest.Matchers.notNullValue;
 
 public class IngestFileSettingsIT extends ESIntegTestCase {
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings));
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(CustomIngestTestPlugin.class);

server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/ComponentTemplatesFileSettingsIT.java

@@ -52,6 +52,11 @@ import static org.hamcrest.Matchers.notNullValue;
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class ComponentTemplatesFileSettingsIT extends ESIntegTestCase {
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings));
+    }
+
     private static AtomicLong versionCounter = new AtomicLong(1);
 
     private static String emptyJSON = """

server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java

@@ -45,6 +45,11 @@ import static org.hamcrest.Matchers.nullValue;
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class FileSettingsServiceIT extends ESIntegTestCase {
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings));
+    }
+
     private static AtomicLong versionCounter = new AtomicLong(1);
 
     private static String testJSON = """

server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/RepositoriesFileSettingsIT.java

@@ -50,6 +50,11 @@ import static org.hamcrest.Matchers.notNullValue;
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class RepositoriesFileSettingsIT extends ESIntegTestCase {
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings));
+    }
+
     private static AtomicLong versionCounter = new AtomicLong(1);
 
     private static String testJSON = """

server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/SnapshotsAndFileSettingsIT.java

@@ -47,6 +47,12 @@ import static org.hamcrest.Matchers.equalTo;
  */
 @LuceneTestCase.SuppressFileSystems("*")
 public class SnapshotsAndFileSettingsIT extends AbstractSnapshotIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings));
+    }
+
     private static AtomicLong versionCounter = new AtomicLong(1);
 
     private static String testFileSettingsJSON = """

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -21,6 +21,7 @@ import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TotalHits;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteResponse;
@@ -185,6 +186,7 @@ import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_SEED_PROVIDE
 import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING;
 import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING;
 import static org.elasticsearch.test.XContentTestUtils.convertToMap;
 import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -1915,6 +1917,11 @@ public abstract class ESIntegTestCase extends ESTestCase {
         return builder.build();
     }
 
+    protected static Settings applyWorkaroundForIssue92812(Settings settings) {
+        assertTrue("this setting hides a blocking bug, we must remove it ASAP", Version.CURRENT.onOrBefore(Version.V_8_7_0));
+        return Settings.builder().put(settings).put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "30s").build();
+    }
+
     protected Path nodeConfigPath(int nodeOrdinal) {
         return null;
     }

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -152,7 +152,6 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
@@ -603,6 +602,12 @@ public final class InternalTestCluster extends TestCluster {
             );
         }
 
+        if (random.nextBoolean()) {
+            // Nodes may fail to join a cluster within the default 30s timeout so we must sometimes skip that wait to ensure there's nothing
+            // in the startup process which relies on having joined a cluster.
+            builder.put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s");
+        }
+
         return builder.build();
     }
 
@@ -731,14 +736,6 @@ public final class InternalTestCluster extends TestCluster {
         updatedSettings.put("node.name", name);
         updatedSettings.put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed);
 
-        if (autoManageMasterNodes) {
-            assertThat(
-                "if master nodes are automatically managed then nodes must complete a join cycle when starting",
-                updatedSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()),
-                nullValue()
-            );
-        }
-
         return updatedSettings.build();
     }
 

x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/AutoscalingFileSettingsIT.java

@@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata;
 import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
 import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.reservedstate.service.FileSettingsService;
@@ -44,6 +45,11 @@ import static org.hamcrest.Matchers.notNullValue;
  */
 public class AutoscalingFileSettingsIT extends AutoscalingIntegTestCase {
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings));
+    }
+
     private static AtomicLong versionCounter = new AtomicLong(1);
 
     private static String testJSON = """

x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/slm/SLMFileSettingsIT.java

@@ -148,7 +148,7 @@ public class SLMFileSettingsIT extends AbstractSnapshotIntegTestCase {
     @Override
     protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
         return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal, otherSettings))
+            .put(applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings)))
             .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED, false)
             .put(LifecycleSettings.SLM_HISTORY_INDEX_ENABLED, false)
             .build();

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java

@@ -62,6 +62,11 @@ import static org.hamcrest.Matchers.notNullValue;
  */
 public class RoleMappingFileSettingsIT extends NativeRealmIntegTestCase {
 
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings));
+    }
+
     private static AtomicLong versionCounter = new AtomicLong(1);
 
     private static String emptyJSON = """

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsRestartIT.java

@@ -13,6 +13,7 @@ import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.ReservedStateHandlerMetadata;
 import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.reservedstate.service.FileSettingsService;
@@ -38,6 +39,12 @@ import static org.hamcrest.Matchers.notNullValue;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class FileSettingsRoleMappingsRestartIT extends SecurityIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings));
+    }
+
     private static AtomicLong versionCounter = new AtomicLong(1);
 
     private static String testJSONOnlyRoleMappings = """

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/FileSettingsRoleMappingsStartupIT.java

@@ -30,6 +30,12 @@ import static org.elasticsearch.test.NodeRoles.dataOnlyNode;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class FileSettingsRoleMappingsStartupIT extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
+        return applyWorkaroundForIssue92812(super.nodeSettings(nodeOrdinal, otherSettings));
+    }
+
     private static AtomicLong versionCounter = new AtomicLong(1);
     private static String testJSONForFailedCase = """
         {