[Tests] Fix flakiness by ensuring stable cluster (#39300)

In integration tests where `setBootstrapMasterNodeIndex()` is used in
combination with `autoMinMasterNodes = false` the cluster can start
bootstrapping once the number of nodes set with the
`setBootstrapMasterNodeIndex` have been started but it's not ensured
that all nodes have successfully joined to form the cluster.

This behaviour was introduced with 5db7ed22a0aab6e630a89952ec94327d205fc3a3
and in order to ensure that the cluster is properly formed before proceeding
with the integration test, use `ensureStableCluster()` with the
appropriate number of expected nodes.

Fixes: #39220

modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java

@@ -118,6 +118,7 @@ public class Zen2RestApiIT extends ESNetty4IntegTestCase {
     public void testClearVotingTombstonesNotWaitingForRemoval() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(2);
         List<String> nodes = internalCluster().startNodes(3);
+        ensureStableCluster(3);
         RestClient restClient = getRestClient();
         Response response = restClient.performRequest(new Request("POST", "/_cluster/voting_config_exclusions/" + nodes.get(2)));
         assertThat(response.getStatusLine().getStatusCode(), is(200));
@@ -131,6 +132,7 @@ public class Zen2RestApiIT extends ESNetty4IntegTestCase {
     public void testClearVotingTombstonesWaitingForRemoval() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(2);
         List<String> nodes = internalCluster().startNodes(3);
+        ensureStableCluster(3);
         RestClient restClient = getRestClient();
         String nodeToWithdraw = nodes.get(randomIntBetween(0, 2));
         Response response = restClient.performRequest(new Request("POST", "/_cluster/voting_config_exclusions/" + nodeToWithdraw));
@@ -145,6 +147,7 @@ public class Zen2RestApiIT extends ESNetty4IntegTestCase {
     public void testFailsOnUnknownNode() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(2);
         internalCluster().startNodes(3);
+        ensureStableCluster(3);
         RestClient restClient = getRestClient();
         try {
             restClient.performRequest(new Request("POST", "/_cluster/voting_config_exclusions/invalid"));

server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java

@@ -284,7 +284,7 @@ public class MinimumMasterNodesIT extends ESIntegTestCase {
                 .build();
 
         internalCluster().startNodes(3, settings);
-        ensureGreen(); // ensure cluster state is recovered before we disrupt things
+        ensureStableCluster(3);
 
         final String master = internalCluster().getMasterName();
         Set<String> otherNodes = new HashSet<>(Arrays.asList(internalCluster().getNodeNames()));

server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java

@@ -250,7 +250,6 @@ public class ElasticsearchNodeCommandIT extends ESIntegTestCase {
         expectThrows(() -> detachCluster(environment, 0, true), ElasticsearchNodeCommand.ABORTED_BY_USER_MSG);
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/39220")
     public void test3MasterNodes2Failed() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(2);
         List<String> masterNodes = new ArrayList<>();
@@ -268,8 +267,12 @@ public class ElasticsearchNodeCommandIT extends ESIntegTestCase {
         logger.info("--> start 2nd and 3rd master-eligible nodes and bootstrap");
         masterNodes.addAll(internalCluster().startMasterOnlyNodes(2)); // node ordinals 2 and 3
 
+        logger.info("--> wait for all nodes to join the cluster");
+        ensureStableCluster(4);
+
         logger.info("--> create index test");
         createIndex("test");
+        ensureGreen("test");
 
         logger.info("--> stop 2nd and 3d master eligible node");
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(1)));