testDifferentRolesMaintainPathOnRestart shouldn't use auto managing of min master nodes

It starts nodes in any order and thus it disabled the wait for first cluster state at node start up time
the later is required for the auto management logic.

Closes #23728

test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java

@@ -83,8 +83,8 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeValidationException;
 import org.elasticsearch.node.NodeService;
+import org.elasticsearch.node.NodeValidationException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchService;
@@ -129,6 +129,7 @@ import java.util.stream.Stream;
 
 import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY;
 import static org.apache.lucene.util.LuceneTestCase.rarely;
+import static org.elasticsearch.discovery.DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING;
 import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
 import static org.elasticsearch.test.ESTestCase.assertBusy;
 import static org.elasticsearch.test.ESTestCase.awaitBusy;
@@ -592,6 +593,9 @@ public final class InternalTestCluster extends TestCluster {
         if (autoManageMinMasterNodes) {
             assert finalSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()) == null :
                 "min master nodes may not be set when auto managed";
+            assert finalSettings.get(INITIAL_STATE_TIMEOUT_SETTING.getKey()) == null :
+                "automatically managing min master nodes require nodes to complete a join cycle" +
+                    "when starting";
             finalSettings
                 // don't wait too long not to slow down tests
                 .put(ZenDiscovery.MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING.getKey(), "5s")

test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java

@@ -40,6 +40,7 @@ import org.elasticsearch.transport.TransportSettings;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -344,12 +345,11 @@ public class InternalTestClusterTests extends ESTestCase {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/23728")
     public void testDifferentRolesMaintainPathOnRestart() throws Exception {
         final Path baseDir = createTempDir();
         final int numNodes = 5;
-        InternalTestCluster cluster = new InternalTestCluster(randomLong(), baseDir, true, true, 0, 0, "test",
-            new NodeConfigurationSource() {
+        InternalTestCluster cluster = new InternalTestCluster(randomLong(), baseDir, false,
+            false, 0, 0, "test", new NodeConfigurationSource() {
                 @Override
                 public Settings nodeSettings(int nodeOrdinal) {
                     return Settings.builder()
@@ -366,22 +366,32 @@ public class InternalTestClusterTests extends ESTestCase {
                 }
             }, 0, randomBoolean(), "", Arrays.asList(MockTcpTransportPlugin.class, TestZenDiscovery.TestPlugin.class), Function.identity());
         cluster.beforeTest(random(), 0.0);
+        List<DiscoveryNode.Role> roles = new ArrayList<>();
+        for (int i = 0; i < numNodes; i++) {
+            final DiscoveryNode.Role role = i == numNodes - 1 && roles.contains(MASTER) == false ?
+                MASTER : // last node and still no master
+                randomFrom(MASTER, DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST);
+            roles.add(role);
+        }
+
+        final Settings minMasterNodes = Settings.builder()
+            .put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(),
+                roles.stream().filter(role -> role == MASTER).count() / 2 + 1
+            ).build();
         try {
             Map<DiscoveryNode.Role, Set<String>> pathsPerRole = new HashMap<>();
             for (int i = 0; i < numNodes; i++) {
-                final DiscoveryNode.Role role = i == numNodes -1 && pathsPerRole.containsKey(MASTER) == false ?
-                    MASTER : // last noe and still no master ofr the cluster
-                    randomFrom(MASTER, DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST);
+                final DiscoveryNode.Role role = roles.get(i);
                 final String node;
                 switch (role) {
                     case MASTER:
-                        node = cluster.startMasterOnlyNode(Settings.EMPTY);
+                        node = cluster.startMasterOnlyNode(minMasterNodes);
                         break;
                     case DATA:
-                        node = cluster.startDataOnlyNode(Settings.EMPTY);
+                        node = cluster.startDataOnlyNode(minMasterNodes);
                         break;
                     case INGEST:
-                        node = cluster.startCoordinatingOnlyNode(Settings.EMPTY);
+                        node = cluster.startCoordinatingOnlyNode(minMasterNodes);
                         break;
                     default:
                         throw new IllegalStateException("get your story straight");