Add a flag to use minimim_master_nodes on the integ test cluster (#21173)

Setting `discovery.initial_state_timeout: 0s` to make `discovery.zen.minimum_master_nodes: N`
work reliably can cause issues in clusters that rely on state recovery once the cluster is available.
This change makes the use or `discovery.zen.minimum_master_nodes` optional for clusters where this behavior is desirable.

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -62,6 +62,15 @@ class ClusterConfiguration {
     @Input
     boolean debug = false
 
+    /**
+     * if <code>true</code> each node will be configured with <tt>discovery.zen.minimum_master_nodes</tt> set
+     * to the total number of nodes in the cluster. This will also cause that each node has `0s` state recovery
+     * timeout which can lead to issues if for instance an existing clusterstate is expected to be recovered
+     * before any tests start
+     */
+    @Input
+    boolean useMinimumMasterNodes = true
+
     @Input
     String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
             " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
@@ -95,11 +104,11 @@ class ClusterConfiguration {
     @Input
     Closure waitCondition = { NodeInfo node, AntBuilder ant ->
         File tmpFile = new File(node.cwd, 'wait.success')
-        ant.echo("==> [${new Date()}] checking health: http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}")
+        ant.echo("==> [${new Date()}] checking health: http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow")
         // checking here for wait_for_nodes to be >= the number of nodes because its possible
         // this cluster is attempting to connect to nodes created by another task (same cluster name),
         // so there will be more nodes in that case in the cluster state
-        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}",
+        ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
                 dest: tmpFile.toString(),
                 ignoreerrors: true, // do not fail on error, so logging buffers can be flushed by the wait task
                 retries: 10)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -259,7 +259,7 @@ class ClusterFormationTasks {
         // basically skip initial state recovery to allow the cluster to form using a realistic master election
         // this means all nodes must be up, join the seed node and do a master election. This will also allow new and
         // old nodes in the BWC case to become the master
-        if (node.config.numNodes > 1) {
+        if (node.config.useMinimumMasterNodes && node.config.numNodes > 1) {
             esConfig['discovery.zen.minimum_master_nodes'] = node.config.numNodes
             esConfig['discovery.initial_state_timeout'] = '0s' // don't wait for state.. just start up quickly
         }