Zen2: Move all mixed-version REST tests to Zen2 (#36398)

Moves all remaining (rolling-upgrade and mixed-version) REST tests to use Zen2. To avoid adding
extra configuration, it relies on Zen2 being set as the default discovery type. This required a few
smaller changes in other tests. I've removed AzureMinimumMasterNodesTests which tests Zen1
functionality and dates from a time where host providers were not configurable and each cloud
plugin had its own discovery.type, subclassing the ZenDiscovery class. I've also adapted a few tests
which were unnecessarily adding addTestZenDiscovery = false for the same legacy reasons. Finally,
this also moves the unconfigured-node-name REST test to Zen2, testing the auto-bootstrapping
functionality in development mode when no discovery configuration is provided.

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -76,6 +76,22 @@ class ClusterConfiguration {
         }
     }
 
+    /**
+     * Whether the initial_master_nodes setting should be automatically derived from the nodes
+     * in the cluster. Only takes effect if all nodes in the cluster understand this setting
+     * and the discovery type is not explicitly set.
+     */
+    @Input
+    boolean autoSetInitialMasterNodes = true
+
+    /**
+     * Whether the file-based discovery provider should be automatically setup based on
+     * the nodes in the cluster. Only takes effect if no other hosts provider is already
+     * configured.
+     */
+    @Input
+    boolean autoSetHostsProvider = true
+
     @Input
     String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
         " " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -131,13 +131,15 @@ class ClusterFormationTasks {
             Object dependsOn
             if (node.nodeVersion.onOrAfter("6.5.0")) {
                 writeConfigSetup = { Map esConfig ->
-                    // Don't force discovery provider if one is set by the test cluster specs already
-                    if (esConfig.containsKey('discovery.zen.hosts_provider') == false) {
-                        esConfig['discovery.zen.hosts_provider'] = 'file'
+                    if (config.getAutoSetHostsProvider()) {
+                        // Don't force discovery provider if one is set by the test cluster specs already
+                        if (esConfig.containsKey('discovery.zen.hosts_provider') == false) {
+                            esConfig['discovery.zen.hosts_provider'] = 'file'
+                        }
+                        esConfig['discovery.zen.ping.unicast.hosts'] = []
                     }
-                    esConfig['discovery.zen.ping.unicast.hosts'] = []
-                    if (hasBwcNodes == false && esConfig['discovery.type'] == null) {
-                        esConfig['discovery.type'] = 'zen2'
+                    boolean supportsInitialMasterNodes = hasBwcNodes == false || config.bwcVersion.onOrAfter("7.0.0")
+                    if (esConfig['discovery.type'] == null && config.getAutoSetInitialMasterNodes() && supportsInitialMasterNodes) {
                         esConfig['cluster.initial_master_nodes'] = nodes.stream().map({ n ->
                             if (n.config.settings['node.name'] == null) {
                                 return "node-" + n.nodeNum

plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java

@@ -84,11 +84,6 @@ public abstract class AbstractAzureComputeServiceTestCase extends ESIntegTestCas
         return Collections.singletonList(TestPlugin.class);
     }
 
-    @Override
-    protected boolean addTestZenDiscovery() {
-        return false;
-    }
-
     /**
      * Register an existing node as a Azure node, exposing its address and details htrough
      *

plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java

@@ -1,71 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.discovery.azure.classic;
-
-import org.elasticsearch.cloud.azure.classic.AbstractAzureComputeServiceTestCase;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.discovery.MasterNotDiscoveredException;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.io.IOException;
-
-/**
- * Reported issue in #15
- * (https://github.com/elastic/elasticsearch-cloud-azure/issues/15)
- */
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE,
-        numDataNodes = 0,
-        transportClientRatio = 0.0,
-        numClientNodes = 0,
-        autoMinMasterNodes = false)
-public class AzureMinimumMasterNodesTests extends AbstractAzureComputeServiceTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put("discovery.zen.minimum_master_nodes", 2)
-                .put("discovery.initial_state_timeout", "1s")
-            .build();
-    }
-
-    public void testSimpleOnlyMasterNodeElection() throws IOException {
-        final String node1 = internalCluster().startNode();
-        registerAzureNode(node1);
-        expectThrows(MasterNotDiscoveredException.class, () ->
-            // master is not elected yet
-            client().admin().cluster().prepareState().setMasterNodeTimeout("100ms").get().getState().nodes().getMasterNodeId()
-        );
-
-        final String node2 = internalCluster().startNode();
-        registerAzureNode(node2);
-        assertNotNull(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId());
-
-        internalCluster().stopCurrentMasterNode();
-        expectThrows(MasterNotDiscoveredException.class, () ->
-            // master has been stopped
-            client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId()
-        );
-
-        final String node3 = internalCluster().startNode();
-        registerAzureNode(node3);
-        assertNotNull(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId());
-    }
-}

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractAwsTestCase.java

@@ -69,9 +69,4 @@ public abstract class AbstractAwsTestCase extends ESIntegTestCase {
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(Ec2DiscoveryPlugin.class);
     }
-
-    @Override
-    protected boolean addTestZenDiscovery() {
-        return false;
-    }
 }

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsTests.java

@@ -33,7 +33,7 @@ import static org.hamcrest.CoreMatchers.is;
  * starting.
  * This test requires AWS to run.
  */
-@ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0, autoMinMasterNodes = false)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
 public class Ec2DiscoveryUpdateSettingsTests extends AbstractAwsTestCase {
     public void testMinimumMasterNodesStart() {
         Settings nodeSettings = Settings.builder()

plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java

@@ -73,11 +73,6 @@ public class GceDiscoverTests extends ESIntegTestCase {
             .build();
     }
 
-    @Override
-    protected boolean addTestZenDiscovery() {
-        return false;
-    }
-
     public void testJoin() {
         // start master node
         final String masterNode = internalCluster().startMasterOnlyNode();

qa/rolling-upgrade/build.gradle

@@ -78,13 +78,12 @@ for (Version version : bwcVersions.wireCompatible) {
       clusterName = 'rolling-upgrade'
       otherUnicastHostAddresses = { getOtherUnicastHostAddresses() }
       minimumMasterNodes = { 2 }
+      autoSetInitialMasterNodes = false
       /* Override the data directory so the new node always gets the node we
       * just stopped's data directory. */
       dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
       setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
       setting 'node.name', "upgraded-node-${stopNode}"
-      // TODO: Move to Zen2 once we support rolling upgrade with Zen2
-      setting 'discovery.type', 'zen'
     }
   }
 

qa/unconfigured-node-name/build.gradle

@@ -22,8 +22,10 @@ apply plugin: 'elasticsearch.rest-test'
 
 integTestCluster {
   setting 'node.name', null
-  // TODO: Run this using zen2, with no discovery configuration at all, demonstrating that the node forms a cluster on its own without help
-  setting 'discovery.type', 'zen'
+  // Run with no discovery configuration at all, demonstrating that a node in its
+  // "out-of-the-box" configuration can automatically bootstrap a cluster
+  autoSetInitialMasterNodes = false
+  autoSetHostsProvider = false
 }
 
 integTestRunner {

server/src/main/java/org/elasticsearch/cluster/coordination/ClusterBootstrapService.java

@@ -103,6 +103,8 @@ public class ClusterBootstrapService {
                 transportService.getThreadPool().scheduleUnlessShuttingDown(unconfiguredBootstrapTimeout, Names.SAME, new Runnable() {
                     @Override
                     public void run() {
+                        // TODO: remove the following line once schedule method properly preserves thread context
+                        threadContext.markAsSystemContext();
                         final GetDiscoveredNodesRequest request = new GetDiscoveredNodesRequest();
                         logger.trace("sending {}", request);
                         transportService.sendRequest(transportService.getLocalNode(), GetDiscoveredNodesAction.NAME, request,
@@ -212,6 +214,8 @@ public class ClusterBootstrapService {
                     transportService.getThreadPool().scheduleUnlessShuttingDown(TimeValue.timeValueSeconds(10), Names.SAME, new Runnable() {
                         @Override
                         public void run() {
+                            // TODO: remove the following line once schedule method properly preserves thread context
+                            transportService.getThreadPool().getThreadContext().markAsSystemContext();
                             awaitBootstrap(bootstrapConfiguration);
                         }
 

server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java

@@ -73,7 +73,7 @@ public class DiscoveryModule {
     public static final String ZEN2_DISCOVERY_TYPE = "zen2";
 
     public static final Setting<String> DISCOVERY_TYPE_SETTING =
-        new Setting<>("discovery.type", ZEN_DISCOVERY_TYPE, Function.identity(), Property.NodeScope);
+        new Setting<>("discovery.type", ZEN2_DISCOVERY_TYPE, Function.identity(), Property.NodeScope);
     public static final Setting<List<String>> DISCOVERY_HOSTS_PROVIDER_SETTING =
         Setting.listSetting("discovery.zen.hosts_provider", Collections.emptyList(), Function.identity(), Property.NodeScope);
 

server/src/test/java/org/elasticsearch/action/admin/cluster/bootstrap/TransportBootstrapClusterActionTests.java

@@ -33,6 +33,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -102,7 +103,10 @@ public class TransportBootstrapClusterActionTests extends ESTestCase {
         final Discovery discovery = mock(Discovery.class);
         verifyZeroInteractions(discovery);
 
-        new TransportBootstrapClusterAction(Settings.EMPTY, EMPTY_FILTERS, transportService, discovery); // registers action
+        final String nonstandardDiscoveryType = randomFrom(DiscoveryModule.ZEN_DISCOVERY_TYPE, "single-node", "unknown");
+        new TransportBootstrapClusterAction(
+            Settings.builder().put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), nonstandardDiscoveryType).build(),
+            EMPTY_FILTERS, transportService, discovery); // registers action
         transportService.start();
         transportService.acceptIncomingRequests();
 
@@ -117,7 +121,8 @@ public class TransportBootstrapClusterActionTests extends ESTestCase {
             public void handleException(TransportException exp) {
                 final Throwable rootCause = exp.getRootCause();
                 assertThat(rootCause, instanceOf(IllegalArgumentException.class));
-                assertThat(rootCause.getMessage(), equalTo("cluster bootstrapping is not supported by discovery type [zen]"));
+                assertThat(rootCause.getMessage(), equalTo("cluster bootstrapping is not supported by discovery type [" +
+                    nonstandardDiscoveryType + "]"));
                 countDownLatch.countDown();
             }
         });

server/src/test/java/org/elasticsearch/action/admin/cluster/bootstrap/TransportGetDiscoveredNodesActionTests.java

@@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.Discovery;
+import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.PeersRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransport;
@@ -124,7 +125,10 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
         final Discovery discovery = mock(Discovery.class);
         verifyZeroInteractions(discovery);
 
-        new TransportGetDiscoveredNodesAction(Settings.EMPTY, EMPTY_FILTERS, transportService, discovery); // registers action
+        final String nonstandardDiscoveryType = randomFrom(DiscoveryModule.ZEN_DISCOVERY_TYPE, "single-node", "unknown");
+        new TransportGetDiscoveredNodesAction(
+            Settings.builder().put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), nonstandardDiscoveryType).build(),
+            EMPTY_FILTERS, transportService, discovery); // registers action
         transportService.start();
         transportService.acceptIncomingRequests();
 
@@ -139,7 +143,8 @@ public class TransportGetDiscoveredNodesActionTests extends ESTestCase {
             public void handleException(TransportException exp) {
                 final Throwable rootCause = exp.getRootCause();
                 assertThat(rootCause, instanceOf(IllegalArgumentException.class));
-                assertThat(rootCause.getMessage(), equalTo("discovered nodes are not exposed by discovery type [zen]"));
+                assertThat(rootCause.getMessage(), equalTo("discovered nodes are not exposed by discovery type [" +
+                    nonstandardDiscoveryType + "]"));
                 countDownLatch.countDown();
             }
         });

server/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java

@@ -20,6 +20,7 @@ package org.elasticsearch.discovery;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.coordination.Coordinator;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterApplier;
@@ -107,7 +108,7 @@ public class DiscoveryModuleTests extends ESTestCase {
 
     public void testDefaults() {
         DiscoveryModule module = newModule(Settings.EMPTY, Collections.emptyList());
-        assertTrue(module.getDiscovery() instanceof ZenDiscovery);
+        assertTrue(module.getDiscovery() instanceof Coordinator);
     }
 
     public void testLazyConstructionDiscovery() {
@@ -205,7 +206,9 @@ public class DiscoveryModuleTests extends ESTestCase {
 
     public void testJoinValidator() {
         BiConsumer<DiscoveryNode, ClusterState> consumer = (a, b) -> {};
-        DiscoveryModule module = newModule(Settings.EMPTY, Collections.singletonList(new DiscoveryPlugin() {
+        // TODO: move to zen2 once join validators are implemented
+        DiscoveryModule module = newModule(Settings.builder().put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(),
+            DiscoveryModule.ZEN_DISCOVERY_TYPE).build(), Collections.singletonList(new DiscoveryPlugin() {
             @Override
             public BiConsumer<DiscoveryNode, ClusterState> getJoinValidator() {
                 return consumer;

x-pack/qa/rolling-upgrade-basic/build.gradle

@@ -56,6 +56,7 @@ for (Version version : bwcVersions.wireCompatible) {
             clusterName = 'rolling-upgrade-basic'
             otherUnicastHostAddresses = { getOtherUnicastHostAddresses() }
             minimumMasterNodes = { 2 }
+            autoSetInitialMasterNodes = false
             /* Override the data directory so the new node always gets the node we
             * just stopped's data directory. */
             dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
@@ -66,8 +67,6 @@ for (Version version : bwcVersions.wireCompatible) {
             setting 'xpack.watcher.enabled', 'false'
             setting 'xpack.license.self_generated.type', 'basic'
             setting 'node.name', "upgraded-node-${stopNode}"
-            // TODO: Move to Zen2 once we support rolling upgrade with Zen2
-            setting 'discovery.type', 'zen'
         }
     }
 

x-pack/qa/rolling-upgrade/build.gradle

@@ -188,6 +188,7 @@ subprojects {
         clusterName = 'rolling-upgrade'
         otherUnicastHostAddresses = { getOtherUnicastHostAddresses() }
         minimumMasterNodes = { 2 }
+        autoSetInitialMasterNodes = false
         /* Override the data directory so the new node always gets the node we
         * just stopped's data directory. */
         dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
@@ -215,8 +216,6 @@ subprojects {
         if (version.before('6.0.0')) {
           keystoreSetting 'xpack.security.authc.token.passphrase', 'token passphrase'
         }
-        // TODO: Move to Zen2 once we support rolling upgrade with Zen2
-        setting 'discovery.type', 'zen'
       }
     }
 

x-pack/qa/vagrant/src/test/resources/packaging/tests/certgen.bash

@@ -249,6 +249,7 @@ node.name: "node-master"
 node.master: true
 node.data: false
 discovery.zen.ping.unicast.hosts: ["127.0.0.1:9301"]
+cluster.initial_master_nodes: ["node-master"]
 
 xpack.ssl.key: $ESCONFIG/certs/node-master/node-master.key
 xpack.ssl.certificate: $ESCONFIG/certs/node-master/node-master.crt