Migrate RecoveryFromGatewayIT to Zen2 (#37520)

* Fixes `testTwoNodeFirstNodeCleared` by manipulating voting config exclusions.

* Removes `testRecoveryDifferentNodeOrderStartup` since state recovery is now
  handled entirely on the elected master, so the order in which the data nodes
  start is irrelevant.

server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java

@@ -20,7 +20,10 @@
 package org.elasticsearch.gateway;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
-
+import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
+import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
+import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction;
+import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
@@ -33,7 +36,6 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
@@ -49,9 +51,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.InternalTestCluster.RestartCallback;
-import org.elasticsearch.test.discovery.TestZenDiscovery;
 import org.elasticsearch.test.store.MockFSIndexStore;
 
 import java.nio.file.DirectoryStream;
@@ -66,9 +66,11 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.IntStream;
 
+import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.gateway.GatewayService.RECOVER_AFTER_NODES_SETTING;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -88,14 +90,6 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
         return Arrays.asList(MockFSIndexStore.TestPlugin.class, InternalSettingsPlugin.class);
     }
 
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
-            // testTwoNodeFirstNodeCleared does unsafe things, and testLatestVersionLoaded / testRecoveryDifferentNodeOrderStartup also fail
-            .put(TestZenDiscovery.USE_ZEN2.getKey(), false)
-            .build();
-    }
-
     public void testOneNodeRecoverFromGateway() throws Exception {
 
         internalCluster().startNode();
@@ -312,10 +306,15 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
 
         Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
 
+        client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(new String[]{firstNode})).get();
+
         internalCluster().fullRestart(new RestartCallback() {
             @Override
-            public Settings onNodeStopped(String nodeName) throws Exception {
-                return Settings.builder().put("gateway.recover_after_nodes", 2).build();
+            public Settings onNodeStopped(String nodeName) {
+                return Settings.builder()
+                    .put(RECOVER_AFTER_NODES_SETTING.getKey(), 2)
+                    .putList(INITIAL_MASTER_NODES_SETTING.getKey()) // disable bootstrapping
+                    .build();
             }
 
             @Override
@@ -332,6 +331,8 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
         for (int i = 0; i < 10; i++) {
             assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2);
         }
+
+        client().execute(ClearVotingConfigExclusionsAction.INSTANCE, new ClearVotingConfigExclusionsRequest()).get();
     }
 
     public void testLatestVersionLoaded() throws Exception {
@@ -523,37 +524,6 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
         }
     }
 
-    public void testRecoveryDifferentNodeOrderStartup() throws Exception {
-        // we need different data paths so we make sure we start the second node fresh
-
-        final Path pathNode1 = createTempDir();
-        final String node_1 =
-            internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), pathNode1).build());
-
-        client().prepareIndex("test", "type1", "1").setSource("field", "value").execute().actionGet();
-
-        final Path pathNode2 = createTempDir();
-        final String node_2 =
-            internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), pathNode2).build());
-
-        ensureGreen();
-        Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
-
-        if (randomBoolean()) {
-            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_1));
-            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_2));
-        } else {
-            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_2));
-            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_1));
-        }
-        // start the second node again
-        internalCluster().startNode(Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), pathNode2).build());
-        ensureYellow();
-        primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
-        assertThat(client().admin().indices().prepareExists("test").execute().actionGet().isExists(), equalTo(true));
-        assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 1);
-    }
-
     public void testStartedShardFoundIfStateNotYetProcessed() throws Exception {
         // nodes may need to report the shards they processed the initial recovered cluster state from the master
         final String nodeName = internalCluster().startNode();
@@ -569,7 +539,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
             @Override
             public Settings onNodeStopped(String nodeName) throws Exception {
                 // make sure state is not recovered
-                return Settings.builder().put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 2).build();
+                return Settings.builder().put(RECOVER_AFTER_NODES_SETTING.getKey(), 2).build();
             }
         });