Add 7.1 and 8.0 version constants to master (#38514)

* Add 7.1 and 8.0 version constants to master

This commit adds the 7.1 and 8.0 version constants to the master branch,
and bumps the master branch to 8.0.0.

Fixing many BWC and other tests  affected by the version bumps.

buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java

@@ -171,30 +171,38 @@ public class VersionCollection {
     }
 
     public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
-        getUnreleased().stream()
+        List<UnreleasedVersionInfo> collect = getUnreleased().stream()
             .filter(version -> version.equals(currentVersion) == false)
-            .forEach(version -> consumer.accept(
-                new UnreleasedVersionInfo(
+            .map(version -> new UnreleasedVersionInfo(
                     version,
                     getBranchFor(version),
                     getGradleProjectNameFor(version)
                 )
-            ));
+            )
+            .collect(Collectors.toList());
+
+        collect.forEach(uvi -> consumer.accept(uvi));
     }
 
     private String getGradleProjectNameFor(Version version) {
         if (version.equals(currentVersion)) {
             throw new IllegalArgumentException("The Gradle project to build " + version + " is the current build.");
         }
+
         Map<Integer, List<Version>> releasedMajorGroupedByMinor = getReleasedMajorGroupedByMinor();
 
         if (version.getRevision() == 0) {
-            if (releasedMajorGroupedByMinor
-                .get(releasedMajorGroupedByMinor.keySet().stream().max(Integer::compareTo).orElse(0))
-                .contains(version)) {
-                return "minor";
+            List<Version> unreleasedStagedOrMinor = getUnreleased().stream()
+                .filter(v -> v.getRevision() == 0)
+                .collect(Collectors.toList());
+            if (unreleasedStagedOrMinor.size() > 2) {
+                if (unreleasedStagedOrMinor.get(unreleasedStagedOrMinor.size() - 2).equals(version)) {
+                    return "minor";
+                } else{
+                    return "staged";
+                }
             } else {
-                return "staged";
+                return "minor";
             }
         } else {
             if (releasedMajorGroupedByMinor
@@ -239,8 +247,10 @@ public class VersionCollection {
             unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 1));
             if (groupByMinor.getOrDefault(greatestMinor - 1, emptyList()).size() == 1) {
                 // we found that the previous minor is staged but not yet released
-                // in this case, the minor before that has a bugfix
-                unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2));
+                // in this case, the minor before that has a bugfix, should there be such a minor
+                if (greatestMinor >= 2) {
+                    unreleased.add(getLatestVersionByKey(groupByMinor, greatestMinor - 2));
+                }
             }
         }
 

buildSrc/src/test/java/org/elasticsearch/gradle/VersionCollectionTests.java

@@ -81,6 +81,9 @@ public class VersionCollectionTests extends GradleUnitTestCase {
             "6_0_0", "6_0_1", "6_1_0", "6_1_1", "6_1_2", "6_1_3", "6_1_4", "6_2_0", "6_2_1", "6_2_2", "6_2_3",
             "6_2_4", "6_3_0", "6_3_1", "6_3_2", "6_4_0", "6_4_1", "6_4_2"
         ));
+        sampleVersions.put("7.1.0", asList(
+            "7_1_0", "7_0_0", "6_7_0", "6_6_1", "6_6_0"
+        ));
     }
 
     @Test(expected = IllegalArgumentException.class)
@@ -145,6 +148,11 @@ public class VersionCollectionTests extends GradleUnitTestCase {
             singletonList("7.3.0"),
             getVersionCollection("8.0.0").getWireCompatible()
         );
+        assertVersionsEquals(
+            asList("6.7.0", "7.0.0"),
+            getVersionCollection("7.1.0").getWireCompatible()
+        );
+
     }
 
     public void testWireCompatibleUnreleased() {
@@ -171,6 +179,10 @@ public class VersionCollectionTests extends GradleUnitTestCase {
             singletonList("7.3.0"),
             getVersionCollection("8.0.0").getUnreleasedWireCompatible()
         );
+        assertVersionsEquals(
+            asList("6.7.0", "7.0.0"),
+            getVersionCollection("7.1.0").getWireCompatible()
+        );
     }
 
     public void testIndexCompatible() {
@@ -286,7 +298,7 @@ public class VersionCollectionTests extends GradleUnitTestCase {
             getVersionCollection("6.4.2")
         );
         assertUnreleasedBranchNames(
-            asList("5.6", "6.4", "6.5"),
+            asList("5.6", "6.4", "6.x"),
             getVersionCollection("6.6.0")
         );
         assertUnreleasedBranchNames(
@@ -309,13 +321,17 @@ public class VersionCollectionTests extends GradleUnitTestCase {
             getVersionCollection("6.4.2")
         );
         assertUnreleasedGradleProjectNames(
-            asList("maintenance", "bugfix", "staged"),
+            asList("maintenance", "bugfix", "minor"),
             getVersionCollection("6.6.0")
         );
         assertUnreleasedGradleProjectNames(
             asList("bugfix", "staged", "minor"),
             getVersionCollection("8.0.0")
         );
+        assertUnreleasedGradleProjectNames(
+            asList("staged", "minor"),
+            getVersionCollection("7.1.0")
+        );
     }
 
     public void testCompareToAuthoritative() {

buildSrc/version.properties

@@ -1,4 +1,4 @@
-elasticsearch     = 7.0.0
+elasticsearch     = 8.0.0
 lucene            = 8.0.0-snapshot-83f9835
 
 # optional dependencies

distribution/bwc/build.gradle

@@ -141,6 +141,12 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
             extension += '.gz'
           }
         }
+        if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) {
+          classifier = "-amd64"
+        }
+        if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) {
+          classifier = "-x86_64"
+        }
         if (bwcVersion.onOrAfter('6.3.0')) {
             baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages'
             // add oss variant first

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java

@@ -137,12 +137,21 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     public void testRemoveOldVersion() throws Exception {
+        Version previous = VersionUtils.getPreviousVersion();
+        if (previous.before(Version.CURRENT.minimumIndexCompatibilityVersion()) ) {
+            // Can happen when bumping majors: 8.0 is only compat back to 7.0, but that's not released yet
+            // In this case, ignore what's released and just find that latest version before current
+            previous = VersionUtils.allVersions().stream()
+                .filter(v -> v.before(Version.CURRENT))
+                .max(Version::compareTo)
+                .get();
+        }
         createPlugin(
                 "fake",
                 VersionUtils.randomVersionBetween(
                         random(),
                         Version.CURRENT.minimumIndexCompatibilityVersion(),
-                        VersionUtils.getPreviousVersion()));
+                        previous));
         removePlugin("fake", home, randomBoolean());
         assertThat(Files.exists(env.pluginsFile().resolve("fake")), equalTo(false));
         assertRemoveCleaned(env);

docs/Versions.asciidoc

@@ -1,5 +1,5 @@
-:version:               7.0.0-alpha2
-:major-version:         7.x
+:version:               8.0.0-alpha1
+:major-version:         8.x
 :lucene_version:        8.0.0
 :lucene_version_path:   8_0_0
 :branch:                master

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.rest.action.document.RestBulkAction;
 import org.elasticsearch.rest.action.document.RestGetAction;
+import org.elasticsearch.rest.action.document.RestIndexAction;
 import org.elasticsearch.rest.action.document.RestUpdateAction;
 import org.elasticsearch.rest.action.search.RestExplainAction;
 import org.elasticsearch.test.NotEqualMessageBuilder;
@@ -80,15 +81,20 @@ import static org.hamcrest.Matchers.startsWith;
  */
 public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
     private final boolean supportsLenientBooleans = getOldClusterVersion().before(Version.V_6_0_0_alpha1);
-    private static final Version VERSION_5_1_0_UNRELEASED = Version.fromString("5.1.0");
 
     private String index;
+    private String type;
 
     @Before
     public void setIndex() throws IOException {
         index = getTestName().toLowerCase(Locale.ROOT);
     }
 
+    @Before
+    public void setType() {
+        type = getOldClusterVersion().before(Version.V_6_7_0) ? "doc" : "_doc";
+    }
+
     public void testSearch() throws Exception {
         int count;
         if (isRunningAgainstOldCluster()) {
@@ -102,7 +108,9 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             }
             {
                 mappingsAndSettings.startObject("mappings");
-                mappingsAndSettings.startObject("doc");
+                if (isRunningAgainstAncientCluster()) {
+                    mappingsAndSettings.startObject(type);
+                }
                 mappingsAndSettings.startObject("properties");
                 {
                     mappingsAndSettings.startObject("string");
@@ -121,7 +129,9 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
                     mappingsAndSettings.endObject();
                 }
                 mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
+                if (isRunningAgainstAncientCluster()) {
+                    mappingsAndSettings.endObject();
+                }
                 mappingsAndSettings.endObject();
             }
             mappingsAndSettings.endObject();
@@ -135,17 +145,20 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             count = randomIntBetween(2000, 3000);
             byte[] randomByteArray = new byte[16];
             random().nextBytes(randomByteArray);
-            indexRandomDocuments(count, true, true, i -> {
-                return JsonXContent.contentBuilder().startObject()
-                .field("string", randomAlphaOfLength(10))
-                .field("int", randomInt(100))
-                .field("float", randomFloat())
-                // be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct
-                .field("bool", i > 0 && supportsLenientBooleans ? randomLenientBoolean() : randomBoolean())
-                .field("field.with.dots", randomAlphaOfLength(10))
-                .field("binary", Base64.getEncoder().encodeToString(randomByteArray))
-                .endObject();
-            });
+            indexRandomDocuments(
+                    count,
+                    true,
+                    true,
+                    i -> JsonXContent.contentBuilder().startObject()
+                            .field("string", randomAlphaOfLength(10))
+                            .field("int", randomInt(100))
+                            .field("float", randomFloat())
+                            // be sure to create a "proper" boolean (True, False) for the first document so that automapping is correct
+                            .field("bool", i > 0 && supportsLenientBooleans ? randomLenientBoolean() : randomBoolean())
+                            .field("field.with.dots", randomAlphaOfLength(10))
+                            .field("binary", Base64.getEncoder().encodeToString(randomByteArray))
+                            .endObject()
+            );
             refresh();
         } else {
             count = countOfIndexedRandomDocuments();
@@ -155,7 +168,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         assertBasicSearchWorks(count);
         assertAllSearchWorks(count);
         assertBasicAggregationWorks();
-        assertRealtimeGetWorks();
+        assertRealtimeGetWorks(type);
         assertStoredBinaryFields(count);
     }
 
@@ -171,7 +184,9 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             }
             {
                 mappingsAndSettings.startObject("mappings");
-                mappingsAndSettings.startObject("doc");
+                if (isRunningAgainstAncientCluster()) {
+                    mappingsAndSettings.startObject(type);
+                }
                 mappingsAndSettings.startObject("properties");
                 {
                     mappingsAndSettings.startObject("field");
@@ -179,7 +194,9 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
                     mappingsAndSettings.endObject();
                 }
                 mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
+                if (isRunningAgainstAncientCluster()) {
+                    mappingsAndSettings.endObject();
+                }
                 mappingsAndSettings.endObject();
             }
             mappingsAndSettings.endObject();
@@ -191,11 +208,8 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             client().performRequest(createIndex);
 
             int numDocs = randomIntBetween(2000, 3000);
-            indexRandomDocuments(numDocs, true, false, i -> {
-                return JsonXContent.contentBuilder().startObject()
-                    .field("field", "value")
-                    .endObject();
-            });
+            indexRandomDocuments(
+                    numDocs, true, false, i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject());
             logger.info("Refreshing [{}]", index);
             client().performRequest(new Request("POST", "/" + index + "/_refresh"));
         } else {
@@ -225,76 +239,6 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         }
     }
 
-    /**
-     * Search on an alias that contains illegal characters that would prevent it from being created after 5.1.0. It should still be
-     * search-able though.
-     */
-    public void testAliasWithBadName() throws Exception {
-        assumeTrue("Can only test bad alias name if old cluster is on 5.1.0 or before",
-            getOldClusterVersion().before(VERSION_5_1_0_UNRELEASED));
-
-        int count;
-        if (isRunningAgainstOldCluster()) {
-            XContentBuilder mappingsAndSettings = jsonBuilder();
-            mappingsAndSettings.startObject();
-            {
-                mappingsAndSettings.startObject("settings");
-                mappingsAndSettings.field("number_of_shards", 1);
-                mappingsAndSettings.field("number_of_replicas", 0);
-                mappingsAndSettings.endObject();
-            }
-            {
-                mappingsAndSettings.startObject("mappings");
-                mappingsAndSettings.startObject("doc");
-                mappingsAndSettings.startObject("properties");
-                {
-                    mappingsAndSettings.startObject("key");
-                    mappingsAndSettings.field("type", "keyword");
-                    mappingsAndSettings.endObject();
-                }
-                mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
-            }
-            mappingsAndSettings.endObject();
-            Request createIndex = new Request("PUT", "/" + index);
-            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
-            client().performRequest(createIndex);
-
-            String aliasName = "%23" + index; // %23 == #
-            client().performRequest(new Request("PUT", "/" + index + "/_alias/" + aliasName));
-            Response response = client().performRequest(new Request("HEAD", "/" + index + "/_alias/" + aliasName));
-            assertEquals(200, response.getStatusLine().getStatusCode());
-
-            count = randomIntBetween(32, 128);
-            indexRandomDocuments(count, true, true, i -> {
-                return JsonXContent.contentBuilder().startObject()
-                    .field("key", "value")
-                    .endObject();
-            });
-            refresh();
-        } else {
-            count = countOfIndexedRandomDocuments();
-        }
-
-        Request request = new Request("GET", "/_cluster/state");
-        request.addParameter("metric", "metadata");
-        logger.error("clusterState=" + entityAsMap(client().performRequest(request)));
-        // We can read from the alias just like we can read from the index.
-        String aliasName = "%23" + index; // %23 == #
-        Map<String, Object> searchRsp = entityAsMap(client().performRequest(new Request("GET", "/" + aliasName + "/_search")));
-        int totalHits = extractTotalHits(searchRsp);
-        assertEquals(count, totalHits);
-        if (isRunningAgainstOldCluster() == false) {
-            // We can remove the alias.
-            Response response = client().performRequest(new Request("DELETE", "/" + index + "/_alias/" + aliasName));
-            assertEquals(200, response.getStatusLine().getStatusCode());
-            // and check that it is gone:
-            response = client().performRequest(new Request("HEAD", "/" + index + "/_alias/" + aliasName));
-            assertEquals(404, response.getStatusLine().getStatusCode());
-        }
-    }
-
     public void testClusterState() throws Exception {
         if (isRunningAgainstOldCluster()) {
             XContentBuilder mappingsAndSettings = jsonBuilder();
@@ -348,31 +292,45 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             mappingsAndSettings.startObject();
             {
                 mappingsAndSettings.startObject("mappings");
-                mappingsAndSettings.startObject("doc");
-                mappingsAndSettings.startObject("properties");
                 {
-                    mappingsAndSettings.startObject("field");
-                    mappingsAndSettings.field("type", "text");
+                    if (isRunningAgainstAncientCluster()) {
+                        mappingsAndSettings.startObject(type);
+                    }
+                    mappingsAndSettings.startObject("properties");
+                    {
+                        mappingsAndSettings.startObject("field");
+                        {
+                            mappingsAndSettings.field("type", "text");
+                        }
+                        mappingsAndSettings.endObject();
+                    }
                     mappingsAndSettings.endObject();
+                    if (isRunningAgainstAncientCluster()) {
+                        mappingsAndSettings.endObject();
+                    }
                 }
                 mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
+                if (isRunningAgainstAncientCluster() == false) {
+                    // the default number of shards is now one so we have to set the number of shards to be more than one explicitly
+                    mappingsAndSettings.startObject("settings");
+                    {
+                        mappingsAndSettings.field("index.number_of_shards", 5);
+                    }
+                    mappingsAndSettings.endObject();
+                }
             }
             mappingsAndSettings.endObject();
             Request createIndex = new Request("PUT", "/" + index);
             createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
             RequestOptions.Builder options = createIndex.getOptions().toBuilder();
             options.setWarningsHandler(WarningsHandler.PERMISSIVE);
+            expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE);
             createIndex.setOptions(options);
             client().performRequest(createIndex);
 
             numDocs = randomIntBetween(512, 1024);
-            indexRandomDocuments(numDocs, true, true, i -> {
-                return JsonXContent.contentBuilder().startObject()
-                    .field("field", "value")
-                    .endObject();
-            });
+            indexRandomDocuments(
+                    numDocs, true, true, i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject());
 
             ensureGreen(index); // wait for source index to be available on both nodes before starting shrink
 
@@ -381,9 +339,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             client().performRequest(updateSettingsRequest);
 
             Request shrinkIndexRequest = new Request("PUT", "/" + index + "/_shrink/" + shrunkenIndex);
-            if (getOldClusterVersion().onOrAfter(Version.V_6_4_0)) {
-                shrinkIndexRequest.addParameter("copy_settings", "true");
-            }
+
             shrinkIndexRequest.setJsonEntity("{\"settings\": {\"index.number_of_shards\": 1}}");
             client().performRequest(shrinkIndexRequest);
 
@@ -419,16 +375,30 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             mappingsAndSettings.startObject();
             {
                 mappingsAndSettings.startObject("mappings");
-                mappingsAndSettings.startObject("doc");
-                mappingsAndSettings.startObject("properties");
                 {
-                    mappingsAndSettings.startObject("field");
-                    mappingsAndSettings.field("type", "text");
+                    if (isRunningAgainstAncientCluster()) {
+                        mappingsAndSettings.startObject(type);
+                    }
+                    mappingsAndSettings.startObject("properties");
+                    {
+                        mappingsAndSettings.startObject("field");
+                        {
+                            mappingsAndSettings.field("type", "text");
+                        }
+                        mappingsAndSettings.endObject();
+                    }
                     mappingsAndSettings.endObject();
+                    if (isRunningAgainstAncientCluster()) {
+                        mappingsAndSettings.endObject();
+                    }
                 }
                 mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
+                if (isRunningAgainstAncientCluster() == false) {
+                    // the default number of shards is now one so we have to set the number of shards to be more than one explicitly
+                    mappingsAndSettings.startObject("settings");
+                    mappingsAndSettings.field("index.number_of_shards", 5);
+                    mappingsAndSettings.endObject();
+                }
             }
             mappingsAndSettings.endObject();
             Request createIndex = new Request("PUT", "/" + index);
@@ -439,11 +409,12 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             client().performRequest(createIndex);
 
             numDocs = randomIntBetween(512, 1024);
-            indexRandomDocuments(numDocs, true, true, i -> {
-                return JsonXContent.contentBuilder().startObject()
-                    .field("field", "value")
-                    .endObject();
-            });
+            indexRandomDocuments(
+                    numDocs,
+                    true,
+                    true,
+                    i -> JsonXContent.contentBuilder().startObject().field("field", "value").endObject()
+            );
         } else {
             ensureGreen(index); // wait for source index to be available on both nodes before starting shrink
 
@@ -510,7 +481,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             bulk.append("{\"index\":{}}\n");
             bulk.append("{\"test\":\"test\"}\n");
         }
-        Request bulkRequest = new Request("POST", "/" + index + "_write/doc/_bulk");
+        Request bulkRequest = new Request("POST", "/" + index + "_write/" + type + "/_bulk");
         bulkRequest.setJsonEntity(bulk.toString());
         bulkRequest.addParameter("refresh", "");
         bulkRequest.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE));
@@ -633,7 +604,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         assertTotalHits(termsCount, boolTerms);
     }
 
-    void assertRealtimeGetWorks() throws IOException {
+    void assertRealtimeGetWorks(final String typeName) throws IOException {
         Request disableAutoRefresh = new Request("PUT", "/" + index + "/_settings");
         disableAutoRefresh.setJsonEntity("{ \"index\": { \"refresh_interval\" : -1 }}");
         client().performRequest(disableAutoRefresh);
@@ -644,13 +615,15 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         Map<?, ?> hit = (Map<?, ?>) ((List<?>)(XContentMapValues.extractValue("hits.hits", searchResponse))).get(0);
         String docId = (String) hit.get("_id");
 
-        Request updateRequest = new Request("POST", "/" + index + "/doc/" + docId + "/_update");
+        Request updateRequest = new Request("POST", "/" + index + "/" + typeName + "/" + docId + "/_update");
         updateRequest.setOptions(expectWarnings(RestUpdateAction.TYPES_DEPRECATION_MESSAGE));
         updateRequest.setJsonEntity("{ \"doc\" : { \"foo\": \"bar\"}}");
         client().performRequest(updateRequest);
 
-        Request getRequest = new Request("GET", "/" + index + "/doc/" + docId);
-        getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        Request getRequest = new Request("GET", "/" + index + "/" + typeName + "/" + docId);
+        if (getOldClusterVersion().before(Version.V_6_7_0)) {
+            getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        }
         Map<String, Object> getRsp = entityAsMap(client().performRequest(getRequest));
         Map<?, ?> source = (Map<?, ?>) getRsp.get("_source");
         assertTrue("doc does not contain 'foo' key: " + source, source.containsKey("foo"));
@@ -689,7 +662,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
 
     void assertTotalHits(int expectedTotalHits, Map<?, ?> response) {
         int actualTotalHits = extractTotalHits(response);
-        assertEquals(expectedTotalHits, actualTotalHits);
+        assertEquals(response.toString(), expectedTotalHits, actualTotalHits);
     }
 
     int extractTotalHits(Map<?, ?> response) {
@@ -704,7 +677,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
      * Tests that a single document survives. Super basic smoke test.
      */
     public void testSingleDoc() throws IOException {
-        String docLocation = "/" + index + "/doc/1";
+        String docLocation = "/" + index + "/" + type + "/1";
         String doc = "{\"test\": \"test\"}";
 
         if (isRunningAgainstOldCluster()) {
@@ -715,7 +688,9 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
 
 
         Request request = new Request("GET", docLocation);
-        request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        if (getOldClusterVersion().before(Version.V_6_7_0)) {
+            request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        }
         assertThat(toStr(client().performRequest(request)), containsString(doc));
     }
 
@@ -779,8 +754,12 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             }
             if (shouldHaveTranslog) {
                 // Update a few documents so we are sure to have a translog
-                indexRandomDocuments(count / 10, false /* Flushing here would invalidate the whole thing....*/, false,
-                    i -> jsonBuilder().startObject().field("field", "value").endObject());
+                indexRandomDocuments(
+                        count / 10,
+                        false, // flushing here would invalidate the whole thing
+                        false,
+                        i -> jsonBuilder().startObject().field("field", "value").endObject()
+                );
             }
             saveInfoDocument("should_have_translog", Boolean.toString(shouldHaveTranslog));
         } else {
@@ -791,6 +770,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         // Count the documents in the index to make sure we have as many as we put there
         Request countRequest = new Request("GET", "/" + index + "/_search");
         countRequest.addParameter("size", "0");
+        refresh();
         Map<String, Object> countResponse = entityAsMap(client().performRequest(countRequest));
         assertTotalHits(count, countResponse);
 
@@ -863,7 +843,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
      */
     public void testSnapshotRestore() throws IOException {
         int count;
-        if (isRunningAgainstOldCluster()) {
+        if (isRunningAgainstOldCluster() && getOldClusterVersion().major < 8) {
             // Create the index
             count = between(200, 300);
             indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject());
@@ -894,13 +874,19 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         }
         templateBuilder.endObject();
         templateBuilder.startObject("mappings"); {
-            templateBuilder.startObject("doc"); {
-                templateBuilder.startObject("_source"); {
+            if (isRunningAgainstAncientCluster()) {
+                templateBuilder.startObject(type);
+            }
+            {
+                templateBuilder.startObject("_source");
+                {
                     templateBuilder.field("enabled", true);
                 }
                 templateBuilder.endObject();
             }
-            templateBuilder.endObject();
+            if (isRunningAgainstAncientCluster()) {
+                templateBuilder.endObject();
+            }
         }
         templateBuilder.endObject();
         templateBuilder.startObject("aliases"); {
@@ -921,8 +907,8 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         createTemplateRequest.setJsonEntity(Strings.toString(templateBuilder));
 
         // In 7.0, type names are no longer expected by default in put index template requests.
-        // We therefore use the deprecated typed APIs when running against the current version.
-        if (isRunningAgainstOldCluster() == false) {
+        // We therefore use the deprecated typed APIs when running against the current version, but testing with a pre-7 version
+        if (isRunningAgainstOldCluster() == false && getOldClusterVersion().major < 7) {
             createTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true");
         }
         createTemplateRequest.setOptions(allowTypeRemovalWarnings());
@@ -1016,12 +1002,13 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             int numDocs = between(10, 100);
             for (int i = 0; i < numDocs; i++) {
                 String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject());
-                Request request = new Request("POST", "/" + index + "/doc/" + i);
+                Request request = new Request("POST", "/" + index + "/" + type + "/" + i);
+                if (isRunningAgainstAncientCluster() == false) {
+                    request.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
+                }
                 request.setJsonEntity(doc);
                 client().performRequest(request);
-                if (rarely()) {
-                    refresh();
-                }
+                refresh();
             }
             client().performRequest(new Request("POST", "/" + index + "/_flush"));
             int liveDocs = numDocs;
@@ -1029,11 +1016,11 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             for (int i = 0; i < numDocs; i++) {
                 if (randomBoolean()) {
                     String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v2").endObject());
-                    Request request = new Request("POST", "/" + index + "/doc/" + i);
+                    Request request = new Request("POST", "/" + index + "/" + type + "/" + i);
                     request.setJsonEntity(doc);
                     client().performRequest(request);
                 } else if (randomBoolean()) {
-                    client().performRequest(new Request("DELETE", "/" + index + "/doc/" + i));
+                    client().performRequest(new Request("DELETE", "/" + index + "/" + type + "/" + i));
                     liveDocs--;
                 }
             }
@@ -1046,7 +1033,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         }
     }
 
-    private void checkSnapshot(String snapshotName, int count, Version tookOnVersion) throws IOException {
+    private void checkSnapshot(final String snapshotName, final int count, final Version tookOnVersion) throws IOException {
         // Check the snapshot metadata, especially the version
         Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName);
         Map<String, Object> listSnapshotResponse = entityAsMap(client().performRequest(listSnapshotRequest));
@@ -1103,7 +1090,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             bulk.append("{\"index\":{\"_id\":\"").append(count + i).append("\"}}\n");
             bulk.append("{\"test\":\"test\"}\n");
         }
-        Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/doc/_bulk");
+        Request writeToRestoredRequest = new Request("POST", "/restored_" + index + "/" + type + "/_bulk");
         writeToRestoredRequest.addParameter("refresh", "true");
         writeToRestoredRequest.setJsonEntity(bulk.toString());
         writeToRestoredRequest.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE));
@@ -1132,7 +1119,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
 
         // In 7.0, type names are no longer returned by default in get index template requests.
         // We therefore use the deprecated typed APIs when running against the current version.
-        if (isRunningAgainstOldCluster() == false) {
+        if (isRunningAgainstAncientCluster() == false) {
             getTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true");
         }
         getTemplateRequest.setOptions(allowTypeRemovalWarnings());
@@ -1145,7 +1132,14 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             expectedTemplate.put("index_patterns", singletonList("evil_*"));
         }
         expectedTemplate.put("settings", singletonMap("index", singletonMap("number_of_shards", "1")));
-        expectedTemplate.put("mappings", singletonMap("doc", singletonMap("_source", singletonMap("enabled", true))));
+        // We don't have the type in the response starting with 7.0, but we won't have it on old cluster after upgrade
+        // either so look at the response to figure out the correct assertions
+        if (isTypeInTemplateResponse(getTemplateResponse)) {
+            expectedTemplate.put("mappings", singletonMap(type, singletonMap("_source", singletonMap("enabled", true))));
+        } else {
+            expectedTemplate.put("mappings", singletonMap("_source", singletonMap("enabled", true)));
+        }
+
         expectedTemplate.put("order", 0);
         Map<String, Object> aliases = new HashMap<>();
         aliases.put("alias1", emptyMap());
@@ -1155,18 +1149,33 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         if (false == expectedTemplate.equals(getTemplateResponse)) {
             NotEqualMessageBuilder builder = new NotEqualMessageBuilder();
             builder.compareMaps(getTemplateResponse, expectedTemplate);
+            logger.info("expected: {}\nactual:{}", expectedTemplate, getTemplateResponse);
             fail("template doesn't match:\n" + builder.toString());
         }
     }
 
+    @SuppressWarnings("unchecked")
+    private boolean isTypeInTemplateResponse(Map<String, Object> getTemplateResponse) {
+        return ( (Map<String, Object>) (
+            (Map<String, Object>) getTemplateResponse.getOrDefault("test_template", emptyMap())
+        ).get("mappings")).get("_source") == null;
+    }
+
     // TODO tests for upgrades after shrink. We've had trouble with shrink in the past.
 
-    private void indexRandomDocuments(int count, boolean flushAllowed, boolean saveInfo,
-                                      CheckedFunction<Integer, XContentBuilder, IOException> docSupplier) throws IOException {
+    private void indexRandomDocuments(
+            final int count,
+            final boolean flushAllowed,
+            final boolean saveInfo,
+            final CheckedFunction<Integer, XContentBuilder, IOException> docSupplier)
+            throws IOException {
         logger.info("Indexing {} random documents", count);
         for (int i = 0; i < count; i++) {
             logger.debug("Indexing document [{}]", i);
-            Request createDocument = new Request("POST", "/" + index + "/doc/" + i);
+            Request createDocument = new Request("POST", "/" + index + "/" + type + "/" + i);
+            if (isRunningAgainstAncientCluster() == false) {
+                createDocument.setOptions(expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE));
+            }
             createDocument.setJsonEntity(Strings.toString(docSupplier.apply(i)));
             client().performRequest(createDocument);
             if (rarely()) {
@@ -1191,16 +1200,21 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         infoDoc.field("value", value);
         infoDoc.endObject();
         // Only create the first version so we know how many documents are created when the index is first created
-        Request request = new Request("PUT", "/info/doc/" + index + "_" + type);
+        Request request = new Request("PUT", "/info/" + this.type + "/" + index + "_" + type);
         request.addParameter("op_type", "create");
         request.setJsonEntity(Strings.toString(infoDoc));
+        if (isRunningAgainstAncientCluster() == false) {
+            request.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
+        }
         client().performRequest(request);
     }
 
     private String loadInfoDocument(String type) throws IOException {
-        Request request = new Request("GET", "/info/doc/" + index + "_" + type);
+        Request request = new Request("GET", "/info/" + this.type + "/" + index + "_" + type);
         request.addParameter("filter_path", "_source");
-        request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        if (isRunningAgainstAncientCluster()) {
+            request.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        }
         String doc = toStr(client().performRequest(request));
         Matcher m = Pattern.compile("\"value\":\"(.+)\"").matcher(doc);
         assertTrue(doc, m.find());

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.upgrades;
 
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
@@ -145,6 +146,7 @@ public class QueryBuilderBWCIT extends AbstractFullClusterRestartTestCase {
     }
 
     public void testQueryBuilderBWC() throws Exception {
+        final String type = getOldClusterVersion().before(Version.V_7_0_0) ? "doc" : "_doc";
         String index = "queries";
         if (isRunningAgainstOldCluster()) {
             XContentBuilder mappingsAndSettings = jsonBuilder();
@@ -157,7 +159,9 @@ public class QueryBuilderBWCIT extends AbstractFullClusterRestartTestCase {
             }
             {
                 mappingsAndSettings.startObject("mappings");
-                mappingsAndSettings.startObject("doc");
+                if (isRunningAgainstAncientCluster()) {
+                    mappingsAndSettings.startObject(type);
+                }
                 mappingsAndSettings.startObject("properties");
                 {
                     mappingsAndSettings.startObject("query");
@@ -176,7 +180,9 @@ public class QueryBuilderBWCIT extends AbstractFullClusterRestartTestCase {
                 }
                 mappingsAndSettings.endObject();
                 mappingsAndSettings.endObject();
-                mappingsAndSettings.endObject();
+                if (isRunningAgainstAncientCluster()) {
+                    mappingsAndSettings.endObject();
+                }
             }
             mappingsAndSettings.endObject();
             Request request = new Request("PUT", "/" + index);
@@ -188,7 +194,7 @@ public class QueryBuilderBWCIT extends AbstractFullClusterRestartTestCase {
             assertEquals(200, rsp.getStatusLine().getStatusCode());
 
             for (int i = 0; i < CANDIDATES.size(); i++) {
-                request = new Request("PUT", "/" + index + "/doc/" + Integer.toString(i));
+                request = new Request("PUT", "/" + index + "/" + type + "/" + Integer.toString(i));
                 request.setJsonEntity((String) CANDIDATES.get(i)[0]);
                 rsp = client().performRequest(request);
                 assertEquals(201, rsp.getStatusLine().getStatusCode());

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java

@@ -19,20 +19,15 @@
 package org.elasticsearch.upgrades;
 
 import org.apache.http.util.EntityUtils;
-import org.elasticsearch.common.Booleans;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.rest.action.document.RestBulkAction;
-import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.common.Booleans;
+import org.elasticsearch.rest.action.document.RestBulkAction;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
-import static org.hamcrest.Matchers.equalTo;
 
 /**
  * Basic test that indexed documents survive the rolling restart. See
@@ -68,26 +63,6 @@ public class IndexingIT extends AbstractRollingTestCase {
         }
 
         if (CLUSTER_TYPE == ClusterType.OLD) {
-            {
-                Version minimumIndexCompatibilityVersion = Version.CURRENT.minimumIndexCompatibilityVersion();
-                assertThat("this branch is not needed if we aren't compatible with 6.0",
-                        minimumIndexCompatibilityVersion.onOrBefore(Version.V_6_0_0), equalTo(true));
-                if (minimumIndexCompatibilityVersion.before(Version.V_7_0_0)) {
-                    XContentBuilder template = jsonBuilder();
-                    template.startObject();
-                    {
-                        template.field("index_patterns", "*");
-                        template.startObject("settings");
-                        template.field("number_of_shards", 5);
-                        template.endObject();
-                    }
-                    template.endObject();
-                    Request createTemplate = new Request("PUT", "/_template/template");
-                    createTemplate.setJsonEntity(Strings.toString(template));
-                    client().performRequest(createTemplate);
-                }
-            }
-
             Request createTestIndex = new Request("PUT", "/test_index");
             createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}");
             client().performRequest(createTestIndex);

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -300,7 +300,11 @@ public class RecoveryIT extends AbstractRollingTestCase {
                 if (randomBoolean()) {
                     indexDocs(index, i, 1); // update
                 } else if (randomBoolean()) {
-                    client().performRequest(new Request("DELETE", index + "/test/" + i));
+                    if (getNodeId(v -> v.onOrAfter(Version.V_7_0_0)) == null) {
+                        client().performRequest(new Request("DELETE", index + "/test/" + i));
+                    } else {
+                        client().performRequest(new Request("DELETE", index + "/_doc/" + i));
+                    }
                 }
             }
         }

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml

@@ -67,8 +67,3 @@
   - match: { hits.total: 1 }
   - match: { hits.hits.0._id: q3 }
 
----
-"Index with _all is available":
-  - do:
-      indices.get:
-        index: all-index

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml

@@ -200,19 +200,3 @@
         wait_for_completion: true
         task_id: $task
 
----
-"Create an index with _all explicitly disabled":
-  - skip:
-      features: warnings
-  - do:
-      warnings:
-        - "[_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field."
-      indices.create:
-        index: all-index
-        body:
-          mappings:
-            _all:
-              enabled: false
-            properties:
-              field:
-                type: text

qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml

@@ -125,17 +125,4 @@
         task_id: $task_id
   - match: { task.headers.X-Opaque-Id: "Reindexing Again" }
 
----
-"Index with _all is available":
-  - do:
-      indices.get:
-        index: all-index
-
-  - do:
-      indices.get_mapping:
-        include_type_name: false
-        index: all-index
-
-  - is_true: all-index.mappings._all
-  - match: { all-index.mappings._all.enabled: false}
 

qa/verify-version-constants/src/test/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java

@@ -37,7 +37,7 @@ public class VerifyVersionConstantsIT extends ESRestTestCase {
         assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
         final ObjectPath objectPath = ObjectPath.createFromResponse(response);
         final String elasticsearchVersionString = objectPath.evaluate("version.number").toString();
-        final Version elasticsearchVersion = Version.fromString(elasticsearchVersionString);
+        final Version elasticsearchVersion = Version.fromString(elasticsearchVersionString.replace("-SNAPSHOT", ""));
         final String luceneVersionString = objectPath.evaluate("version.lucene_version").toString();
         final org.apache.lucene.util.Version luceneVersion = org.apache.lucene.util.Version.parse(luceneVersionString);
         assertThat(elasticsearchVersion.luceneVersion, equalTo(luceneVersion));

rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json

@@ -18,10 +18,6 @@
         }
       },
       "params": {
-        "copy_settings": {
-          "type" : "boolean",
-          "description" : "whether or not to copy settings from the source index (defaults to false)"
-        },
         "timeout": {
           "type" : "time",
           "description" : "Explicit operation timeout"

rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json

@@ -18,10 +18,6 @@
         }
       },
       "params": {
-        "copy_settings": {
-          "type" : "boolean",
-          "description" : "whether or not to copy settings from the source index (defaults to false)"
-        },
         "timeout": {
           "type" : "time",
           "description" : "Explicit operation timeout"

rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/30_copy_settings.yml

@@ -43,13 +43,10 @@
         target: "copy-settings-target"
         wait_for_active_shards: 1
         master_timeout: 10s
-        copy_settings: true
         body:
           settings:
             index.number_of_replicas: 0
             index.merge.scheduler.max_thread_count: 2
-      warnings:
-        - "parameter [copy_settings] is deprecated and will be removed in 8.0.0"
 
   - do:
       cluster.health:
@@ -64,42 +61,3 @@
   - match: { copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" }
   - match: { copy-settings-target.settings.index.blocks.write: "true" }
   - match: { copy-settings-target.settings.index.routing.allocation.include._id: $master }
-
-  # now we do a actual shrink and copy settings (by default)
-  - do:
-      indices.shrink:
-        index: "source"
-        target: "default-copy-settings-target"
-        wait_for_active_shards: 1
-        master_timeout: 10s
-        body:
-          settings:
-            index.number_of_replicas: 0
-            index.merge.scheduler.max_thread_count: 2
-
-  - do:
-      cluster.health:
-        wait_for_status: green
-
-  - do:
-      indices.get_settings:
-        index: "default-copy-settings-target"
-
-  # settings should be copied
-  - match: { default-copy-settings-target.settings.index.merge.scheduler.max_merge_count: "4" }
-  - match: { default-copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" }
-  - match: { default-copy-settings-target.settings.index.blocks.write: "true" }
-  - match: { default-copy-settings-target.settings.index.routing.allocation.include._id: $master }
-
-  # now we do a actual shrink and try to set no copy settings
-  - do:
-      catch: /illegal_argument_exception/
-      indices.shrink:
-        index: "source"
-        target: "explicit-no-copy-settings-target"
-        wait_for_active_shards: 1
-        master_timeout: 10s
-        copy_settings: false
-        body:
-          settings:
-            index.number_of_replicas: 0

rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/30_copy_settings.yml

@@ -44,15 +44,11 @@
         target: "copy-settings-target"
         wait_for_active_shards: 1
         master_timeout: 10s
-        copy_settings: true
         body:
           settings:
             index.number_of_replicas: 0
             index.number_of_shards: 2
             index.merge.scheduler.max_thread_count: 2
-      warnings:
-        - "parameter [copy_settings] is deprecated and will be removed in 8.0.0"
-
 
   - do:
       cluster.health:
@@ -67,42 +63,3 @@
   - match: { copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" }
   - match: { copy-settings-target.settings.index.blocks.write: "true" }
   - match: { copy-settings-target.settings.index.routing.allocation.include._id: $master }
-
-  # now we do a actual shrink and copy settings (by default)
-  - do:
-      indices.split:
-        index: "source"
-        target: "default-copy-settings-target"
-        wait_for_active_shards: 1
-        master_timeout: 10s
-        body:
-          settings:
-            index.number_of_replicas: 0
-            index.number_of_shards: 2
-            index.merge.scheduler.max_thread_count: 2
-
-  - do:
-      cluster.health:
-        wait_for_status: green
-
-  - do:
-      indices.get_settings:
-        index: "default-copy-settings-target"
-
-  # settings should be copied
-  - match: { default-copy-settings-target.settings.index.merge.scheduler.max_merge_count: "4" }
-  - match: { default-copy-settings-target.settings.index.merge.scheduler.max_thread_count: "2" }
-  - match: { default-copy-settings-target.settings.index.blocks.write: "true" }
-  - match: { default-copy-settings-target.settings.index.routing.allocation.include._id: $master }
-
-  - do:
-      catch: /illegal_argument_exception/
-      indices.split:
-        index: "source"
-        target: "explicit-no-copy-settings-target"
-        wait_for_active_shards: 1
-        master_timeout: 10s
-        copy_settings: false
-        body:
-          settings:
-            index.number_of_replicas: 0

server/src/main/java/org/elasticsearch/Version.java

@@ -124,7 +124,11 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_6_7_0 = new Version(V_6_7_0_ID, org.apache.lucene.util.Version.LUCENE_7_7_0);
     public static final int V_7_0_0_ID = 7000099;
     public static final Version V_7_0_0 = new Version(V_7_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0);
-    public static final Version CURRENT = V_7_0_0;
+    public static final int V_7_1_0_ID = 7010099;
+    public static final Version V_7_1_0 = new Version(V_7_1_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final int V_8_0_0_ID = 8000099;
+    public static final Version V_8_0_0 = new Version(V_8_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0);
+    public static final Version CURRENT = V_8_0_0;
 
 
     static {
@@ -138,6 +142,10 @@ public class Version implements Comparable<Version>, ToXContentFragment {
 
     public static Version fromId(int id) {
         switch (id) {
+            case V_8_0_0_ID:
+                return V_8_0_0;
+            case V_7_1_0_ID:
+                return V_7_1_0;
             case V_7_0_0_ID:
                 return V_7_0_0;
             case V_6_7_0_ID:

server/src/main/java/org/elasticsearch/action/resync/ResyncReplicationRequest.java

@@ -66,7 +66,8 @@ public final class ResyncReplicationRequest extends ReplicatedWriteRequest<Resyn
 
     @Override
     public void readFrom(final StreamInput in) throws IOException {
-        assert Version.CURRENT.major <= 7;
+        //  TODO: https://github.com/elastic/elasticsearch/issues/38556
+        //assert Version.CURRENT.major <= 7;
         if (in.getVersion().equals(Version.V_6_0_0)) {
             /*
              * Resync replication request serialization was broken in 6.0.0 due to the elements of the stream not being prefixed with a

server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java

@@ -127,7 +127,6 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery
     private long maxTermSeen;
     private final Reconfigurator reconfigurator;
     private final ClusterBootstrapService clusterBootstrapService;
-    private final DiscoveryUpgradeService discoveryUpgradeService;
     private final LagDetector lagDetector;
     private final ClusterFormationFailureHelper clusterFormationFailureHelper;
 
@@ -169,8 +168,6 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery
         this.reconfigurator = new Reconfigurator(settings, clusterSettings);
         this.clusterBootstrapService = new ClusterBootstrapService(settings, transportService, this::getFoundPeers,
             this::isInitialConfigurationSet, this::setInitialConfiguration);
-        this.discoveryUpgradeService = new DiscoveryUpgradeService(settings, transportService,
-            this::isInitialConfigurationSet, joinHelper, peerFinder::getFoundPeers, this::setInitialConfiguration);
         this.lagDetector = new LagDetector(settings, transportService.getThreadPool(), n -> removeNode(n, "lagging"),
             transportService::getLocalNode);
         this.clusterFormationFailureHelper = new ClusterFormationFailureHelper(settings, this::getClusterFormationState,
@@ -508,10 +505,6 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery
             peerFinder.activate(coordinationState.get().getLastAcceptedState().nodes());
             clusterFormationFailureHelper.start();
 
-            if (getCurrentTerm() == ZEN1_BWC_TERM) {
-                discoveryUpgradeService.activate(lastKnownLeader, coordinationState.get().getLastAcceptedState());
-            }
-
             leaderChecker.setCurrentNodes(DiscoveryNodes.EMPTY_NODES);
             leaderChecker.updateLeader(null);
 
@@ -543,7 +536,6 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery
 
         lastKnownLeader = Optional.of(getLocalNode());
         peerFinder.deactivate(getLocalNode());
-        discoveryUpgradeService.deactivate();
         clusterFormationFailureHelper.stop();
         closePrevotingAndElectionScheduler();
         preVoteCollector.update(getPreVoteResponse(), getLocalNode());
@@ -575,7 +567,6 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery
 
         lastKnownLeader = Optional.of(leaderNode);
         peerFinder.deactivate(leaderNode);
-        discoveryUpgradeService.deactivate();
         clusterFormationFailureHelper.stop();
         closePrevotingAndElectionScheduler();
         cancelActivePublication("become follower: " + method);

server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java

@@ -1,347 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.cluster.coordination;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.CountDown;
-import org.elasticsearch.discovery.zen.ElectMasterService;
-import org.elasticsearch.discovery.zen.ElectMasterService.MasterCandidate;
-import org.elasticsearch.discovery.zen.UnicastZenPing;
-import org.elasticsearch.discovery.zen.UnicastZenPing.UnicastPingRequest;
-import org.elasticsearch.discovery.zen.UnicastZenPing.UnicastPingResponse;
-import org.elasticsearch.discovery.zen.ZenPing.PingResponse;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPool.Names;
-import org.elasticsearch.transport.TransportException;
-import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportResponseHandler;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Optional;
-import java.util.Set;
-import java.util.function.BooleanSupplier;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-
-import static java.lang.Math.max;
-import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING;
-import static org.elasticsearch.cluster.ClusterState.UNKNOWN_VERSION;
-import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentSet;
-import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING;
-import static org.elasticsearch.discovery.zen.ZenDiscovery.PING_TIMEOUT_SETTING;
-
-/**
- * Deals with rolling upgrades of the cluster coordination layer. In mixed clusters we prefer to elect the older nodes, but
- * when the last old node shuts down then as long as there are enough new nodes we can assume that they form the whole cluster and
- * define them as the initial configuration.
- */
-public class DiscoveryUpgradeService {
-
-    private static Logger logger = LogManager.getLogger(DiscoveryUpgradeService.class);
-
-    // how long to wait after activation before attempting to join a master or perform a bootstrap upgrade
-    public static final Setting<TimeValue> BWC_PING_TIMEOUT_SETTING =
-        Setting.timeSetting("discovery.zen.bwc_ping_timeout",
-            PING_TIMEOUT_SETTING, TimeValue.timeValueMillis(1), Setting.Property.NodeScope, Setting.Property.Deprecated);
-
-    // whether to try and bootstrap all the discovered Zen2 nodes when the last Zen1 node leaves the cluster.
-    public static final Setting<Boolean> ENABLE_UNSAFE_BOOTSTRAPPING_ON_UPGRADE_SETTING =
-        Setting.boolSetting("discovery.zen.unsafe_rolling_upgrades_enabled", true, Setting.Property.NodeScope, Setting.Property.Deprecated);
-
-    /**
-     * Dummy {@link ElectMasterService} that is only used to choose the best 6.x master from the discovered nodes, ignoring the
-     * `minimum_master_nodes` setting.
-     */
-    private static final ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY);
-
-    private final TransportService transportService;
-    private final BooleanSupplier isBootstrappedSupplier;
-    private final JoinHelper joinHelper;
-    private final Supplier<Iterable<DiscoveryNode>> peersSupplier;
-    private final Consumer<VotingConfiguration> initialConfigurationConsumer;
-    private final TimeValue bwcPingTimeout;
-    private final boolean enableUnsafeBootstrappingOnUpgrade;
-    private final ClusterName clusterName;
-
-    @Nullable // null if no active joining round
-    private volatile JoiningRound joiningRound;
-
-    public DiscoveryUpgradeService(Settings settings, TransportService transportService,
-                                   BooleanSupplier isBootstrappedSupplier, JoinHelper joinHelper,
-                                   Supplier<Iterable<DiscoveryNode>> peersSupplier,
-                                   Consumer<VotingConfiguration> initialConfigurationConsumer) {
-        assert Version.CURRENT.major == Version.V_6_6_0.major + 1 : "remove this service once unsafe upgrades are no longer needed";
-        this.transportService = transportService;
-        this.isBootstrappedSupplier = isBootstrappedSupplier;
-        this.joinHelper = joinHelper;
-        this.peersSupplier = peersSupplier;
-        this.initialConfigurationConsumer = initialConfigurationConsumer;
-        this.bwcPingTimeout = BWC_PING_TIMEOUT_SETTING.get(settings);
-        this.enableUnsafeBootstrappingOnUpgrade = ENABLE_UNSAFE_BOOTSTRAPPING_ON_UPGRADE_SETTING.get(settings);
-        this.clusterName = CLUSTER_NAME_SETTING.get(settings);
-    }
-
-    public void activate(Optional<DiscoveryNode> lastKnownLeader, ClusterState lastAcceptedClusterState) {
-        // called under coordinator mutex
-
-        if (isBootstrappedSupplier.getAsBoolean()) {
-            return;
-        }
-
-        assert lastKnownLeader.isPresent() == false || Coordinator.isZen1Node(lastKnownLeader.get()) : lastKnownLeader;
-        // if there was a leader and it's not a old node then we must have been bootstrapped
-
-        final Settings dynamicSettings = lastAcceptedClusterState.metaData().settings();
-        final int minimumMasterNodes = DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(dynamicSettings)
-            ? DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(dynamicSettings)
-            : lastAcceptedClusterState.getMinimumMasterNodesOnPublishingMaster();
-
-        assert joiningRound == null : joiningRound;
-        final Set<String> knownMasterNodeIds = new HashSet<>();
-        lastAcceptedClusterState.nodes().getMasterNodes().forEach(c -> knownMasterNodeIds.add(c.key));
-
-        joiningRound
-            = new JoiningRound(enableUnsafeBootstrappingOnUpgrade && lastKnownLeader.isPresent(), minimumMasterNodes, knownMasterNodeIds);
-        joiningRound.scheduleNextAttempt();
-    }
-
-    public void deactivate() {
-        // called under coordinator mutex
-        joiningRound = null;
-    }
-
-    /**
-     * Waits for some number of calls to {@link ListenableCountDown#countDown()} and then notifies a listener. The listener
-     * is only ever notified once, whether successful or not.
-     */
-    private static class ListenableCountDown {
-        private final CountDown countDown;
-        private final ActionListener<Void> listener;
-
-        ListenableCountDown(int count, ActionListener<Void> listener) {
-            this.countDown = new CountDown(count);
-            this.listener = listener;
-        }
-
-        void onFailure(Exception e) {
-            if (countDown.fastForward()) {
-                listener.onFailure(e);
-            }
-        }
-
-        void countDown() {
-            if (countDown.countDown()) {
-                listener.onResponse(null);
-            }
-        }
-    }
-
-    private class JoiningRound {
-        private final boolean upgrading;
-        private final int minimumMasterNodes;
-        private final Set<String> knownMasterNodeIds;
-
-        JoiningRound(boolean upgrading, int minimumMasterNodes, Set<String> knownMasterNodeIds) {
-            this.upgrading = upgrading;
-            this.minimumMasterNodes = minimumMasterNodes;
-            this.knownMasterNodeIds = knownMasterNodeIds;
-        }
-
-        private boolean isRunning() {
-            return joiningRound == this && isBootstrappedSupplier.getAsBoolean() == false;
-        }
-
-        private boolean canBootstrap(Set<DiscoveryNode> discoveryNodes) {
-            return upgrading && minimumMasterNodes <= discoveryNodes.stream().filter(DiscoveryNode::isMasterNode).count();
-        }
-
-        void scheduleNextAttempt() {
-            if (isRunning() == false) {
-                return;
-            }
-
-            final ThreadPool threadPool = transportService.getThreadPool();
-            threadPool.scheduleUnlessShuttingDown(bwcPingTimeout, Names.SAME, new Runnable() {
-
-                @Override
-                public void run() {
-                    if (isRunning() == false) {
-                        return;
-                    }
-
-                    final Set<DiscoveryNode> discoveryNodes = Stream.concat(StreamSupport.stream(peersSupplier.get().spliterator(), false),
-                        Stream.of(transportService.getLocalNode())).filter(DiscoveryNode::isMasterNode).collect(Collectors.toSet());
-
-                    // this set of nodes is reasonably fresh - the PeerFinder cleans up nodes to which the transport service is not
-                    // connected each time it wakes up (every second by default)
-
-                    logger.debug("upgrading={}, minimumMasterNodes={}, nodes={}", upgrading, minimumMasterNodes, discoveryNodes);
-
-                    if (discoveryNodes.stream().anyMatch(Coordinator::isZen1Node)) {
-                        electBestOldMaster(discoveryNodes);
-                    } else if (canBootstrap(discoveryNodes)) {
-                        // no Zen1 nodes found, but the last-known master was a Zen1 node, so this is a rolling upgrade
-                        transportService.getThreadPool().generic().execute(() -> {
-                            try {
-                                Set<String> nodeIds = new HashSet<>();
-                                discoveryNodes.forEach(n -> nodeIds.add(n.getId()));
-
-                                final Iterator<String> knownNodeIdIterator = knownMasterNodeIds.iterator();
-                                while (nodeIds.size() < 2 * minimumMasterNodes - 1 && knownNodeIdIterator.hasNext()) {
-                                    nodeIds.add(knownNodeIdIterator.next());
-                                }
-
-                                final VotingConfiguration votingConfiguration = new VotingConfiguration(nodeIds);
-                                assert votingConfiguration.hasQuorum(
-                                    discoveryNodes.stream().map(DiscoveryNode::getId).collect(Collectors.toList()));
-                                assert 2 * minimumMasterNodes - 2 <= nodeIds.size() : nodeIds + " too small for " + minimumMasterNodes;
-
-                                initialConfigurationConsumer.accept(votingConfiguration);
-                            } catch (Exception e) {
-                                logger.debug("exception during bootstrapping upgrade, retrying", e);
-                            } finally {
-                                scheduleNextAttempt();
-                            }
-                        });
-                    } else {
-                        scheduleNextAttempt();
-                    }
-                }
-
-                /**
-                 * Ping all the old discovered masters one more time to obtain their cluster state versions, and then vote for the best one.
-                 * @param discoveryNodes The master nodes (old and new).
-                 */
-                private void electBestOldMaster(Set<DiscoveryNode> discoveryNodes) {
-                    final Set<MasterCandidate> masterCandidates = newConcurrentSet();
-                    final ListenableCountDown listenableCountDown
-                        = new ListenableCountDown(discoveryNodes.size(), new ActionListener<Void>() {
-
-                        @Override
-                        public void onResponse(Void value) {
-                            assert masterCandidates.size() == discoveryNodes.size()
-                                : masterCandidates + " does not match " + discoveryNodes;
-
-                            // TODO we shouldn't elect a master with a version that's older than ours
-                            // If the only Zen1 nodes left are stale, and we can bootstrap, maybe we should bootstrap?
-                            // Do we ever need to elect a freshly-started Zen1 node?
-                            if (isRunning()) {
-                                final MasterCandidate electedMaster = electMasterService.electMaster(masterCandidates);
-                                logger.debug("elected {}, sending join", electedMaster);
-                                joinHelper.sendJoinRequest(electedMaster.getNode(), Optional.empty(),
-                                    JoiningRound.this::scheduleNextAttempt);
-                            }
-                        }
-
-                        @Override
-                        public void onFailure(Exception e) {
-                            scheduleNextAttempt();
-                        }
-                    });
-
-                    boolean foundOldMaster = false;
-                    for (final DiscoveryNode discoveryNode : discoveryNodes) {
-                        assert discoveryNode.isMasterNode() : discoveryNode;
-                        if (Coordinator.isZen1Node(discoveryNode)) {
-                            foundOldMaster = true;
-                            transportService.sendRequest(discoveryNode, UnicastZenPing.ACTION_NAME,
-                                new UnicastPingRequest(0, TimeValue.ZERO,
-                                    new PingResponse(createDiscoveryNodeWithImpossiblyHighId(transportService.getLocalNode()),
-                                        null, clusterName, UNKNOWN_VERSION)),
-                                TransportRequestOptions.builder().withTimeout(bwcPingTimeout).build(),
-                                new TransportResponseHandler<UnicastPingResponse>() {
-                                    @Override
-                                    public void handleResponse(UnicastPingResponse response) {
-                                        long clusterStateVersion = UNKNOWN_VERSION;
-                                        for (PingResponse pingResponse : response.pingResponses) {
-                                            if (discoveryNode.equals(pingResponse.node())) {
-                                                clusterStateVersion
-                                                    = max(clusterStateVersion, pingResponse.getClusterStateVersion());
-                                            }
-                                        }
-                                        masterCandidates.add(new MasterCandidate(discoveryNode, clusterStateVersion));
-                                        listenableCountDown.countDown();
-                                    }
-
-                                    @Override
-                                    public void handleException(TransportException exp) {
-                                        logger.debug(
-                                            new ParameterizedMessage("unexpected exception when pinging {}", discoveryNode), exp);
-                                        listenableCountDown.onFailure(exp);
-                                    }
-
-                                    @Override
-                                    public String executor() {
-                                        return Names.SAME;
-                                    }
-
-                                    @Override
-                                    public UnicastPingResponse read(StreamInput in) throws IOException {
-                                        return new UnicastPingResponse(in);
-                                    }
-                                });
-
-                        } else {
-                            masterCandidates.add(
-                                new MasterCandidate(createDiscoveryNodeWithImpossiblyHighId(discoveryNode), UNKNOWN_VERSION));
-                            listenableCountDown.countDown();
-                        }
-                    }
-                    assert foundOldMaster;
-                }
-
-                @Override
-                public String toString() {
-                    return "discovery upgrade service retry";
-                }
-            });
-        }
-    }
-
-    /**
-     * Pre-7.0 nodes select the best master by comparing their IDs (as strings) and selecting the lowest one amongst those nodes with
-     * the best cluster state version. We want 7.0+ nodes to participate in these elections in a mixed cluster but never to win one, so
-     * we lie and claim to have an impossible ID that compares above all genuine IDs.
-     */
-    public static DiscoveryNode createDiscoveryNodeWithImpossiblyHighId(DiscoveryNode node) {
-        // IDs are base-64-encoded UUIDs, which means they use the character set [0-9A-Za-z_-]. The highest character in this set is 'z',
-        // and 'z' < '{', so by starting the ID with '{' we can be sure it's greater. This is terrible.
-        return new DiscoveryNode(node.getName(), "{zen2}" + node.getId(), node.getEphemeralId(), node.getHostName(),
-            node.getHostAddress(), node.getAddress(), node.getAttributes(), node.getRoles(), node.getVersion());
-    }
-}

server/src/main/java/org/elasticsearch/cluster/coordination/PublicationTransportHandler.java

@@ -93,31 +93,9 @@ public class PublicationTransportHandler {
         transportService.registerRequestHandler(PUBLISH_STATE_ACTION_NAME, BytesTransportRequest::new, ThreadPool.Names.GENERIC,
             false, false, (request, channel, task) -> channel.sendResponse(handleIncomingPublishRequest(request)));
 
-        transportService.registerRequestHandler(PublishClusterStateAction.SEND_ACTION_NAME, BytesTransportRequest::new,
-            ThreadPool.Names.GENERIC,
-            false, false, (request, channel, task) -> {
-                handleIncomingPublishRequest(request);
-                channel.sendResponse(TransportResponse.Empty.INSTANCE);
-            });
-
         transportService.registerRequestHandler(COMMIT_STATE_ACTION_NAME, ThreadPool.Names.GENERIC, false, false,
             ApplyCommitRequest::new,
             (request, channel, task) -> handleApplyCommit.accept(request, transportCommitCallback(channel)));
-
-        transportService.registerRequestHandler(PublishClusterStateAction.COMMIT_ACTION_NAME,
-            PublishClusterStateAction.CommitClusterStateRequest::new,
-            ThreadPool.Names.GENERIC, false, false,
-            (request, channel, task) -> {
-                final Optional<ClusterState> matchingClusterState = Optional.ofNullable(lastSeenClusterState.get()).filter(
-                    cs -> cs.stateUUID().equals(request.stateUUID));
-                if (matchingClusterState.isPresent() == false) {
-                    throw new IllegalStateException("can't resolve cluster state with uuid" +
-                        " [" + request.stateUUID + "] to commit");
-                }
-                final ApplyCommitRequest applyCommitRequest = new ApplyCommitRequest(matchingClusterState.get().getNodes().getMasterNode(),
-                    matchingClusterState.get().term(), matchingClusterState.get().version());
-                handleApplyCommit.accept(applyCommitRequest, transportCommitCallback(channel));
-            });
     }
 
     private ActionListener<Void> transportCommitCallback(TransportChannel channel) {

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -590,7 +590,8 @@ public class MetaDataCreateIndexService {
 
         static int getNumberOfShards(final Settings.Builder indexSettingsBuilder) {
             // TODO: this logic can be removed when the current major version is 8
-            assert Version.CURRENT.major == 7;
+            // TODO: https://github.com/elastic/elasticsearch/issues/38556
+            // assert Version.CURRENT.major == 7;
             final int numberOfShards;
             final Version indexVersionCreated =
                     Version.fromId(Integer.parseInt(indexSettingsBuilder.get(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey())));

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -35,7 +35,6 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.coordination.ClusterBootstrapService;
 import org.elasticsearch.cluster.coordination.ClusterFormationFailureHelper;
 import org.elasticsearch.cluster.coordination.Coordinator;
-import org.elasticsearch.cluster.coordination.DiscoveryUpgradeService;
 import org.elasticsearch.cluster.coordination.ElectionSchedulerFactory;
 import org.elasticsearch.cluster.coordination.FollowersChecker;
 import org.elasticsearch.cluster.coordination.JoinHelper;
@@ -67,9 +66,9 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.PeerFinder;
+import org.elasticsearch.discovery.SettingsBasedSeedHostsProvider;
 import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.discovery.zen.FaultDetection;
-import org.elasticsearch.discovery.SettingsBasedSeedHostsProvider;
 import org.elasticsearch.discovery.zen.UnicastZenPing;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.env.Environment;
@@ -493,9 +492,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     TransportAddVotingConfigExclusionsAction.MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING,
                     ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING,
                     ClusterBootstrapService.UNCONFIGURED_BOOTSTRAP_TIMEOUT_SETTING,
-                    LagDetector.CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING,
-                    DiscoveryUpgradeService.BWC_PING_TIMEOUT_SETTING,
-                    DiscoveryUpgradeService.ENABLE_UNSAFE_BOOTSTRAPPING_ON_UPGRADE_SETTING
+                    LagDetector.CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING
             )));
 
     public static List<SettingUpgrader<?>> BUILT_IN_SETTING_UPGRADERS = Collections.unmodifiableList(Arrays.asList(

server/src/main/java/org/elasticsearch/discovery/PeerFinder.java

@@ -25,8 +25,6 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.coordination.PeersResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -37,24 +35,13 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.discovery.zen.UnicastZenPing;
-import org.elasticsearch.discovery.zen.ZenDiscovery;
-import org.elasticsearch.discovery.zen.ZenPing;
-import org.elasticsearch.discovery.zen.ZenPing.PingResponse;
-import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool.Names;
-import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -64,8 +51,6 @@ import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyList;
-import static org.elasticsearch.cluster.coordination.Coordinator.isZen1Node;
-import static org.elasticsearch.cluster.coordination.DiscoveryUpgradeService.createDiscoveryNodeWithImpossiblyHighId;
 
 public abstract class PeerFinder {
 
@@ -82,8 +67,6 @@ public abstract class PeerFinder {
         Setting.timeSetting("discovery.request_peers_timeout",
             TimeValue.timeValueMillis(3000), TimeValue.timeValueMillis(1), Setting.Property.NodeScope);
 
-    private final Settings settings;
-
     private final TimeValue findPeersInterval;
     private final TimeValue requestPeersTimeout;
 
@@ -101,7 +84,6 @@ public abstract class PeerFinder {
 
     public PeerFinder(Settings settings, TransportService transportService, TransportAddressConnector transportAddressConnector,
                       ConfiguredHostsResolver configuredHostsResolver) {
-        this.settings = settings;
         findPeersInterval = DISCOVERY_FIND_PEERS_INTERVAL_SETTING.get(settings);
         requestPeersTimeout = DISCOVERY_REQUEST_PEERS_TIMEOUT_SETTING.get(settings);
         this.transportService = transportService;
@@ -111,9 +93,6 @@ public abstract class PeerFinder {
         transportService.registerRequestHandler(REQUEST_PEERS_ACTION_NAME, Names.GENERIC, false, false,
             PeersRequest::new,
             (request, channel, task) -> channel.sendResponse(handlePeersRequest(request)));
-
-        transportService.registerRequestHandler(UnicastZenPing.ACTION_NAME, Names.GENERIC, false, false,
-            UnicastZenPing.UnicastPingRequest::new, new Zen1UnicastPingRequestHandler());
     }
 
     public void activate(final DiscoveryNodes lastAcceptedNodes) {
@@ -453,36 +432,10 @@ public abstract class PeerFinder {
                     return Names.GENERIC;
                 }
             };
-            final String actionName;
-            final TransportRequest transportRequest;
-            final TransportResponseHandler<?> transportResponseHandler;
-            if (isZen1Node(discoveryNode)) {
-                actionName = UnicastZenPing.ACTION_NAME;
-                transportRequest = new UnicastZenPing.UnicastPingRequest(1, ZenDiscovery.PING_TIMEOUT_SETTING.get(settings),
-                    new ZenPing.PingResponse(createDiscoveryNodeWithImpossiblyHighId(getLocalNode()), null,
-                        ClusterName.CLUSTER_NAME_SETTING.get(settings), ClusterState.UNKNOWN_VERSION));
-                transportResponseHandler = peersResponseHandler.wrap(ucResponse -> {
-                    Optional<DiscoveryNode> optionalMasterNode = Arrays.stream(ucResponse.pingResponses)
-                        .filter(pr -> discoveryNode.equals(pr.node()) && discoveryNode.equals(pr.master()))
-                        .map(ZenPing.PingResponse::node)
-                        .findFirst();
-                    List<DiscoveryNode> discoveredNodes = new ArrayList<>();
-                    if (optionalMasterNode.isPresent() == false) {
-                        Arrays.stream(ucResponse.pingResponses).map(PingResponse::master).filter(Objects::nonNull)
-                            .forEach(discoveredNodes::add);
-                        Arrays.stream(ucResponse.pingResponses).map(PingResponse::node).forEach(discoveredNodes::add);
-                    }
-                    return new PeersResponse(optionalMasterNode, discoveredNodes, 0L);
-                }, UnicastZenPing.UnicastPingResponse::new);
-            } else {
-                actionName = REQUEST_PEERS_ACTION_NAME;
-                transportRequest = new PeersRequest(getLocalNode(), knownNodes);
-                transportResponseHandler = peersResponseHandler;
-            }
-            transportService.sendRequest(discoveryNode, actionName,
-                transportRequest,
+            transportService.sendRequest(discoveryNode, REQUEST_PEERS_ACTION_NAME,
+                new PeersRequest(getLocalNode(), knownNodes),
                 TransportRequestOptions.builder().withTimeout(requestPeersTimeout).build(),
-                transportResponseHandler);
+                peersResponseHandler);
         }
 
         @Override
@@ -494,22 +447,4 @@ public abstract class PeerFinder {
                 '}';
         }
     }
-
-    private class Zen1UnicastPingRequestHandler implements TransportRequestHandler<UnicastZenPing.UnicastPingRequest> {
-        @Override
-        public void messageReceived(UnicastZenPing.UnicastPingRequest request, TransportChannel channel, Task task) throws Exception {
-            final PeersRequest peersRequest = new PeersRequest(request.pingResponse.node(),
-                Optional.ofNullable(request.pingResponse.master()).map(Collections::singletonList).orElse(emptyList()));
-            final PeersResponse peersResponse = handlePeersRequest(peersRequest);
-            final List<ZenPing.PingResponse> pingResponses = new ArrayList<>();
-            final ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
-            pingResponses.add(new ZenPing.PingResponse(createDiscoveryNodeWithImpossiblyHighId(transportService.getLocalNode()),
-                peersResponse.getMasterNode().orElse(null),
-                clusterName, ClusterState.UNKNOWN_VERSION));
-            peersResponse.getKnownPeers().forEach(dn -> pingResponses.add(
-                new ZenPing.PingResponse(ZenPing.PingResponse.FAKE_PING_ID,
-                    isZen1Node(dn) ? dn : createDiscoveryNodeWithImpossiblyHighId(dn), null, clusterName, ClusterState.UNKNOWN_VERSION)));
-            channel.sendResponse(new UnicastZenPing.UnicastPingResponse(request.id, pingResponses.toArray(new ZenPing.PingResponse[0])));
-        }
-    }
 }

server/src/main/java/org/elasticsearch/gateway/MetaStateService.java

@@ -21,7 +21,6 @@ package org.elasticsearch.gateway;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -118,7 +117,8 @@ public class MetaStateService {
 
         if (globalMetaData != null) {
             metaDataBuilder = MetaData.builder(globalMetaData);
-            assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0";
+            // TODO https://github.com/elastic/elasticsearch/issues/38556
+            // assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0";
         } else {
             metaDataBuilder = MetaData.builder();
         }
@@ -127,7 +127,8 @@ public class MetaStateService {
             Tuple<IndexMetaData, Long> indexMetaDataAndGeneration =
                     INDEX_META_DATA_FORMAT.loadLatestStateWithGeneration(logger, namedXContentRegistry,
                             nodeEnv.resolveIndexFolder(indexFolderName));
-            assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0";
+            // TODO https://github.com/elastic/elasticsearch/issues/38556
+            // assert Version.CURRENT.major < 8 : "failed to find manifest file, which is mandatory staring with Elasticsearch version 8.0";
             IndexMetaData indexMetaData = indexMetaDataAndGeneration.v1();
             long generation = indexMetaDataAndGeneration.v2();
             if (indexMetaData != null) {
@@ -291,4 +292,4 @@ public class MetaStateService {
         writeManifestAndCleanup(reason, manifest);
         cleanupGlobalState(generation);
     }
-}
+}

server/src/main/java/org/elasticsearch/node/InternalSettingsPreparer.java

@@ -19,6 +19,11 @@
 
 package org.elasticsearch.node;
 
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.env.Environment;
+
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -26,21 +31,11 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Supplier;
 import java.util.function.Function;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.Node;
+import java.util.function.Supplier;
 
 public class InternalSettingsPreparer {
 
-    private static final String SECRET_PROMPT_VALUE = "${prompt.secret}";
-    private static final String TEXT_PROMPT_VALUE = "${prompt.text}";
-
     /**
      * Prepares settings for the transport client by gathering all
      * elasticsearch system properties and setting defaults.
@@ -88,7 +83,6 @@ public class InternalSettingsPreparer {
 
         // re-initialize settings now that the config file has been loaded
         initializeSettings(output, input, properties);
-        checkSettingsForTerminalDeprecation(output);
         finalizeSettings(output, defaultNodeName);
 
         environment = new Environment(output.build(), configPath);
@@ -112,25 +106,6 @@ public class InternalSettingsPreparer {
         output.replacePropertyPlaceholders();
     }
 
-    /**
-     * Checks all settings values to make sure they do not have the old prompt settings. These were deprecated in 6.0.0.
-     * This check should be removed in 8.0.0.
-     */
-    private static void checkSettingsForTerminalDeprecation(final Settings.Builder output) throws SettingsException {
-        // This method to be removed in 8.0.0, as it was deprecated in 6.0 and removed in 7.0
-        assert Version.CURRENT.major != 8: "Logic pertaining to config driven prompting should be removed";
-        for (String setting : output.keys()) {
-            switch (output.get(setting)) {
-                case SECRET_PROMPT_VALUE:
-                    throw new SettingsException("Config driven secret prompting was deprecated in 6.0.0. Use the keystore" +
-                        " for secure settings.");
-                case TEXT_PROMPT_VALUE:
-                    throw new SettingsException("Config driven text prompting was deprecated in 6.0.0. Use the keystore" +
-                        " for secure settings.");
-            }
-        }
-    }
-
     /**
      * Finish preparing settings by replacing forced settings and any defaults that need to be added.
      */

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java

@@ -19,15 +19,10 @@
 
 package org.elasticsearch.rest.action.admin.indices;
 
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.Booleans;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
@@ -37,8 +32,6 @@ import org.elasticsearch.rest.action.RestToXContentListener;
 import java.io.IOException;
 
 public abstract class RestResizeHandler extends BaseRestHandler {
-    private static final Logger logger = LogManager.getLogger(RestResizeHandler.class);
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
     RestResizeHandler(final Settings settings) {
         super(settings);
@@ -53,24 +46,6 @@ public abstract class RestResizeHandler extends BaseRestHandler {
     public final RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         final ResizeRequest resizeRequest = new ResizeRequest(request.param("target"), request.param("index"));
         resizeRequest.setResizeType(getResizeType());
-        // copy_settings should be removed in Elasticsearch 8.0.0; cf. https://github.com/elastic/elasticsearch/issues/28347
-        assert Version.CURRENT.major < 8;
-        final String rawCopySettings = request.param("copy_settings");
-        final Boolean copySettings;
-        if (rawCopySettings == null) {
-            copySettings = resizeRequest.getCopySettings();
-        } else {
-            if (rawCopySettings.isEmpty()) {
-                copySettings = true;
-            } else {
-                copySettings = Booleans.parseBoolean(rawCopySettings);
-                if (copySettings == false) {
-                    throw new IllegalArgumentException("parameter [copy_settings] can not be explicitly set to [false]");
-                }
-            }
-            deprecationLogger.deprecated("parameter [copy_settings] is deprecated and will be removed in 8.0.0");
-        }
-        resizeRequest.setCopySettings(copySettings);
         request.applyContentParser(resizeRequest::fromXContent);
         resizeRequest.timeout(request.paramAsTime("timeout", resizeRequest.timeout()));
         resizeRequest.masterNodeTimeout(request.paramAsTime("master_timeout", resizeRequest.masterNodeTimeout()));

server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java

@@ -57,7 +57,8 @@ public abstract class RemoteClusterAware {
 
     static {
         // remove search.remote.* settings in 8.0.0
-        assert Version.CURRENT.major < 8;
+        // TODO https://github.com/elastic/elasticsearch/issues/38556
+        // assert Version.CURRENT.major < 8;
     }
 
     public static final Setting.AffixSetting<List<String>> SEARCH_REMOTE_CLUSTERS_SEEDS =

server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java

@@ -69,7 +69,8 @@ public final class RemoteClusterService extends RemoteClusterAware implements Cl
 
     static {
         // remove search.remote.* settings in 8.0.0
-        assert Version.CURRENT.major < 8;
+        // TODO
+        // assert Version.CURRENT.major < 8;
     }
 
     public static final Setting<Integer> SEARCH_REMOTE_CONNECTIONS_PER_CLUSTER =

server/src/test/java/org/elasticsearch/VersionTests.java

@@ -367,10 +367,9 @@ public class VersionTests extends ESTestCase {
                                 () -> new IllegalStateException("expected previous minor version for [" + currentOrNextMajorVersion + "]"));
         final Version previousMinorVersion = VersionUtils.getPreviousMinorVersion();
 
-        assert previousMinorVersion.major == currentOrNextMajorVersion.major
-                || previousMinorVersion.major == lastMinorFromPreviousMajor.major;
-        boolean isCompatible = previousMinorVersion.major == currentOrNextMajorVersion.major
-                || previousMinorVersion.minor == lastMinorFromPreviousMajor.minor;
+        boolean isCompatible =
+                previousMinorVersion.major == currentOrNextMajorVersion.major
+                        || previousMinorVersion.minor == lastMinorFromPreviousMajor.minor;
 
         final String message = String.format(
                 Locale.ROOT,

server/src/test/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeServiceTests.java

@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.cluster.coordination;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.lessThan;
-
-public class DiscoveryUpgradeServiceTests extends ESTestCase {
-    public void testCreateDiscoveryNodeWithImpossiblyHighId() {
-        final DiscoveryNode discoveryNode
-            = new DiscoveryNode(UUIDs.randomBase64UUID(random()), buildNewFakeTransportAddress(), Version.CURRENT);
-        final DiscoveryNode fakeNode = DiscoveryUpgradeService.createDiscoveryNodeWithImpossiblyHighId(discoveryNode);
-        assertThat(discoveryNode.getId(), lessThan(fakeNode.getId()));
-        assertThat(UUIDs.randomBase64UUID(random()), lessThan(fakeNode.getId()));
-        assertThat(fakeNode.getId(), containsString(discoveryNode.getId()));
-    }
-}

server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java

@@ -1,373 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.cluster.coordination;
-
-import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
-import org.elasticsearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
-import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction;
-import org.elasticsearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.Manifest;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.Allocation;
-import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.Discovery;
-import org.elasticsearch.discovery.DiscoveryModule;
-import org.elasticsearch.discovery.zen.ElectMasterService;
-import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.gateway.MetaStateService;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalTestCluster.RestartCallback;
-import org.elasticsearch.test.transport.MockTransportService;
-import org.elasticsearch.transport.TransportService;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.StreamSupport;
-
-import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
-import static org.elasticsearch.cluster.coordination.Coordinator.ZEN1_BWC_TERM;
-import static org.elasticsearch.cluster.coordination.FollowersChecker.FOLLOWER_CHECK_ACTION_NAME;
-import static org.elasticsearch.cluster.coordination.JoinHelper.START_JOIN_ACTION_NAME;
-import static org.elasticsearch.cluster.coordination.PublicationTransportHandler.PUBLISH_STATE_ACTION_NAME;
-import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING;
-import static org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING;
-import static org.elasticsearch.node.Node.NODE_NAME_SETTING;
-import static org.elasticsearch.test.InternalTestCluster.REMOVED_MINIMUM_MASTER_NODES;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
-public class Zen1IT extends ESIntegTestCase {
-
-    private static Settings ZEN1_SETTINGS = Coordinator.addZen1Attribute(true, Settings.builder()
-        .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.ZEN_DISCOVERY_TYPE)).build();
-
-    private static Settings ZEN2_SETTINGS = Settings.builder()
-        .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.ZEN2_DISCOVERY_TYPE)
-        .build();
-
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singletonList(MockTransportService.TestPlugin.class);
-    }
-
-    public void testZen2NodesJoiningZen1Cluster() {
-        internalCluster().startNodes(randomIntBetween(1, 3), ZEN1_SETTINGS);
-        internalCluster().startNodes(randomIntBetween(1, 3), ZEN2_SETTINGS);
-        createIndex("test");
-    }
-
-    public void testZen1NodesJoiningZen2Cluster() {
-        internalCluster().startNodes(randomIntBetween(1, 3), ZEN2_SETTINGS);
-        internalCluster().startNodes(randomIntBetween(1, 3), ZEN1_SETTINGS);
-        createIndex("test");
-    }
-
-    public void testMixedClusterDisruption() throws Exception {
-        final List<String> nodes = internalCluster().startNodes(IntStream.range(0, 5)
-            .mapToObj(i -> i < 2 ? ZEN1_SETTINGS : ZEN2_SETTINGS).toArray(Settings[]::new));
-
-        final List<MockTransportService> transportServices = nodes.stream()
-            .map(n -> (MockTransportService) internalCluster().getInstance(TransportService.class, n)).collect(Collectors.toList());
-
-        logger.info("--> disrupting communications");
-
-        // The idea here is to make some of the Zen2 nodes believe the Zen1 nodes have gone away by introducing a network partition, so that
-        // they bootstrap themselves, but keep the Zen1 side of the cluster alive.
-
-        // Set up a bridged network partition with the Zen1 nodes {0,1} on one side, Zen2 nodes {3,4} on the other, and node {2} in both
-        transportServices.get(0).addFailToSendNoConnectRule(transportServices.get(3));
-        transportServices.get(0).addFailToSendNoConnectRule(transportServices.get(4));
-        transportServices.get(1).addFailToSendNoConnectRule(transportServices.get(3));
-        transportServices.get(1).addFailToSendNoConnectRule(transportServices.get(4));
-        transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(0));
-        transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(1));
-        transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(0));
-        transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(1));
-
-        // Nodes 3 and 4 will bootstrap, but we want to keep node 2 as part of the Zen1 cluster, so prevent any messages that might switch
-        // its allegiance
-        transportServices.get(3).addFailToSendNoConnectRule(transportServices.get(2),
-            PUBLISH_STATE_ACTION_NAME, FOLLOWER_CHECK_ACTION_NAME, START_JOIN_ACTION_NAME);
-        transportServices.get(4).addFailToSendNoConnectRule(transportServices.get(2),
-            PUBLISH_STATE_ACTION_NAME, FOLLOWER_CHECK_ACTION_NAME, START_JOIN_ACTION_NAME);
-
-        logger.info("--> waiting for disconnected nodes to be removed");
-        ensureStableCluster(3, nodes.get(0));
-
-        logger.info("--> creating index on Zen1 side");
-        assertAcked(client(nodes.get(0)).admin().indices().create(new CreateIndexRequest("test")).get());
-        assertFalse(client(nodes.get(0)).admin().cluster().health(new ClusterHealthRequest("test")
-            .waitForGreenStatus()).get().isTimedOut());
-
-        logger.info("--> waiting for disconnected nodes to bootstrap themselves");
-        assertBusy(() -> assertTrue(IntStream.range(3, 5)
-            .mapToObj(n -> (Coordinator) internalCluster().getInstance(Discovery.class, nodes.get(n)))
-            .anyMatch(Coordinator::isInitialConfigurationSet)));
-
-        logger.info("--> clearing disruption and waiting for cluster to reform");
-        transportServices.forEach(MockTransportService::clearAllRules);
-
-        ensureStableCluster(5, nodes.get(0));
-        assertFalse(client(nodes.get(0)).admin().cluster().health(new ClusterHealthRequest("test")
-            .waitForGreenStatus()).get().isTimedOut());
-    }
-
-    public void testMixedClusterFormation() throws Exception {
-        final int zen1NodeCount = randomIntBetween(1, 3);
-        final int zen2NodeCount = randomIntBetween(zen1NodeCount == 1 ? 2 : 1, 3);
-        logger.info("starting cluster of [{}] Zen1 nodes and [{}] Zen2 nodes", zen1NodeCount, zen2NodeCount);
-        final List<String> nodes = internalCluster().startNodes(IntStream.range(0, zen1NodeCount + zen2NodeCount)
-            .mapToObj(i -> i < zen1NodeCount ? ZEN1_SETTINGS : ZEN2_SETTINGS).toArray(Settings[]::new));
-
-        createIndex("test",
-            Settings.builder()
-                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) // assign shards
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, zen1NodeCount + zen2NodeCount + randomIntBetween(0, 2)) // causes rebalancing
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                .build());
-        ensureGreen("test");
-
-        for (final String node : nodes) {
-            // With 1 Zen1 node when you stop the Zen1 node the Zen2 nodes might auto-bootstrap.
-            // But there are only 2 Zen2 nodes so you must do the right things with voting config exclusions to keep the cluster
-            // alive through the other two restarts.
-            final boolean masterNodeIsZen2 = zen1NodeCount <= nodes.indexOf(internalCluster().getMasterName());
-            final boolean thisNodeIsZen2 = zen1NodeCount <= nodes.indexOf(node);
-            final boolean requiresVotingConfigExclusions = zen1NodeCount == 1 && zen2NodeCount == 2 && masterNodeIsZen2 && thisNodeIsZen2;
-
-            if (requiresVotingConfigExclusions) {
-                client().execute(AddVotingConfigExclusionsAction.INSTANCE,
-                    new AddVotingConfigExclusionsRequest(new String[]{node})).get();
-            }
-
-            internalCluster().restartNode(node, new RestartCallback() {
-                @Override
-                public Settings onNodeStopped(String restartingNode) {
-                    String viaNode = randomValueOtherThan(restartingNode, () -> randomFrom(nodes));
-                    final ClusterHealthRequestBuilder clusterHealthRequestBuilder = client(viaNode).admin().cluster().prepareHealth()
-                        .setWaitForEvents(Priority.LANGUID)
-                        .setWaitForNodes(Integer.toString(zen1NodeCount + zen2NodeCount - 1))
-                        .setTimeout(TimeValue.timeValueSeconds(30));
-                    ClusterHealthResponse clusterHealthResponse = clusterHealthRequestBuilder.get();
-                    assertFalse(restartingNode, clusterHealthResponse.isTimedOut());
-                    return Settings.EMPTY;
-                }
-            });
-            ensureStableCluster(zen1NodeCount + zen2NodeCount);
-            ensureGreen("test");
-
-            if (requiresVotingConfigExclusions) {
-                final ClearVotingConfigExclusionsRequest clearVotingTombstonesRequest = new ClearVotingConfigExclusionsRequest();
-                clearVotingTombstonesRequest.setWaitForRemoval(false);
-                client().execute(ClearVotingConfigExclusionsAction.INSTANCE, clearVotingTombstonesRequest).get();
-            }
-        }
-    }
-
-    public void testRollingMigrationFromZen1ToZen2() throws Exception {
-        final int nodeCount = randomIntBetween(2, 5);
-        final List<String> zen1Nodes = internalCluster().startNodes(nodeCount, ZEN1_SETTINGS);
-
-        createIndex("test",
-            Settings.builder()
-                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) // assign shards
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, nodeCount) // causes rebalancing
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                .build());
-        ensureGreen("test");
-
-        for (final String zen1Node : zen1Nodes) {
-            logger.info("--> shutting down {}", zen1Node);
-            internalCluster().stopRandomNode(s -> NODE_NAME_SETTING.get(s).equals(zen1Node));
-
-            ensureStableCluster(nodeCount - 1);
-            if (nodeCount > 2) {
-                ensureGreen("test");
-            } else {
-                ensureYellow("test");
-            }
-
-            logger.info("--> starting replacement for {}", zen1Node);
-            final String newNode = internalCluster().startNode(ZEN2_SETTINGS);
-            ensureStableCluster(nodeCount);
-            ensureGreen("test");
-            logger.info("--> successfully replaced {} with {}", zen1Node, newNode);
-        }
-
-        assertThat(internalCluster().size(), equalTo(nodeCount));
-    }
-
-    public void testRollingUpgradeFromZen1ToZen2() throws Exception {
-        final int nodeCount = randomIntBetween(2, 5);
-        final List<String> nodes = internalCluster().startNodes(nodeCount, ZEN1_SETTINGS);
-
-        createIndex("test",
-            Settings.builder()
-                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) // assign shards
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, nodeCount) // causes rebalancing
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-                .build());
-        ensureGreen("test");
-
-        internalCluster().rollingRestart(new RestartCallback() {
-            @Override
-            public void doAfterNodes(int n, Client client) {
-                ensureGreen("test");
-            }
-
-            @Override
-            public Settings onNodeStopped(String nodeName) {
-                String viaNode = randomValueOtherThan(nodeName, () -> randomFrom(nodes));
-                final ClusterHealthRequestBuilder clusterHealthRequestBuilder = client(viaNode).admin().cluster().prepareHealth()
-                    .setWaitForEvents(Priority.LANGUID)
-                    .setWaitForNodes(Integer.toString(nodeCount - 1))
-                    .setTimeout(TimeValue.timeValueSeconds(30));
-                if (nodeCount == 2) {
-                    clusterHealthRequestBuilder.setWaitForYellowStatus();
-                } else {
-                    clusterHealthRequestBuilder.setWaitForGreenStatus();
-                }
-                ClusterHealthResponse clusterHealthResponse = clusterHealthRequestBuilder.get();
-                assertFalse(nodeName, clusterHealthResponse.isTimedOut());
-                return Coordinator.addZen1Attribute(false, Settings.builder().put(ZEN2_SETTINGS)
-                    .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES)).build();
-            }
-        });
-
-        ensureStableCluster(nodeCount);
-        ensureGreen("test");
-        assertThat(internalCluster().size(), equalTo(nodeCount));
-    }
-
-    private void testMultipleNodeMigrationFromZen1ToZen2(int nodeCount) throws Exception {
-        final List<String> oldNodes = internalCluster().startNodes(nodeCount, ZEN1_SETTINGS);
-        createIndex("test",
-            Settings.builder()
-                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.ZERO) // assign shards
-                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, nodeCount) // causes rebalancing
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, nodeCount > 1 ? 1 : 0)
-                .build());
-        ensureGreen("test");
-
-        internalCluster().startNodes(nodeCount, ZEN2_SETTINGS);
-
-        logger.info("--> updating settings to exclude old nodes");
-        client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
-            .put(CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(), String.join(",", oldNodes))).get();
-
-        logger.info("--> waiting for old nodes to be vacated");
-        waitForRelocation();
-
-        while (internalCluster().size() > nodeCount) {
-            internalCluster().stopRandomNode(settings -> oldNodes.contains(NODE_NAME_SETTING.get(settings)));
-        }
-
-        ensureGreen("test");
-    }
-
-    public void testMultipleNodeMigrationFromZen1ToZen2WithOneNode() throws Exception {
-        testMultipleNodeMigrationFromZen1ToZen2(1);
-    }
-
-    public void testMultipleNodeMigrationFromZen1ToZen2WithTwoNodes() throws Exception {
-        testMultipleNodeMigrationFromZen1ToZen2(2);
-    }
-
-    public void testMultipleNodeMigrationFromZen1ToZen2WithThreeNodes() throws Exception {
-        testMultipleNodeMigrationFromZen1ToZen2(3);
-    }
-
-    public void testFreshestMasterElectedAfterFullClusterRestart() throws Exception {
-        final List<String> nodeNames = internalCluster().startNodes(3, ZEN1_SETTINGS);
-
-        assertTrue(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
-            .put(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), Allocation.ALL)).get().isAcknowledged());
-
-        final List<NodeEnvironment> nodeEnvironments
-            = StreamSupport.stream(internalCluster().getDataOrMasterNodeInstances(NodeEnvironment.class).spliterator(), false)
-            .collect(Collectors.toList());
-
-        final boolean randomiseVersions = rarely();
-
-        internalCluster().fullRestart(new RestartCallback() {
-            int nodesStopped;
-
-            @Override
-            public Settings onNodeStopped(String nodeName) throws Exception {
-                nodesStopped += 1;
-
-                if (nodesStopped == 1) {
-                    final Client client = internalCluster().client(randomValueOtherThan(nodeName, () -> randomFrom(nodeNames)));
-
-                    assertFalse(client.admin().cluster().health(Requests.clusterHealthRequest()
-                        .waitForEvents(Priority.LANGUID)
-                        .waitForNoRelocatingShards(true)
-                        .waitForNodes("2")).actionGet().isTimedOut());
-
-                    assertTrue(client.admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
-                        .put(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), Allocation.NONE)).get().isAcknowledged());
-                }
-
-                if (nodesStopped == nodeNames.size()) {
-                    for (final NodeEnvironment nodeEnvironment : nodeEnvironments) {
-                        // The versions written by nodes following a Zen1 master cannot be trusted. Randomise them to demonstrate they are
-                        // not important.
-                        final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry());
-                        final Manifest manifest = metaStateService.loadManifestOrEmpty();
-                        assertThat(manifest.getCurrentTerm(), is(ZEN1_BWC_TERM));
-                        final long newVersion = randomiseVersions ? randomNonNegativeLong() : 0L;
-                        metaStateService.writeManifestAndCleanup("altering version to " + newVersion,
-                            new Manifest(manifest.getCurrentTerm(), newVersion, manifest.getGlobalGeneration(),
-                                manifest.getIndexGenerations()));
-                    }
-                }
-
-                return Coordinator.addZen1Attribute(false, Settings.builder())
-                    .put(ZEN2_SETTINGS)
-                    .putList(INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames)
-                    .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), REMOVED_MINIMUM_MASTER_NODES)
-                    .build();
-            }
-        });
-
-        assertFalse(client().admin().cluster().health(Requests.clusterHealthRequest()
-            .waitForEvents(Priority.LANGUID)
-            .waitForNoRelocatingShards(true)
-            .waitForNodes("3")).actionGet().isTimedOut());
-
-        assertThat(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.get(
-            client().admin().cluster().state(new ClusterStateRequest()).get().getState().metaData().settings()),
-            equalTo(Allocation.NONE));
-    }
-}

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java

@@ -89,7 +89,10 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
             new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
                 IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.emptyList());
         Version minCompat = Version.CURRENT.minimumIndexCompatibilityVersion();
-        Version indexUpgraded = VersionUtils.randomVersionBetween(random(), minCompat, VersionUtils.getPreviousVersion(Version.CURRENT));
+        Version indexUpgraded = VersionUtils.randomVersionBetween(random(),
+            minCompat,
+            Version.max(minCompat, VersionUtils.getPreviousVersion(Version.CURRENT))
+        );
         Version indexCreated = Version.fromString((minCompat.major - 1) + "." + randomInt(5) + "." + randomInt(5));
         final IndexMetaData metaData = newIndexMeta("foo", Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_UPGRADED, indexUpgraded)

server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseIT.java

@@ -260,6 +260,7 @@ public class RetentionLeaseIT extends ESIntegTestCase  {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38588")
     public void testRetentionLeasesSyncOnRecovery() throws Exception {
         final int numberOfReplicas = 2 - scaledRandomIntBetween(0, 2);
         internalCluster().ensureAtLeastNumDataNodes(1 + numberOfReplicas);

server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandlerTests.java

@@ -1,77 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest.action.admin.indices;
-
-import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.FakeRestRequest;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Locale;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.hasToString;
-import static org.mockito.Mockito.mock;
-
-public class RestResizeHandlerTests extends ESTestCase {
-
-    public void testShrinkCopySettingsDeprecated() throws IOException {
-        final RestResizeHandler.RestShrinkIndexAction handler =
-                new RestResizeHandler.RestShrinkIndexAction(Settings.EMPTY, mock(RestController.class));
-        for (final String copySettings : new String[]{null, "", "true", "false"}) {
-            runTestResizeCopySettingsDeprecated(handler, "shrink", copySettings);
-        }
-    }
-
-    public void testSplitCopySettingsDeprecated() throws IOException {
-        final RestResizeHandler.RestSplitIndexAction handler =
-                new RestResizeHandler.RestSplitIndexAction(Settings.EMPTY, mock(RestController.class));
-        for (final String copySettings : new String[]{null, "", "true", "false"}) {
-            runTestResizeCopySettingsDeprecated(handler, "split", copySettings);
-        }
-    }
-
-    private void runTestResizeCopySettingsDeprecated(
-            final RestResizeHandler handler, final String resizeOperation, final String copySettings) throws IOException {
-        final FakeRestRequest.Builder builder =
-                new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY)
-                        .withParams(Collections.singletonMap("copy_settings", copySettings))
-                        .withPath(String.format(Locale.ROOT, "source/_%s/target", resizeOperation));
-        if (copySettings != null) {
-            builder.withParams(Collections.singletonMap("copy_settings", copySettings));
-        }
-        final FakeRestRequest request = builder.build();
-        if ("false".equals(copySettings)) {
-            final IllegalArgumentException e =
-                    expectThrows(IllegalArgumentException.class, () -> handler.prepareRequest(request, mock(NodeClient.class)));
-            assertThat(e, hasToString(containsString("parameter [copy_settings] can not be explicitly set to [false]")));
-        } else {
-            handler.prepareRequest(request, mock(NodeClient.class));
-            if ("".equals(copySettings) || "true".equals(copySettings)) {
-                assertWarnings("parameter [copy_settings] is deprecated and will be removed in 8.0.0");
-            }
-        }
-    }
-
-}

test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java

@@ -52,7 +52,7 @@ public class VersionUtils {
         // this breaks b/c 5.x is still in version list but master doesn't care about it!
         //assert majorVersions.size() == 2;
         // TODO: remove oldVersions, we should only ever have 2 majors in Version
-        List<Version> oldVersions = majorVersions.getOrDefault((int)current.major - 2, Collections.emptyList());
+        List<List<Version>> oldVersions = splitByMinor(majorVersions.getOrDefault((int)current.major - 2, Collections.emptyList()));
         List<List<Version>> previousMajor = splitByMinor(majorVersions.get((int)current.major - 1));
         List<List<Version>> currentMajor = splitByMinor(majorVersions.get((int)current.major));
 
@@ -78,12 +78,21 @@ public class VersionUtils {
                 moveLastToUnreleased(stableVersions, unreleasedVersions);
             }
             // remove the next bugfix
-            moveLastToUnreleased(stableVersions, unreleasedVersions);
+            if (stableVersions.isEmpty() == false) {
+                moveLastToUnreleased(stableVersions, unreleasedVersions);
+            }
         }
 
-        List<Version> releasedVersions = Stream.concat(oldVersions.stream(),
-            Stream.concat(previousMajor.stream(), currentMajor.stream()).flatMap(List::stream))
-            .collect(Collectors.toList());
+        // If none of the previous major was released, then the last minor and bugfix of the old version was not released either.
+        if (previousMajor.isEmpty()) {
+            assert currentMajor.isEmpty() : currentMajor;
+            // minor of the old version is being staged
+            moveLastToUnreleased(oldVersions, unreleasedVersions);
+            // bugix of the old version is also being staged
+            moveLastToUnreleased(oldVersions, unreleasedVersions);
+        }
+        List<Version> releasedVersions = Stream.of(oldVersions, previousMajor, currentMajor)
+            .flatMap(List::stream).flatMap(List::stream).collect(Collectors.toList());
         Collections.sort(unreleasedVersions); // we add unreleased out of order, so need to sort here
         return new Tuple<>(Collections.unmodifiableList(releasedVersions), Collections.unmodifiableList(unreleasedVersions));
     }

test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java

@@ -2387,7 +2387,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                 assertEquals(1, transportStats.getRxCount());
                 assertEquals(1, transportStats.getTxCount());
                 assertEquals(25, transportStats.getRxSize().getBytes());
-                assertEquals(50, transportStats.getTxSize().getBytes());
+                assertEquals(51, transportStats.getTxSize().getBytes());
             });
             serviceC.sendRequest(connection, "internal:action", new TestRequest("hello world"), TransportRequestOptions.EMPTY,
                 transportResponseHandler);
@@ -2397,7 +2397,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                 assertEquals(1, transportStats.getRxCount());
                 assertEquals(2, transportStats.getTxCount());
                 assertEquals(25, transportStats.getRxSize().getBytes());
-                assertEquals(106, transportStats.getTxSize().getBytes());
+                assertEquals(107, transportStats.getTxSize().getBytes());
             });
             sendResponseLatch.countDown();
             responseLatch.await();
@@ -2405,7 +2405,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
             assertEquals(2, stats.getRxCount());
             assertEquals(2, stats.getTxCount());
             assertEquals(46, stats.getRxSize().getBytes());
-            assertEquals(106, stats.getTxSize().getBytes());
+            assertEquals(107, stats.getTxSize().getBytes());
         } finally {
             serviceC.close();
         }
@@ -2502,7 +2502,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                 assertEquals(1, transportStats.getRxCount());
                 assertEquals(1, transportStats.getTxCount());
                 assertEquals(25, transportStats.getRxSize().getBytes());
-                assertEquals(50, transportStats.getTxSize().getBytes());
+                assertEquals(51, transportStats.getTxSize().getBytes());
             });
             serviceC.sendRequest(connection, "internal:action", new TestRequest("hello world"), TransportRequestOptions.EMPTY,
                 transportResponseHandler);
@@ -2512,7 +2512,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                 assertEquals(1, transportStats.getRxCount());
                 assertEquals(2, transportStats.getTxCount());
                 assertEquals(25, transportStats.getRxSize().getBytes());
-                assertEquals(106, transportStats.getTxSize().getBytes());
+                assertEquals(107, transportStats.getTxSize().getBytes());
             });
             sendResponseLatch.countDown();
             responseLatch.await();
@@ -2527,7 +2527,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
             // 49 bytes are the non-exception message bytes that have been received. It should include the initial
             // handshake message and the header, version, etc bytes in the exception message.
             assertEquals(failedMessage, 49 + streamOutput.bytes().length(), stats.getRxSize().getBytes());
-            assertEquals(106, stats.getTxSize().getBytes());
+            assertEquals(107, stats.getTxSize().getBytes());
         } finally {
             serviceC.close();
         }

test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java

@@ -20,49 +20,23 @@
 package org.elasticsearch.upgrades;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.client.Request;
 import org.elasticsearch.common.Booleans;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.junit.Before;
-
-import java.io.IOException;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.hamcrest.Matchers.equalTo;
 
 public abstract class AbstractFullClusterRestartTestCase extends ESRestTestCase {
 
     private final boolean runningAgainstOldCluster = Booleans.parseBoolean(System.getProperty("tests.is_old_cluster"));
 
-    @Before
-    public void init() throws IOException {
-        assertThat("we don't need this branch if we aren't compatible with 6.0",
-                Version.CURRENT.minimumIndexCompatibilityVersion().onOrBefore(Version.V_6_0_0), equalTo(true));
-        if (isRunningAgainstOldCluster() && getOldClusterVersion().before(Version.V_7_0_0)) {
-            XContentBuilder template = jsonBuilder();
-            template.startObject();
-            {
-                template.field("index_patterns", "*");
-                template.field("order", "0");
-                template.startObject("settings");
-                template.field("number_of_shards", 5);
-                template.endObject();
-            }
-            template.endObject();
-            Request createTemplate = new Request("PUT", "/_template/template");
-            createTemplate.setJsonEntity(Strings.toString(template));
-            client().performRequest(createTemplate);
-        }
-    }
-
     public final boolean isRunningAgainstOldCluster() {
         return runningAgainstOldCluster;
     }
 
     private final Version oldClusterVersion = Version.fromString(System.getProperty("tests.old_cluster_version"));
 
+    public final boolean isRunningAgainstAncientCluster() {
+        return isRunningAgainstOldCluster() && oldClusterVersion.before(Version.V_7_0_0);
+    }
+
     public final Version getOldClusterVersion() {
         return oldClusterVersion;
     }

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherIndexingListener.java

@@ -32,7 +32,6 @@ import org.elasticsearch.xpack.watcher.watch.WatchStoreUtils;
 
 import java.io.IOException;
 import java.time.Clock;
-import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
@@ -102,7 +101,7 @@ final class WatcherIndexingListener implements IndexingOperationListener, Cluste
     @Override
     public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
         if (isWatchDocument(shardId.getIndexName(), operation.type())) {
-            ZonedDateTime now = Instant.ofEpochMilli(clock.millis()).atZone(ZoneOffset.UTC);
+            ZonedDateTime now = clock.instant().atZone(ZoneOffset.UTC);
             try {
                 Watch watch = parser.parseWithSecrets(operation.id(), true, operation.source(), now, XContentType.JSON,
                     operation.getIfSeqNo(), operation.getIfPrimaryTerm());

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionTests.java

@@ -36,6 +36,7 @@ import org.mockito.ArgumentCaptor;
 
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -275,6 +276,7 @@ public class IndexActionTests extends ESTestCase {
                 fieldName + "] or [ctx.payload._doc." + fieldName + "]"));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38581")
     public void testIndexActionExecuteSingleDoc() throws Exception {
         boolean customId = randomBoolean();
         boolean docIdAsParam = customId && randomBoolean();
@@ -324,8 +326,9 @@ public class IndexActionTests extends ESTestCase {
         assertThat(indexRequest.getRefreshPolicy(), is(expectedRefreshPolicy));
 
         if (timestampField != null) {
+            final DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME;
             assertThat(indexRequest.sourceAsMap().keySet(), is(hasSize(2)));
-            assertThat(indexRequest.sourceAsMap(), hasEntry(timestampField, executionTime.toString()));
+            assertThat(indexRequest.sourceAsMap(), hasEntry(timestampField, formatter.format(executionTime)));
         } else {
             assertThat(indexRequest.sourceAsMap().keySet(), is(hasSize(1)));
         }

x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java

@@ -32,6 +32,7 @@ import org.elasticsearch.xpack.watcher.support.search.WatcherSearchTemplateReque
 import org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule;
 import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleTrigger;
 import org.hamcrest.Matcher;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -60,6 +61,13 @@ import static org.hamcrest.Matchers.startsWith;
 
 public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
 
+    private String type;
+
+    @Before
+    public void setType() {
+        type = getOldClusterVersion().before(Version.V_6_7_0) ? "doc" : "_doc";
+    }
+
     @Override
     protected Settings restClientSettings() {
         String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
@@ -76,7 +84,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
      * Tests that a single document survives. Super basic smoke test.
      */
     public void testSingleDoc() throws IOException {
-        String docLocation = "/testsingledoc/doc/1";
+        String docLocation = "/testsingledoc/" + type + "/1";
         String doc = "{\"test\": \"test\"}";
 
         if (isRunningAgainstOldCluster()) {
@@ -87,7 +95,9 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         }
 
         Request getRequest = new Request("GET", docLocation);
-        getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        if (getOldClusterVersion().before(Version.V_6_7_0)) {
+            getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        }
         assertThat(toStr(client().performRequest(getRequest)), containsString(doc));
     }
 
@@ -148,17 +158,21 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
     public void testWatcher() throws Exception {
         if (isRunningAgainstOldCluster()) {
             logger.info("Adding a watch on old cluster {}", getOldClusterVersion());
-            Request createBwcWatch = new Request("PUT", "/_xpack/watcher/watch/bwc_watch");
+            Request createBwcWatch = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_watch");
+            Request createBwcThrottlePeriod = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_throttle_period");
+            if (getOldClusterVersion().onOrAfter(Version.V_7_0_0)) {
+                createBwcWatch.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE));
+                createBwcThrottlePeriod.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE));
+            }
             createBwcWatch.setJsonEntity(loadWatch("simple-watch.json"));
             client().performRequest(createBwcWatch);
 
             logger.info("Adding a watch with \"fun\" throttle periods on old cluster");
-            Request createBwcThrottlePeriod = new Request("PUT", "_xpack/watcher/watch/bwc_throttle_period");
             createBwcThrottlePeriod.setJsonEntity(loadWatch("throttle-period-watch.json"));
             client().performRequest(createBwcThrottlePeriod);
 
             logger.info("Adding a watch with \"fun\" read timeout on old cluster");
-            Request createFunnyTimeout = new Request("PUT", "_xpack/watcher/watch/bwc_funny_timeout");
+            Request createFunnyTimeout = new Request("PUT", getWatcherEndpoint() + "/watch/bwc_funny_timeout");
             createFunnyTimeout.setJsonEntity(loadWatch("funny-timeout-watch.json"));
             client().performRequest(createFunnyTimeout);
 
@@ -246,7 +260,11 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             // index documents for the rollup job
             final StringBuilder bulk = new StringBuilder();
             for (int i = 0; i < numDocs; i++) {
-                bulk.append("{\"index\":{\"_index\":\"rollup-docs\",\"_type\":\"doc\"}}\n");
+                if (getOldClusterVersion().onOrAfter(Version.V_7_0_0)) {
+                    bulk.append("{\"index\":{\"_index\":\"rollup-docs\"}}\n");
+                } else {
+                    bulk.append("{\"index\":{\"_index\":\"rollup-docs\",\"_type\":\"doc\"}}\n");
+                }
                 String date = String.format(Locale.ROOT, "%04d-01-01T00:%02d:00Z", year, i);
                 bulk.append("{\"timestamp\":\"").append(date).append("\",\"value\":").append(i).append("}\n");
             }
@@ -257,7 +275,8 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             client().performRequest(bulkRequest);
 
             // create the rollup job
-            final Request createRollupJobRequest = new Request("PUT", "/_xpack/rollup/job/rollup-job-test");
+            final Request createRollupJobRequest = new Request("PUT", getRollupEndpoint() + "/job/rollup-job-test");
+
             createRollupJobRequest.setJsonEntity("{"
                     + "\"index_pattern\":\"rollup-*\","
                     + "\"rollup_index\":\"results-rollup\","
@@ -278,7 +297,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE));
 
             // start the rollup job
-            final Request startRollupJobRequest = new Request("POST", "/_xpack/rollup/job/rollup-job-test/_start");
+            final Request startRollupJobRequest = new Request("POST", getRollupEndpoint() + "/job/rollup-job-test/_start");
             Map<String, Object> startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest));
             assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE));
 
@@ -304,12 +323,12 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         assumeTrue("Rollup ID scheme changed in 6.4", getOldClusterVersion().before(Version.V_6_4_0));
         if (isRunningAgainstOldCluster()) {
 
-            final Request indexRequest = new Request("POST", "/id-test-rollup/doc/1");
+            final Request indexRequest = new Request("POST", "/id-test-rollup" + type + "/1");
             indexRequest.setJsonEntity("{\"timestamp\":\"2018-01-01T00:00:01\",\"value\":123}");
             client().performRequest(indexRequest);
 
             // create the rollup job
-            final Request createRollupJobRequest = new Request("PUT", "/_xpack/rollup/job/rollup-id-test");
+            final Request createRollupJobRequest = new Request("PUT", getRollupEndpoint() + "/job/rollup-id-test");
             createRollupJobRequest.setJsonEntity("{"
                 + "\"index_pattern\":\"id-test-rollup\","
                 + "\"rollup_index\":\"id-test-results-rollup\","
@@ -337,7 +356,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE));
 
             // start the rollup job
-            final Request startRollupJobRequest = new Request("POST", "/_xpack/rollup/job/rollup-id-test/_start");
+            final Request startRollupJobRequest = new Request("POST", getRollupEndpoint() + "/job/rollup-id-test/_start");
             Map<String, Object> startRollupJobResponse = entityAsMap(client().performRequest(startRollupJobRequest));
             assertThat(startRollupJobResponse.get("started"), equalTo(Boolean.TRUE));
 
@@ -365,9 +384,11 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
 
         } else {
 
-            final Request indexRequest = new Request("POST", "/id-test-rollup/doc/2");
+            final Request indexRequest = new Request("POST", "/id-test-rollup/" + type + "/2");
             indexRequest.setJsonEntity("{\"timestamp\":\"2018-01-02T00:00:01\",\"value\":345}");
-            indexRequest.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
+            if (getOldClusterVersion().before(Version.V_6_7_0)) {
+                indexRequest.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE));
+            }
             client().performRequest(indexRequest);
 
             assertRollUpJob("rollup-id-test");
@@ -431,12 +452,8 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             client().performRequest(doc2);
             return;
         }
-        final Request sqlRequest;
-        if (isRunningAgainstOldCluster()) {
-            sqlRequest = new Request("POST", "/_xpack/sql");
-        } else {
-            sqlRequest = new Request("POST", "/_sql");
-        }
+        final Request sqlRequest = new Request("POST", getSQLEndpoint());
+
         sqlRequest.setJsonEntity("{\"query\":\"SELECT * FROM testsqlfailsonindexwithtwotypes\"}");
         ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(sqlRequest));
         assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
@@ -458,8 +475,21 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
     private void assertWatchIndexContentsWork() throws Exception {
         // Fetch a basic watch
         Request getRequest = new Request("GET", "_watcher/watch/bwc_watch");
-        getRequest.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE,
-            WatcherSearchTemplateRequest.TYPES_DEPRECATION_MESSAGE));
+        if (getOldClusterVersion().before(Version.V_7_0_0)) {
+            getRequest.setOptions(
+                expectWarnings(
+                    IndexAction.TYPES_DEPRECATION_MESSAGE,
+                    WatcherSearchTemplateRequest.TYPES_DEPRECATION_MESSAGE
+                )
+            );
+        } else {
+            getRequest.setOptions(
+                expectWarnings(
+                    IndexAction.TYPES_DEPRECATION_MESSAGE
+                )
+            );
+        }
+
         Map<String, Object> bwcWatch = entityAsMap(client().performRequest(getRequest));
 
         logger.error("-----> {}", bwcWatch);
@@ -475,8 +505,20 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
 
         // Fetch a watch with "fun" throttle periods
         getRequest = new Request("GET", "_watcher/watch/bwc_throttle_period");
-        getRequest.setOptions(expectWarnings(IndexAction.TYPES_DEPRECATION_MESSAGE,
-            WatcherSearchTemplateRequest.TYPES_DEPRECATION_MESSAGE));
+        if (getOldClusterVersion().before(Version.V_7_0_0)) {
+            getRequest.setOptions(
+                expectWarnings(
+                    IndexAction.TYPES_DEPRECATION_MESSAGE,
+                    WatcherSearchTemplateRequest.TYPES_DEPRECATION_MESSAGE
+                )
+            );
+        } else {
+            getRequest.setOptions(
+                expectWarnings(
+                    IndexAction.TYPES_DEPRECATION_MESSAGE
+                )
+            );
+        }
         bwcWatch = entityAsMap(client().performRequest(getRequest));
         assertThat(bwcWatch.get("found"), equalTo(true));
         source = (Map<String, Object>) bwcWatch.get("watch");
@@ -560,7 +602,13 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
             try {
                 Map<String, Object> response = entityAsMap(client().performRequest(request));
                 Map<String, Object> hits = (Map<String, Object>) response.get("hits");
-                int total = (int) hits.get("total");
+                logger.info("Hits are: {}", hits);
+                int total;
+                if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) {
+                    total = (int) ((Map<String, Object>) hits.get("total")).get("value");
+                } else {
+                    total = (int) hits.get("total");
+                }
                 assertThat(total, greaterThanOrEqualTo(expectedHits));
             } catch (IOException ioe) {
                 if (ioe instanceof ResponseException) {
@@ -580,12 +628,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
 
     private void createUser(final boolean oldCluster) throws Exception {
         final String id = oldCluster ? "preupgrade_user" : "postupgrade_user";
-        Request request;
-        if (oldCluster) {
-            request = new Request("PUT", "/_xpack/security/user/" + id);
-        } else {
-            request = new Request("PUT", "/_security/user/" + id);
-        }
+        Request request = new Request("PUT", getSecurityEndpoint() + "/user/" + id);
         request.setJsonEntity(
             "{\n" +
             "   \"password\" : \"j@rV1s\",\n" +
@@ -599,12 +642,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
 
     private void createRole(final boolean oldCluster) throws Exception {
         final String id = oldCluster ? "preupgrade_role" : "postupgrade_role";
-        Request request;
-        if (oldCluster) {
-            request = new Request("PUT", "/_xpack/security/role/" + id);
-        } else {
-            request = new Request("PUT", "/_security/role/" + id);
-        }
+        Request request = new Request("PUT", getSecurityEndpoint() + "/role/" + id);
         request.setJsonEntity(
             "{\n" +
             "  \"run_as\": [ \"abc\" ],\n" +
@@ -625,20 +663,59 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
 
     private void assertUserInfo(final boolean oldCluster) throws Exception {
         final String user = oldCluster ? "preupgrade_user" : "postupgrade_user";
-        Map<String, Object> response = oldCluster ?
-            entityAsMap(client().performRequest(new Request("GET", "/_xpack/security/user/" + user))) :
-            entityAsMap(client().performRequest(new Request("GET", "/_security/user/" + user)));
+        Request request = new Request("GET", getSecurityEndpoint() + "/user/" + user);;
+        Map<String, Object> response = entityAsMap(client().performRequest(request));
         @SuppressWarnings("unchecked") Map<String, Object> userInfo = (Map<String, Object>) response.get(user);
         assertEquals(user + "@example.com", userInfo.get("email"));
         assertNotNull(userInfo.get("full_name"));
         assertNotNull(userInfo.get("roles"));
     }
 
+    private String getSecurityEndpoint() {
+        String securityEndpoint;
+        if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) {
+            securityEndpoint = "/_security";
+        } else {
+            securityEndpoint = "/_xpack/security";
+        }
+        return securityEndpoint;
+    }
+
+    private String getSQLEndpoint() {
+        String securityEndpoint;
+        if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) {
+            securityEndpoint = "/_sql";
+        } else {
+            securityEndpoint = "/_xpack/sql";
+        }
+        return securityEndpoint;
+    }
+
+    private String getRollupEndpoint() {
+        String securityEndpoint;
+        if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) {
+            securityEndpoint = "/_rollup";
+        } else {
+            securityEndpoint = "/_xpack/rollup";
+        }
+        return securityEndpoint;
+    }
+
+    private String getWatcherEndpoint() {
+        String securityEndpoint;
+        if (getOldClusterVersion().onOrAfter(Version.V_7_0_0) || isRunningAgainstOldCluster() == false) {
+            securityEndpoint = "/_watcher";
+        } else {
+            securityEndpoint = "/_xpack/watcher";
+        }
+        return securityEndpoint;
+    }
+
     private void assertRoleInfo(final boolean oldCluster) throws Exception {
         final String role = oldCluster ? "preupgrade_role" : "postupgrade_role";
-        @SuppressWarnings("unchecked") Map<String, Object> response = oldCluster ?
-            (Map<String, Object>) entityAsMap(client().performRequest(new Request("GET", "/_xpack/security/role/" + role))).get(role) :
-            (Map<String, Object>) entityAsMap(client().performRequest(new Request("GET", "/_security/role/" + role))).get(role);
+        @SuppressWarnings("unchecked") Map<String, Object> response = (Map<String, Object>) entityAsMap(
+            client().performRequest(new Request("GET", getSecurityEndpoint() + "/role/" + role))
+        ).get(role);
         assertNotNull(response.get("run_as"));
         assertNotNull(response.get("cluster"));
         assertNotNull(response.get("indices"));
@@ -650,12 +727,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
         waitForRollUpJob(rollupJob, expectedStates);
 
         // check that the rollup job is started using the RollUp API
-        final Request getRollupJobRequest;
-        if (isRunningAgainstOldCluster()) {
-            getRollupJobRequest = new Request("GET", "/_xpack/rollup/job/" + rollupJob);
-        } else {
-            getRollupJobRequest = new Request("GET", "/_rollup/job/" + rollupJob);
-        }
+        final Request getRollupJobRequest = new Request("GET", getRollupEndpoint() + "/job/" + rollupJob);
         Map<String, Object> getRollupJobResponse = entityAsMap(client().performRequest(getRollupJobRequest));
         Map<String, Object> job = getJob(getRollupJobResponse, rollupJob);
         assertNotNull(job);
@@ -700,12 +772,8 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase {
 
     private void waitForRollUpJob(final String rollupJob, final Matcher<?> expectedStates) throws Exception {
         assertBusy(() -> {
-            final Request getRollupJobRequest;
-            if (isRunningAgainstOldCluster()) {
-                getRollupJobRequest = new Request("GET", "/_xpack/rollup/job/" + rollupJob);
-            } else {
-                getRollupJobRequest = new Request("GET", "/_rollup/job/" + rollupJob);
-            }
+            final Request getRollupJobRequest = new Request("GET", getRollupEndpoint() + "/job/" + rollupJob);
+
             Response getRollupJobResponse = client().performRequest(getRollupJobRequest);
             assertThat(getRollupJobResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
 

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java

@@ -6,20 +6,15 @@
 package org.elasticsearch.upgrades;
 
 import org.apache.http.util.EntityUtils;
-import org.elasticsearch.common.Booleans;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.rest.action.document.RestBulkAction;
-import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.common.Booleans;
+import org.elasticsearch.rest.action.document.RestBulkAction;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
-import static org.hamcrest.Matchers.equalTo;
 
 /**
  * Basic test that indexed documents survive the rolling restart.
@@ -53,25 +48,6 @@ public class IndexingIT extends AbstractUpgradeTestCase {
         }
 
         if (CLUSTER_TYPE == ClusterType.OLD) {
-            {
-                Version minimumIndexCompatibilityVersion = Version.CURRENT.minimumIndexCompatibilityVersion();
-                assertThat("this branch is not needed if we aren't compatible with 6.0",
-                        minimumIndexCompatibilityVersion.onOrBefore(Version.V_6_0_0), equalTo(true));
-                if (minimumIndexCompatibilityVersion.before(Version.V_7_0_0)) {
-                    XContentBuilder template = jsonBuilder();
-                    template.startObject();
-                    {
-                        template.field("index_patterns", "*");
-                        template.startObject("settings");
-                        template.field("number_of_shards", 5);
-                        template.endObject();
-                    }
-                    template.endObject();
-                    Request createTemplate = new Request("PUT", "/_template/template");
-                    createTemplate.setJsonEntity(Strings.toString(template));
-                    client().performRequest(createTemplate);
-                }
-            }
 
             Request createTestIndex = new Request("PUT", "/test_index");
             createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}");

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TokenBackwardsCompatibilityIT.java

@@ -13,9 +13,6 @@ import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.rest.action.document.RestGetAction;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
 
 import java.io.IOException;
@@ -23,32 +20,10 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.hamcrest.Matchers.equalTo;
-
 public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
 
     public void testGeneratingTokenInOldCluster() throws Exception {
         assumeTrue("this test should only run against the old cluster", CLUSTER_TYPE == ClusterType.OLD);
-        {
-            Version minimumIndexCompatibilityVersion = Version.CURRENT.minimumIndexCompatibilityVersion();
-            assertThat("this branch is not needed if we aren't compatible with 6.0",
-                    minimumIndexCompatibilityVersion.onOrBefore(Version.V_6_0_0), equalTo(true));
-            if (minimumIndexCompatibilityVersion.before(Version.V_7_0_0)) {
-                XContentBuilder template = jsonBuilder();
-                template.startObject();
-                {
-                    template.field("index_patterns", "*");
-                    template.startObject("settings");
-                    template.field("number_of_shards", 5);
-                    template.endObject();
-                }
-                template.endObject();
-                Request createTemplate = new Request("PUT", "/_template/template");
-                createTemplate.setJsonEntity(Strings.toString(template));
-                client().performRequest(createTemplate);
-            }
-        }
 
         Request createTokenRequest = new Request("POST", "/_security/oauth2/token");
         createTokenRequest.setJsonEntity(
@@ -64,7 +39,7 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
         assertNotNull(token);
         assertTokenWorks(token);
 
-        Request indexRequest1 = new Request("PUT", "token_backwards_compatibility_it/doc/old_cluster_token1");
+        Request indexRequest1 = new Request("PUT", "token_backwards_compatibility_it/_doc/old_cluster_token1");
         indexRequest1.setJsonEntity(
                 "{\n" +
                 "    \"token\": \"" + token + "\"\n" +
@@ -78,7 +53,7 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
         token = (String) responseMap.get("access_token");
         assertNotNull(token);
         assertTokenWorks(token);
-        Request indexRequest2 = new Request("PUT", "token_backwards_compatibility_it/doc/old_cluster_token2");
+        Request indexRequest2 = new Request("PUT", "token_backwards_compatibility_it/_doc/old_cluster_token2");
         indexRequest2.setJsonEntity(
                 "{\n" +
                 "    \"token\": \"" + token + "\"\n" +
@@ -89,8 +64,7 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
     public void testTokenWorksInMixedOrUpgradedCluster() throws Exception {
         assumeTrue("this test should only run against the mixed or upgraded cluster",
                 CLUSTER_TYPE == ClusterType.MIXED || CLUSTER_TYPE == ClusterType.UPGRADED);
-        Request getRequest = new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token1");
-        getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token1");
         Response getResponse = client().performRequest(getRequest);
         assertOK(getResponse);
         Map<String, Object> source = (Map<String, Object>) entityAsMap(getResponse).get("_source");
@@ -100,8 +74,7 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
     public void testMixedCluster() throws Exception {
         assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.MIXED);
         assumeTrue("the master must be on the latest version before we can write", isMasterOnLatestVersion());
-        Request getRequest = new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token2");
-        getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token2");
 
         Response getResponse = client().performRequest(getRequest);
         Map<String, Object> source = (Map<String, Object>) entityAsMap(getResponse).get("_source");
@@ -152,8 +125,7 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
 
     public void testUpgradedCluster() throws Exception {
         assumeTrue("this test should only run against the mixed cluster", CLUSTER_TYPE == ClusterType.UPGRADED);
-        Request getRequest = new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token2");
-        getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        Request getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token2");
 
         Response getResponse = client().performRequest(getRequest);
         assertOK(getResponse);
@@ -168,8 +140,7 @@ public class TokenBackwardsCompatibilityIT extends AbstractUpgradeTestCase {
         assertOK(invalidationResponse);
         assertTokenDoesNotWork(token);
 
-        getRequest = new Request("GET", "token_backwards_compatibility_it/doc/old_cluster_token1");
-        getRequest.setOptions(expectWarnings(RestGetAction.TYPES_DEPRECATION_MESSAGE));
+        getRequest = new Request("GET", "token_backwards_compatibility_it/_doc/old_cluster_token1");
 
         getResponse = client().performRequest(getRequest);
         source = (Map<String, Object>) entityAsMap(getResponse).get("_source");

x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml

@@ -28,7 +28,6 @@
           {
             "job_id":"old-cluster-datafeed-job",
             "indices":["airline-data"],
-            "types":["response"],
             "scroll_size": 2000
           }